Spellcheck

.vscode/settings.json

@@ -18,5 +18,8 @@
     "[json]": {
         "editor.defaultFormatter": "esbenp.prettier-vscode"
     },
-    "editor.formatOnSave": true
+    "editor.formatOnSave": true,
+    "cSpell.words": [
+        "nessicary"
+    ]
 }

messages/en-US.json

@@ -66,9 +66,15 @@
     "local": "Local",
     "edit": "Edit",
     "siteConfirmDelete": "Confirm Delete Site",
+    "siteConfirmDeleteAndResources": "Confirm Delete Site and Resources",
     "siteDelete": "Delete Site",
-    "siteMessageRemove": "Once removed the site will no longer be accessible. All targets associated with the site will also be removed.",
+    "siteDeleteAndResources": "Delete Site and Resources",
+    "siteMessageRemove": "Once removed the site will no longer be accessible. Targets associated with this site will be removed, but resources will remain.",
+    "siteMessageRemoveAndResources": "This will permanently delete all public and private resources linked to this site, even if a resource is also associated with other sites.",
     "siteQuestionRemove": "Are you sure you want to remove the site from the organization?",
+    "siteQuestionRemoveAndResources": "Are you sure you want to delete this site and all associated resources?",
+    "sitesTableDeleteSite": "Delete Site",
+    "sitesTableDeleteSiteAndResources": "Delete Site and Resources",
     "siteManageSites": "Manage Sites",
     "siteDescription": "Create and manage sites to enable connectivity to private networks",
     "sitesBannerTitle": "Connect Any Network",
@@ -204,7 +210,7 @@
     "proxyResourceTitle": "Manage Public Resources",
     "proxyResourceDescription": "Create and manage resources that are publicly accessible through a web browser",
     "publicResourcesBannerTitle": "Web-based Public Access",
-    "publicResourcesBannerDescription": "Public resources are HTTPS proxies accessible to anyone on the internet through a web browser. Unlike private resources, they do not require client-side software and can include identity and context-aware access policies.",
+    "publicResourcesBannerDescription": "Public resources are proxies accessible to anyone on the internet through a web browser and include identity and context-aware access policies. Unlike private resources, they do not require client-side software.",
     "clientResourceTitle": "Manage Private Resources",
     "clientResourceDescription": "Create and manage resources that are only accessible through a connected client",
     "privateResourcesBannerTitle": "Zero-Trust Private Access",
@@ -1638,7 +1644,7 @@
     "alertingActionType": "Action type",
     "alertingNotifyUsers": "Users",
     "alertingNotifyRoles": "Roles",
-    "alertingNotifyEmails": "Email addresses",
+    "alertingNotifyEmails": "Email Addresses",
     "alertingEmailPlaceholder": "Add email and press Enter",
     "alertingWebhookMethod": "HTTP method",
     "alertingWebhookSecret": "Signing secret (optional)",
@@ -2171,10 +2177,10 @@
     "sshSudoModeCommandsDescription": "User can run only the specified commands with sudo.",
     "sshSudo": "Allow sudo",
     "sshSudoCommands": "Sudo Commands",
-    "sshSudoCommandsDescription": "List of commands the user is allowed to run with sudo, separated by commas, spaces, or new lines. Absolute paths must be used.",
+    "sshSudoCommandsDescription": "List of commands the user is allowed to run with sudo, one per line. Absolute paths must be used.",
     "sshCreateHomeDir": "Create Home Directory",
     "sshUnixGroups": "Unix Groups",
-    "sshUnixGroupsDescription": "Unix groups to add the user to on the target host, separated by commas, spaces, or new lines.",
+    "sshUnixGroupsDescription": "Unix groups to add the user to on the target host, one per line.",
     "roleTextFieldPlaceholder": "Enter values, or drop a .txt or .csv file",
     "roleTextImportTitle": "Import from File",
     "roleTextImportDescription": "Importing {fileName} into {fieldLabel}.",

server/auth/sessions/app.ts

@@ -12,7 +12,7 @@ import {
     users
 } from "@server/db";
 import { db } from "@server/db";
-import { eq, inArray } from "drizzle-orm";
+import { and, eq, inArray, ne } from "drizzle-orm";
 import config from "@server/lib/config";
 import type { RandomReader } from "@oslojs/crypto/random";
 import { generateRandomString } from "@oslojs/crypto/random";
@@ -136,6 +136,45 @@ export async function invalidateAllSessions(userId: string): Promise<void> {
     }
 }
 
+export async function invalidateAllSessionsExceptCurrent(
+    userId: string,
+    currentSessionId: string
+): Promise<void> {
+    try {
+        await db.transaction(async (trx) => {
+            const userSessions = await trx
+                .select()
+                .from(sessions)
+                .where(
+                    and(
+                        eq(sessions.userId, userId),
+                        ne(sessions.sessionId, currentSessionId)
+                    )
+                );
+
+            if (userSessions.length > 0) {
+                await trx.delete(resourceSessions).where(
+                    inArray(
+                        resourceSessions.userSessionId,
+                        userSessions.map((s) => s.sessionId)
+                    )
+                );
+            }
+
+            await trx
+                .delete(sessions)
+                .where(
+                    and(
+                        eq(sessions.userId, userId),
+                        ne(sessions.sessionId, currentSessionId)
+                    )
+                );
+        });
+    } catch (e) {
+        logger.error("Failed to invalidate user sessions except current", e);
+    }
+}
+
 export function serializeSessionCookie(
     token: string,
     isSecure: boolean,

server/lib/blueprints/applyBlueprint.ts

@@ -29,8 +29,11 @@ import { updateResourcePolicies } from "./resourcePolicies";
 import { BlueprintSource } from "@server/routers/blueprints/types";
 import { stringify as stringifyYaml } from "yaml";
 import { generateName } from "@server/db/names";
-import { handleMessagingForUpdatedSiteResource } from "@server/routers/siteResource";
-import { rebuildClientAssociationsFromSiteResource } from "../rebuildClientAssociations";
+import {
+    handleMessagingForUpdatedSiteResource,
+    rebuildClientAssociationsFromSiteResource,
+    waitForSiteResourceRebuildIdle
+} from "../rebuildClientAssociations";
 
 type ApplyBlueprintArgs = {
     orgId: string;
@@ -138,26 +141,25 @@ export async function applyBlueprint({
             for (const result of privateResourcesResults) {
                 rebuildClientAssociationsFromSiteResource(
                     result.newSiteResource
-                ).catch((e) => {
-                    logger.error(
-                        `Failed to rebuild client associations for site resource ${result.newSiteResource.siteResourceId}. Error: ${e}`
-                    );
-                });
-
-                handleMessagingForUpdatedSiteResource(
-                    result.oldSiteResource,
-                    result.newSiteResource,
-                    result.oldSites.map((site) => ({
-                        // only need to run this on the old sites because the new sites are added above
-                        siteId: site.siteId,
-                        orgId: result.newSiteResource.orgId
-                    }))
-                ).catch((err) => {
-                    logger.error(
-                        `Error handling messaging for updated site resource ${result.newSiteResource.siteResourceId}:`,
-                        err
-                    );
-                });
+                )
+                    .then(() =>
+                        waitForSiteResourceRebuildIdle(
+                            result.newSiteResource.siteResourceId
+                        )
+                    )
+                    .then(() =>
+                        handleMessagingForUpdatedSiteResource(
+                            result.oldSiteResource,
+                            result.newSiteResource,
+                            result.oldSites.map((s) => s.siteId),
+                            result.newSites.map((s) => s.siteId)
+                        )
+                    )
+                    .catch((e) => {
+                        logger.error(
+                            `Failed to rebuild and handle messaging for site resource ${result.newSiteResource.siteResourceId}. Error: ${e}`
+                        );
+                    });
             }
 
             logger.debug(

server/lib/deleteResource.ts

@@ -0,0 +1,144 @@
+import { eq, inArray } from "drizzle-orm";
+import {
+    db,
+    newts,
+    resourcePolicies,
+    resources,
+    sites,
+    targetHealthCheck,
+    targets,
+    type Resource,
+    type Target,
+    type TargetHealthCheck,
+    type Transaction
+} from "@server/db";
+import logger from "@server/logger";
+import { removeTargets } from "@server/routers/newt/targets";
+import createHttpError from "http-errors";
+import HttpCode from "@server/types/HttpCode";
+
+export type DeleteResourceResult = {
+    deletedResource: Resource;
+    targetsToBeRemoved: Target[];
+    healthChecksToBeRemoved: TargetHealthCheck[];
+};
+
+export async function performDeleteResources(
+    resourceIds: number[],
+    trx: Transaction | typeof db = db
+): Promise<DeleteResourceResult[]> {
+    if (resourceIds.length === 0) {
+        return [];
+    }
+
+    const targetsToBeRemoved = await trx
+        .select()
+        .from(targets)
+        .where(inArray(targets.resourceId, resourceIds));
+
+    const targetIds = targetsToBeRemoved.map((t) => t.targetId);
+    const healthChecksToBeRemoved =
+        targetIds.length > 0
+            ? await trx
+                  .select()
+                  .from(targetHealthCheck)
+                  .where(inArray(targetHealthCheck.targetId, targetIds))
+            : [];
+
+    const deletedResources = await trx
+        .delete(resources)
+        .where(inArray(resources.resourceId, resourceIds))
+        .returning();
+
+    const policyIds = deletedResources
+        .map((resource) => resource.defaultResourcePolicyId)
+        .filter((id): id is number => id != null);
+
+    if (policyIds.length > 0) {
+        await trx
+            .delete(resourcePolicies)
+            .where(inArray(resourcePolicies.resourcePolicyId, policyIds));
+    }
+
+    if (deletedResources.length > 0) {
+        logger.debug(`Deleted ${deletedResources.length} resources`);
+    }
+
+    const targetsByResourceId = new Map<number, Target[]>();
+    for (const target of targetsToBeRemoved) {
+        const existing = targetsByResourceId.get(target.resourceId) ?? [];
+        existing.push(target);
+        targetsByResourceId.set(target.resourceId, existing);
+    }
+
+    const targetIdToResourceId = new Map(
+        targetsToBeRemoved.map((target) => [target.targetId, target.resourceId])
+    );
+
+    const healthChecksByResourceId = new Map<number, TargetHealthCheck[]>();
+    for (const healthCheck of healthChecksToBeRemoved) {
+        const resourceId = targetIdToResourceId.get(healthCheck.targetId!);
+        if (resourceId == null) {
+            continue;
+        }
+        const existing = healthChecksByResourceId.get(resourceId) ?? [];
+        existing.push(healthCheck);
+        healthChecksByResourceId.set(resourceId, existing);
+    }
+
+    return deletedResources.map((deletedResource) => ({
+        deletedResource,
+        targetsToBeRemoved:
+            targetsByResourceId.get(deletedResource.resourceId) ?? [],
+        healthChecksToBeRemoved:
+            healthChecksByResourceId.get(deletedResource.resourceId) ?? []
+    }));
+}
+
+export async function performDeleteResource(
+    resourceId: number,
+    trx: Transaction | typeof db = db
+): Promise<DeleteResourceResult | null> {
+    const [result] = await performDeleteResources([resourceId], trx);
+    return result ?? null;
+}
+
+export async function runResourceDeleteSideEffects(
+    result: DeleteResourceResult
+): Promise<void> {
+    const { deletedResource, targetsToBeRemoved, healthChecksToBeRemoved } =
+        result;
+
+    for (const target of targetsToBeRemoved) {
+        const [site] = await db
+            .select()
+            .from(sites)
+            .where(eq(sites.siteId, target.siteId))
+            .limit(1);
+
+        if (!site) {
+            throw createHttpError(
+                HttpCode.NOT_FOUND,
+                `Site with ID ${target.siteId} not found`
+            );
+        }
+
+        if (site.pubKey && site.type === "newt") {
+            const [newt] = await db
+                .select()
+                .from(newts)
+                .where(eq(newts.siteId, site.siteId))
+                .limit(1);
+
+            if (newt) {
+                await removeTargets(
+                    newt.newtId,
+                    [],
+                    healthChecksToBeRemoved,
+                    deletedResource.mode === "udp" ? "udp" : "tcp",
+                    newt.version
+                );
+            }
+        }
+    }
+}

server/lib/deleteSiteAssociatedResources.ts

@@ -0,0 +1,126 @@
+import { and, eq, sql } from "drizzle-orm";
+import {
+    db,
+    siteNetworks,
+    siteResources,
+    targets,
+    type SiteResource,
+    type Transaction
+} from "@server/db";
+import {
+    performDeleteResources,
+    runResourceDeleteSideEffects,
+    type DeleteResourceResult
+} from "@server/lib/deleteResource";
+import {
+    performDeleteSiteResources,
+    runSiteResourceDeleteSideEffects
+} from "@server/lib/deleteSiteResource";
+import logger from "@server/logger";
+
+export const MAX_SITE_ASSOCIATED_RESOURCES_FOR_BULK_DELETE = 250;
+
+export type DeleteSiteAssociatedResourcesSideEffects = {
+    resources: DeleteResourceResult[];
+    siteResources: SiteResource[];
+};
+
+export async function getResourceIdsForSite(
+    siteId: number,
+    trx: Transaction | typeof db = db
+): Promise<number[]> {
+    const rows = await trx
+        .selectDistinct({ resourceId: targets.resourceId })
+        .from(targets)
+        .where(eq(targets.siteId, siteId));
+
+    return rows.map((row) => row.resourceId);
+}
+
+export async function getSiteResourceIdsForSite(
+    siteId: number,
+    orgId: string,
+    trx: Transaction | typeof db = db
+): Promise<number[]> {
+    const rows = await trx
+        .selectDistinct({ siteResourceId: siteResources.siteResourceId })
+        .from(siteNetworks)
+        .innerJoin(
+            siteResources,
+            eq(siteResources.networkId, siteNetworks.networkId)
+        )
+        .where(
+            and(eq(siteNetworks.siteId, siteId), eq(siteResources.orgId, orgId))
+        );
+
+    return rows.map((row) => row.siteResourceId);
+}
+
+export async function getAssociatedResourceCountForSite(
+    siteId: number,
+    orgId: string,
+    trx: Transaction | typeof db = db
+): Promise<number> {
+    const [publicCountResult, privateCountResult] = await Promise.all([
+        trx
+            .select({
+                count: sql<number>`count(distinct ${targets.resourceId})`
+            })
+            .from(targets)
+            .where(eq(targets.siteId, siteId)),
+        trx
+            .select({
+                count: sql<number>`count(distinct ${siteResources.siteResourceId})`
+            })
+            .from(siteNetworks)
+            .innerJoin(
+                siteResources,
+                eq(siteResources.networkId, siteNetworks.networkId)
+            )
+            .where(
+                and(
+                    eq(siteNetworks.siteId, siteId),
+                    eq(siteResources.orgId, orgId)
+                )
+            )
+    ]);
+
+    return (
+        Number(publicCountResult[0]?.count ?? 0) +
+        Number(privateCountResult[0]?.count ?? 0)
+    );
+}
+
+export function exceedsSiteAssociatedResourceDeleteLimit(
+    resourceCount: number
+): boolean {
+    return resourceCount > MAX_SITE_ASSOCIATED_RESOURCES_FOR_BULK_DELETE;
+}
+
+export async function deleteAssociatedResourcesForSite(
+    siteId: number,
+    orgId: string,
+    trx: Transaction | typeof db = db
+): Promise<DeleteSiteAssociatedResourcesSideEffects> {
+    const resourceIds = await getResourceIdsForSite(siteId, trx);
+    const siteResourceIds = await getSiteResourceIdsForSite(siteId, orgId, trx);
+
+    const [resources, siteResourcesDeleted] = await Promise.all([
+        performDeleteResources(resourceIds, trx),
+        performDeleteSiteResources(siteResourceIds, trx)
+    ]);
+
+    return { resources, siteResources: siteResourcesDeleted };
+}
+
+export async function runDeleteSiteAssociatedResourcesSideEffects(
+    sideEffects: DeleteSiteAssociatedResourcesSideEffects
+): Promise<void> {
+    for (const result of sideEffects.resources) {
+        await runResourceDeleteSideEffects(result);
+    }
+
+    for (const removed of sideEffects.siteResources) {
+        runSiteResourceDeleteSideEffects(removed);
+    }
+}

server/lib/deleteSiteResource.ts

@@ -0,0 +1,53 @@
+import { inArray } from "drizzle-orm";
+import {
+    db,
+    siteResources,
+    type SiteResource,
+    type Transaction
+} from "@server/db";
+import logger from "@server/logger";
+import { rebuildClientAssociationsFromSiteResource } from "@server/lib/rebuildClientAssociations";
+
+export async function performDeleteSiteResources(
+    siteResourceIds: number[],
+    trx: Transaction | typeof db = db
+): Promise<SiteResource[]> {
+    if (siteResourceIds.length === 0) {
+        return [];
+    }
+
+    const removedSiteResources = await trx
+        .delete(siteResources)
+        .where(inArray(siteResources.siteResourceId, siteResourceIds))
+        .returning();
+
+    if (removedSiteResources.length > 0) {
+        logger.debug(`Deleted ${removedSiteResources.length} site resources`);
+    }
+
+    return removedSiteResources;
+}
+
+export async function performDeleteSiteResource(
+    siteResourceId: number,
+    trx: Transaction | typeof db = db
+): Promise<SiteResource | null> {
+    const [removedSiteResource] = await performDeleteSiteResources(
+        [siteResourceId],
+        trx
+    );
+    return removedSiteResource ?? null;
+}
+
+export function runSiteResourceDeleteSideEffects(
+    removedSiteResource: SiteResource
+): void {
+    rebuildClientAssociationsFromSiteResource(removedSiteResource).catch(
+        (err) => {
+            logger.error(
+                `Error rebuilding client associations for site resource ${removedSiteResource.siteResourceId}:`,
+                err
+            );
+        }
+    );
+}

server/lib/lock.ts

@@ -1,4 +1,24 @@
+const instanceId = `local-${Math.random().toString(36).slice(2)}-${Date.now()}`;
+
+type LocalLockRecord = {
+    owner: string;
+    expiresAt: number;
+};
+
+const localLocks = new Map<string, LocalLockRecord>();
+
 export class LockManager {
+    private clearExpiredLocalLock(lockKey: string): void {
+        const current = localLocks.get(lockKey);
+        if (current && current.expiresAt <= Date.now()) {
+            localLocks.delete(lockKey);
+        }
+    }
+
+    private getLocalOwnerToken(): string {
+        return `${instanceId}:`;
+    }
+
     /**
      * Acquire a distributed lock using Redis SET with NX and PX options
      * @param lockKey - Unique identifier for the lock
@@ -7,22 +27,57 @@ export class LockManager {
      */
     async acquireLock(
         lockKey: string,
-        ttlMs: number = 30000
+        ttlMs: number = 30000,
+        maxRetries: number = 3,
+        retryDelayMs: number = 100
     ): Promise<boolean> {
-        return true;
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
+            this.clearExpiredLocalLock(lockKey);
+
+            const existing = localLocks.get(lockKey);
+            if (!existing) {
+                localLocks.set(lockKey, {
+                    owner: this.getLocalOwnerToken(),
+                    expiresAt: Date.now() + ttlMs
+                });
+                return true;
+            }
+
+            if (existing.owner === this.getLocalOwnerToken()) {
+                existing.expiresAt = Date.now() + ttlMs;
+                localLocks.set(lockKey, existing);
+                return true;
+            }
+
+            if (attempt < maxRetries - 1) {
+                const delay = retryDelayMs * Math.pow(2, attempt);
+                await new Promise((resolve) => setTimeout(resolve, delay));
+            }
+        }
+
+        return false;
     }
 
     /**
      * Release a lock using Lua script to ensure atomicity
      * @param lockKey - Unique identifier for the lock
      */
-    async releaseLock(lockKey: string): Promise<void> {}
+    async releaseLock(lockKey: string): Promise<void> {
+        this.clearExpiredLocalLock(lockKey);
+        const existing = localLocks.get(lockKey);
+
+        if (existing && existing.owner === this.getLocalOwnerToken()) {
+            localLocks.delete(lockKey);
+        }
+    }
 
     /**
      * Force release a lock regardless of owner (use with caution)
      * @param lockKey - Unique identifier for the lock
      */
-    async forceReleaseLock(lockKey: string): Promise<void> {}
+    async forceReleaseLock(lockKey: string): Promise<void> {
+        localLocks.delete(lockKey);
+    }
 
     /**
      * Check if a lock exists and get its info
@@ -35,7 +90,20 @@ export class LockManager {
         ttl: number;
         owner?: string;
     }> {
-        return { exists: true, ownedByMe: true, ttl: 0 };
+        this.clearExpiredLocalLock(lockKey);
+        const existing = localLocks.get(lockKey);
+
+        if (!existing) {
+            return { exists: false, ownedByMe: false, ttl: 0 };
+        }
+
+        const ttl = Math.max(0, existing.expiresAt - Date.now());
+        return {
+            exists: true,
+            ownedByMe: existing.owner === this.getLocalOwnerToken(),
+            ttl,
+            owner: existing.owner.split(":")[0]
+        };
     }
 
     /**
@@ -45,6 +113,15 @@ export class LockManager {
      * @returns Promise<boolean> - true if extended successfully
      */
     async extendLock(lockKey: string, ttlMs: number): Promise<boolean> {
+        this.clearExpiredLocalLock(lockKey);
+        const existing = localLocks.get(lockKey);
+
+        if (!existing || existing.owner !== this.getLocalOwnerToken()) {
+            return false;
+        }
+
+        existing.expiresAt = Date.now() + ttlMs;
+        localLocks.set(lockKey, existing);
         return true;
     }
 
@@ -62,7 +139,26 @@ export class LockManager {
         maxRetries: number = 5,
         baseDelayMs: number = 100
     ): Promise<boolean> {
-        return true;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            const acquired = await this.acquireLock(
+                lockKey,
+                ttlMs,
+                1,
+                baseDelayMs
+            );
+
+            if (acquired) {
+                return true;
+            }
+
+            if (attempt < maxRetries) {
+                const delay =
+                    baseDelayMs * Math.pow(2, attempt) + Math.random() * 100;
+                await new Promise((resolve) => setTimeout(resolve, delay));
+            }
+        }
+
+        return false;
     }
 
     /**
@@ -99,7 +195,21 @@ export class LockManager {
         activeLocksCount: number;
         locksOwnedByMe: number;
     }> {
-        return { activeLocksCount: 0, locksOwnedByMe: 0 };
+        const now = Date.now();
+        for (const [key, value] of localLocks.entries()) {
+            if (value.expiresAt <= now) {
+                localLocks.delete(key);
+            }
+        }
+
+        let locksOwnedByMe = 0;
+        for (const value of localLocks.values()) {
+            if (value.owner === this.getLocalOwnerToken()) {
+                locksOwnedByMe++;
+            }
+        }
+
+        return { activeLocksCount: localLocks.size, locksOwnedByMe };
     }
 
     /**

server/lib/rebuildClientAssociations.ts

@@ -35,10 +35,13 @@ import {
     parseEndpoint
 } from "@server/lib/ip";
 import {
+    addPeerData,
     addPeerDataBatch,
     addTargetsBatch as addSubnetProxyTargetsBatch,
     removePeerDataBatch,
-    removeTargetsBatch as removeSubnetProxyTargetsBatch
+    removeTargetsBatch as removeSubnetProxyTargetsBatch,
+    updatePeerDataBatch,
+    updateTargets
 } from "@server/routers/client/targets";
 import { lockManager } from "#dynamic/lib/lock";
 import { rebuildQueue } from "#dynamic/lib/rebuildQueue";
@@ -47,6 +50,112 @@ import { rebuildQueue } from "#dynamic/lib/rebuildQueue";
 // peer/proxy updates, so give them a generous window.
 const REBUILD_ASSOCIATIONS_LOCK_TTL_MS = 120000;
 
+const REBUILD_IDLE_POLL_INTERVAL_MS = 300;
+const REBUILD_IDLE_DEFAULT_TIMEOUT_MS = 130_000; // slightly longer than lock TTL
+const REBUILD_IDLE_HANDLER_TIMEOUT_MS = 5_000;
+
+/**
+ * Returns true if a rebuild for the given site resource is currently active
+ * (holding the distributed lock) or is pending in the rebuild queue.
+ */
+export async function hasActiveSiteResourceRebuild(
+    siteResourceId: number
+): Promise<boolean> {
+    const lockKey = `rebuild-client-associations:site-resource:${siteResourceId}`;
+    const lockInfo = await lockManager.getLockInfo(lockKey);
+    if (lockInfo.exists) return true;
+    return rebuildQueue.isQueued({ type: "site-resource", id: siteResourceId });
+}
+
+/**
+ * Resolves once there is no active or queued rebuild for the given site resource.
+ * Logs a warning and resolves early if the timeout is reached.
+ */
+export async function waitForSiteResourceRebuildIdle(
+    siteResourceId: number,
+    timeoutMs = REBUILD_IDLE_DEFAULT_TIMEOUT_MS
+): Promise<void> {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (!(await hasActiveSiteResourceRebuild(siteResourceId))) return;
+        await new Promise<void>((r) =>
+            setTimeout(r, REBUILD_IDLE_POLL_INTERVAL_MS)
+        );
+    }
+    logger.warn(
+        `waitForSiteResourceRebuildIdle: timed out after ${timeoutMs}ms waiting for siteResourceId=${siteResourceId}`
+    );
+}
+
+/**
+ * Resolves once there are no active or queued rebuilds for any site resource
+ * associated with the given site.
+ */
+export async function waitForSiteRebuildIdle(
+    siteId: number,
+    timeoutMs = REBUILD_IDLE_HANDLER_TIMEOUT_MS
+): Promise<void> {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        const resourceRows = await db
+            .select({ siteResourceId: siteResources.siteResourceId })
+            .from(siteResources)
+            .innerJoin(
+                siteNetworks,
+                eq(siteNetworks.networkId, siteResources.networkId)
+            )
+            .where(eq(siteNetworks.siteId, siteId));
+        let allIdle = true;
+        for (const { siteResourceId } of resourceRows) {
+            if (await hasActiveSiteResourceRebuild(siteResourceId)) {
+                allIdle = false;
+                break;
+            }
+        }
+        if (allIdle) return;
+        await new Promise<void>((r) =>
+            setTimeout(r, REBUILD_IDLE_POLL_INTERVAL_MS)
+        );
+    }
+    logger.warn(
+        `waitForSiteRebuildIdle: timed out after ${timeoutMs}ms waiting for siteId=${siteId}`
+    );
+}
+
+/**
+ * Resolves once there are no active or queued rebuilds for any site resource
+ * associated with the given client.
+ */
+export async function waitForClientRebuildIdle(
+    clientId: number,
+    timeoutMs = REBUILD_IDLE_HANDLER_TIMEOUT_MS
+): Promise<void> {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        const resourceRows = await db
+            .select({
+                siteResourceId:
+                    clientSiteResourcesAssociationsCache.siteResourceId
+            })
+            .from(clientSiteResourcesAssociationsCache)
+            .where(eq(clientSiteResourcesAssociationsCache.clientId, clientId));
+        let allIdle = true;
+        for (const { siteResourceId } of resourceRows) {
+            if (await hasActiveSiteResourceRebuild(siteResourceId)) {
+                allIdle = false;
+                break;
+            }
+        }
+        if (allIdle) return;
+        await new Promise<void>((r) =>
+            setTimeout(r, REBUILD_IDLE_POLL_INTERVAL_MS)
+        );
+    }
+    logger.warn(
+        `waitForClientRebuildIdle: timed out after ${timeoutMs}ms waiting for clientId=${clientId}`
+    );
+}
+
 export async function getClientSiteResourceAccess(
     siteResource: SiteResource,
     trx: Transaction | typeof db = db
@@ -162,15 +271,10 @@ export async function getClientSiteResourceAccess(
 export async function rebuildClientAssociationsFromSiteResource(
     siteResource: SiteResource
 ) {
-    const trx = primaryDb;
     try {
         return await lockManager.withLock(
             `rebuild-client-associations:site-resource:${siteResource.siteResourceId}`,
-            () =>
-                rebuildClientAssociationsFromSiteResourceImpl(
-                    siteResource,
-                    trx
-                ),
+            () => rebuildClientAssociationsFromSiteResourceImpl(siteResource),
             REBUILD_ASSOCIATIONS_LOCK_TTL_MS
         );
     } catch (err: any) {
@@ -192,15 +296,10 @@ export async function rebuildClientAssociationsFromSiteResource(
 }
 
 async function rebuildClientAssociationsFromSiteResourceImpl(
-    siteResource: SiteResource,
-    trx: Transaction | typeof db = db
-): Promise<{
-    mergedAllClients: {
-        clientId: number;
-        pubKey: string | null;
-        subnet: string | null;
-    }[];
-}> {
+    siteResource: SiteResource
+) {
+    const trx = primaryDb;
+
     logger.debug(
         `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] START siteResourceId=${siteResource.siteResourceId} networkId=${siteResource.networkId} orgId=${siteResource.orgId}`
     );
@@ -214,14 +313,62 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
 
     /////////// process the client-siteResource associations ///////////
 
+    const existingClientSiteResources = await trx
+        .select({
+            clientId: clientSiteResourcesAssociationsCache.clientId
+        })
+        .from(clientSiteResourcesAssociationsCache)
+        .where(
+            eq(
+                clientSiteResourcesAssociationsCache.siteResourceId,
+                siteResource.siteResourceId
+            )
+        );
+
+    const existingClientSiteResourceIds = existingClientSiteResources.map(
+        (row) => row.clientId
+    );
+
     // get all of the clients associated with other site resources that share
     // any of the same sites as this site resource (via siteNetworks). We can't
     // simply filter by networkId since each site resource has its own network;
     // two site resources serving the same site typically belong to different
     // networks that both happen to include the site through siteNetworks.
     const sitesListSiteIds = sitesList.map((s) => s.siteId);
+
+    // We must also consider sites where these clients are currently cached,
+    // otherwise removing a site from this resource can leave stale
+    // client-site cache entries behind for the removed site.
+    const cachedSiteRowsForResourceClients =
+        existingClientSiteResourceIds.length > 0
+            ? await trx
+                  .select({ siteId: clientSitesAssociationsCache.siteId })
+                  .from(clientSitesAssociationsCache)
+                  .where(
+                      inArray(
+                          clientSitesAssociationsCache.clientId,
+                          existingClientSiteResourceIds
+                      )
+                  )
+            : [];
+
+    const allCandidateSiteIds = Array.from(
+        new Set([
+            ...sitesListSiteIds,
+            ...cachedSiteRowsForResourceClients.map((r) => r.siteId)
+        ])
+    );
+
+    const sitesToProcess =
+        allCandidateSiteIds.length > 0
+            ? await trx
+                  .select()
+                  .from(sites)
+                  .where(inArray(sites.siteId, allCandidateSiteIds))
+            : [];
+    const currentSiteIdSet = new Set(sitesListSiteIds);
     const allUpdatedClientsFromOtherResourcesOnThisSite =
-        sitesListSiteIds.length > 0
+        allCandidateSiteIds.length > 0
             ? await trx
                   .select({
                       clientId: clientSiteResourcesAssociationsCache.clientId,
@@ -241,7 +388,7 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
                   )
                   .where(
                       and(
-                          inArray(siteNetworks.siteId, sitesListSiteIds),
+                          inArray(siteNetworks.siteId, allCandidateSiteIds),
                           ne(
                               siteResources.siteResourceId,
                               siteResource.siteResourceId
@@ -260,22 +407,6 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
         clientsFromOtherResourcesBySite.get(row.siteId)!.add(row.clientId);
     }
 
-    const existingClientSiteResources = await trx
-        .select({
-            clientId: clientSiteResourcesAssociationsCache.clientId
-        })
-        .from(clientSiteResourcesAssociationsCache)
-        .where(
-            eq(
-                clientSiteResourcesAssociationsCache.siteResourceId,
-                siteResource.siteResourceId
-            )
-        );
-
-    const existingClientSiteResourceIds = existingClientSiteResources.map(
-        (row) => row.clientId
-    );
-
     logger.debug(
         `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteResourceId=${siteResource.siteResourceId} existingResourceClientIds=[${existingClientSiteResourceIds.join(", ")}]`
     );
@@ -358,10 +489,10 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
     /////////// process the client-site associations ///////////
 
     logger.debug(
-        `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteResourceId=${siteResource.siteResourceId} beginning client-site association loop over ${sitesList.length} site(s)`
+        `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteResourceId=${siteResource.siteResourceId} beginning client-site association loop over ${sitesToProcess.length} site(s) (current=${sitesList.length})`
     );
 
-    for (const site of sitesList) {
+    for (const site of sitesToProcess) {
         const siteId = site.siteId;
 
         logger.debug(
@@ -403,7 +534,13 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
             `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} otherResourceClientIds=[${[...otherResourceClientIds].join(", ")}] mergedAllClientIds=[${mergedAllClientIds.join(", ")}]`
         );
 
-        const clientSitesToAdd = mergedAllClientIds.filter(
+        // Expected clients from this resource are site-scoped: if this site is
+        // no longer attached to the resource, the expected set is empty.
+        const expectedClientIdsForSite = currentSiteIdSet.has(siteId)
+            ? mergedAllClientIds
+            : [];
+
+        const clientSitesToAdd = expectedClientIdsForSite.filter(
             (clientId) =>
                 !existingClientSiteIds.includes(clientId) &&
                 !otherResourceClientIds.has(clientId) // dont add if already connected via another site resource
@@ -438,7 +575,7 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
         // Now remove any client-site associations that should no longer exist
         const clientSitesToRemove = existingClientSiteIds.filter(
             (clientId) =>
-                !mergedAllClientIds.includes(clientId) &&
+                !expectedClientIdsForSite.includes(clientId) &&
                 !otherResourceClientIds.has(clientId) // dont remove if there is still another connection for another site resource
         );
 
@@ -485,10 +622,6 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
         clientSiteResourcesToRemove,
         trx
     );
-
-    return {
-        mergedAllClients
-    };
 }
 
 async function handleMessagesForSiteClients(
@@ -1042,6 +1175,466 @@ async function handleSubnetProxyTargetUpdates(
     await Promise.all([...proxyJobs, ...olmJobs]);
 }
 
+export async function handleMessagingForUpdatedSiteResource(
+    existingSiteResource: SiteResource | undefined,
+    updatedSiteResource: SiteResource,
+    existingSiteIds: number[],
+    updatedSiteIds: number[]
+) {
+    const trx = primaryDb;
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: START siteResourceId=${updatedSiteResource.siteResourceId} existingSiteIds=[${existingSiteIds.join(", ")}] updatedSiteIds=[${updatedSiteIds.join(", ")}]`
+    );
+
+    logger.debug(
+        "handleMessagingForUpdatedSiteResource: existingSiteResource is: ",
+        existingSiteResource
+    );
+    logger.debug(
+        "handleMessagingForUpdatedSiteResource: updatedSiteResource is: ",
+        updatedSiteResource
+    );
+
+    const allSiteIds = [...new Set([...existingSiteIds, ...updatedSiteIds])];
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: allSiteIds=[${allSiteIds.join(", ")}] count=${allSiteIds.length}`
+    );
+
+    const newtsForSites =
+        allSiteIds.length > 0
+            ? await trx
+                  .select()
+                  .from(newts)
+                  .where(inArray(newts.siteId, allSiteIds))
+            : [];
+    const newtBySiteId = new Map(
+        newtsForSites.map((newt) => [newt.siteId, newt])
+    );
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: fetched newts for ${newtsForSites.length}/${allSiteIds.length} site(s)`
+    );
+
+    // WARNING: THIS RELIES ON THE CACHE TABLES BEING UP TO DATE, SO CALL THIS AFTER THE ASSOCIATION CACHE IS UPDATED
+    const mergedAllClients = await trx
+        .select({
+            clientId: clientSiteResourcesAssociationsCache.clientId,
+            pubKey: clients.pubKey,
+            subnet: clients.subnet
+        })
+        .from(clientSiteResourcesAssociationsCache)
+        .innerJoin(
+            clients,
+            eq(clientSiteResourcesAssociationsCache.clientId, clients.clientId)
+        )
+        .where(
+            eq(
+                clientSiteResourcesAssociationsCache.siteResourceId,
+                updatedSiteResource.siteResourceId
+            )
+        );
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: resolved merged clients count=${mergedAllClients.length} clientIds=[${mergedAllClients.map((c) => c.clientId).join(", ")}]`
+    );
+
+    const targets = await generateSubnetProxyTargetV2(
+        updatedSiteResource,
+        mergedAllClients
+    );
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: generated updated targets count=${targets ? targets.length : 0}`
+    );
+
+    const oldDestinationStillInUseClientSitePairs = new Set<string>();
+    if (
+        existingSiteResource?.destination &&
+        allSiteIds.length > 0 &&
+        mergedAllClients.length > 0
+    ) {
+        logger.debug(
+            `handleMessagingForUpdatedSiteResource: checking old destination reuse destination=${existingSiteResource.destination} across siteCount=${allSiteIds.length} clientCount=${mergedAllClients.length}`
+        );
+
+        // we need to do this because the client only knows about peers not resources so we need to make sure that we dont remove it if there is still a another resource
+        const oldDestinationStillInUseRows = await trx
+            .select({
+                clientId: clientSiteResourcesAssociationsCache.clientId,
+                siteId: siteNetworks.siteId
+            })
+            .from(siteResources)
+            .innerJoin(
+                clientSiteResourcesAssociationsCache,
+                eq(
+                    clientSiteResourcesAssociationsCache.siteResourceId,
+                    siteResources.siteResourceId
+                )
+            )
+            .innerJoin(
+                siteNetworks,
+                eq(siteNetworks.networkId, siteResources.networkId)
+            )
+            .where(
+                and(
+                    inArray(
+                        clientSiteResourcesAssociationsCache.clientId,
+                        mergedAllClients.map((c) => c.clientId)
+                    ),
+                    inArray(siteNetworks.siteId, allSiteIds),
+                    eq(
+                        siteResources.destination,
+                        existingSiteResource.destination
+                    ),
+                    ne(
+                        siteResources.siteResourceId,
+                        existingSiteResource.siteResourceId
+                    )
+                )
+            );
+
+        for (const row of oldDestinationStillInUseRows) {
+            oldDestinationStillInUseClientSitePairs.add(
+                `${row.clientId}:${row.siteId}`
+            );
+        }
+
+        logger.debug(
+            `handleMessagingForUpdatedSiteResource: old destination still in use rows=${oldDestinationStillInUseRows.length} uniqueClientSitePairs=${oldDestinationStillInUseClientSitePairs.size}`
+        );
+    } else {
+        logger.debug(
+            "handleMessagingForUpdatedSiteResource: skipping old destination reuse check (missing existing destination or no sites/clients)"
+        );
+    }
+
+    //////////////////////////// FROM HERE DOWN WE ARE DEALING WITH REMOVING SITES
+    const removedSiteIds = existingSiteIds.filter(
+        (id) => !updatedSiteIds.includes(id)
+    );
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: removing sites removedSiteIds=[${removedSiteIds.join(", ")}] count=${removedSiteIds.length}`
+    );
+
+    const targetsToRemoveBatch: {
+        newtId: string;
+        targets: any[];
+        version: string | null;
+    }[] = [];
+    const peerDataRemoves: {
+        clientId: number;
+        siteId: number;
+        remoteSubnets: string[];
+        aliases: ReturnType<typeof generateAliasConfig>;
+    }[] = [];
+    if (targets) {
+        for (const siteId of removedSiteIds) {
+            const newt = newtBySiteId.get(siteId);
+            if (!newt) {
+                logger.debug(
+                    `handleMessagingForUpdatedSiteResource: skipping remove for siteId=${siteId} because no newt found`
+                );
+                continue;
+            }
+
+            logger.debug(
+                `handleMessagingForUpdatedSiteResource: preparing remove batches for siteId=${siteId} newtId=${newt.newtId}`
+            );
+
+            targetsToRemoveBatch.push({
+                newtId: newt.newtId,
+                targets: targets,
+                version: newt.version
+            });
+            for (const client of mergedAllClients) {
+                // we need to do this because the client only knows about peers not resources so we need to make sure that we dont remove it if there is still a another resource
+                const oldDestinationStillInUseBySite =
+                    oldDestinationStillInUseClientSitePairs.has(
+                        `${client.clientId}:${siteId}`
+                    );
+
+                if (existingSiteResource) {
+                    peerDataRemoves.push({
+                        // this might happen twice after the rebuild function but that is okay
+                        clientId: client.clientId,
+                        siteId,
+                        remoteSubnets: !oldDestinationStillInUseBySite
+                            ? generateRemoteSubnets([existingSiteResource])
+                            : [],
+                        aliases: generateAliasConfig([existingSiteResource])
+                    });
+                }
+            }
+        }
+    } else {
+        logger.debug(
+            "handleMessagingForUpdatedSiteResource: skipping removal batch generation because targets were empty"
+        );
+    }
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: remove batches prepared targetBatchCount=${targetsToRemoveBatch.length} peerDataCount=${peerDataRemoves.length}`
+    );
+
+    logger.debug(
+        "handleMessagingForUpdatedSiteResource: dispatching removeSubnetProxyTargetsBatch"
+    );
+
+    removeSubnetProxyTargetsBatch(targetsToRemoveBatch);
+
+    logger.debug(
+        "handleMessagingForUpdatedSiteResource: dispatching removePeerDataBatch"
+    );
+
+    removePeerDataBatch(peerDataRemoves);
+
+    //////////////////////////// FROM HERE DOWN WE ARE DEALING WITH ADDING NEW SITES
+    const addedSiteIds = updatedSiteIds.filter(
+        (id) => !existingSiteIds.includes(id)
+    );
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: adding sites addedSiteIds=[${addedSiteIds.join(", ")}] count=${addedSiteIds.length}`
+    );
+
+    const targetsToAddBatch: {
+        newtId: string;
+        targets: any[];
+        version: string | null;
+    }[] = [];
+    const peerDataAdds: {
+        clientId: number;
+        siteId: number;
+        remoteSubnets: string[];
+        aliases: ReturnType<typeof generateAliasConfig>;
+    }[] = [];
+    if (targets) {
+        for (const siteId of addedSiteIds) {
+            const newt = newtBySiteId.get(siteId);
+            if (!newt) {
+                logger.debug(
+                    `handleMessagingForUpdatedSiteResource: skipping add for siteId=${siteId} because no newt found`
+                );
+                continue;
+            }
+
+            logger.debug(
+                `handleMessagingForUpdatedSiteResource: preparing add batches for siteId=${siteId} newtId=${newt.newtId}`
+            );
+
+            targetsToAddBatch.push({
+                newtId: newt.newtId,
+                targets: targets,
+                version: newt.version
+            });
+            for (const client of mergedAllClients) {
+                peerDataAdds.push({
+                    clientId: client.clientId,
+                    siteId,
+                    remoteSubnets: generateRemoteSubnets([updatedSiteResource]),
+                    aliases: generateAliasConfig([updatedSiteResource])
+                });
+            }
+        }
+    } else {
+        logger.debug(
+            "handleMessagingForUpdatedSiteResource: skipping add batch generation because targets were empty"
+        );
+    }
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: add batches prepared targetBatchCount=${targetsToAddBatch.length} peerDataCount=${peerDataAdds.length}`
+    );
+
+    logger.debug(
+        "handleMessagingForUpdatedSiteResource: dispatching addSubnetProxyTargetsBatch"
+    );
+
+    addSubnetProxyTargetsBatch(targetsToAddBatch);
+
+    logger.debug(
+        "handleMessagingForUpdatedSiteResource: dispatching addPeerDataBatch"
+    );
+
+    addPeerDataBatch(peerDataAdds);
+
+    //////////////////////////// FROM HERE DOWN WE ARE DEALING WITH UPDATING THE EXISTING SITES
+
+    const unchangedSiteIds = existingSiteIds.filter((id) =>
+        updatedSiteIds.includes(id)
+    );
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: unchangedSiteIds=[${unchangedSiteIds.join(", ")}] count=${unchangedSiteIds.length}`
+    );
+
+    // after everything is rebuilt above we still need to update the targets and remote subnets if the destination changed
+    const destinationChanged =
+        existingSiteResource &&
+        existingSiteResource.destination !== updatedSiteResource.destination;
+    const destinationPortChanged =
+        existingSiteResource &&
+        existingSiteResource.destinationPort !==
+            updatedSiteResource.destinationPort;
+    const aliasChanged =
+        existingSiteResource &&
+        existingSiteResource.alias !== updatedSiteResource.alias;
+    const fullDomainChanged =
+        existingSiteResource &&
+        existingSiteResource.fullDomain !== updatedSiteResource.fullDomain;
+    const sslChanged =
+        existingSiteResource &&
+        existingSiteResource.ssl !== updatedSiteResource.ssl;
+    const portRangesChanged =
+        existingSiteResource &&
+        (existingSiteResource.tcpPortRangeString !==
+            updatedSiteResource.tcpPortRangeString ||
+            existingSiteResource.udpPortRangeString !==
+                updatedSiteResource.udpPortRangeString ||
+            existingSiteResource.disableIcmp !==
+                updatedSiteResource.disableIcmp);
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: change flags destinationChanged=${Boolean(destinationChanged)} destinationPortChanged=${Boolean(destinationPortChanged)} aliasChanged=${Boolean(aliasChanged)} fullDomainChanged=${Boolean(fullDomainChanged)} sslChanged=${Boolean(sslChanged)} portRangesChanged=${Boolean(portRangesChanged)}`
+    );
+
+    // if the existingSiteResource is undefined (new resource) we don't need to do anything here, the rebuild above handled it all
+
+    if (
+        destinationChanged ||
+        aliasChanged ||
+        fullDomainChanged ||
+        sslChanged ||
+        portRangesChanged ||
+        destinationPortChanged
+    ) {
+        const shouldUpdateTargets =
+            destinationChanged ||
+            sslChanged ||
+            portRangesChanged ||
+            fullDomainChanged ||
+            destinationPortChanged;
+
+        logger.debug(
+            `handleMessagingForUpdatedSiteResource: entering unchanged-site update path shouldUpdateTargets=${shouldUpdateTargets}`
+        );
+
+        const oldTargets = shouldUpdateTargets
+            ? await generateSubnetProxyTargetV2(
+                  existingSiteResource,
+                  mergedAllClients
+              )
+            : [];
+        const newTargets = shouldUpdateTargets
+            ? await generateSubnetProxyTargetV2(
+                  updatedSiteResource,
+                  mergedAllClients
+              )
+            : [];
+
+        logger.debug(
+            `handleMessagingForUpdatedSiteResource: target update payload sizes oldTargets=${oldTargets ? oldTargets.length : 0} newTargets=${newTargets ? newTargets.length : 0}`
+        );
+
+        const peerDataUpdateBatch: Parameters<typeof updatePeerDataBatch>[0] =
+            [];
+
+        for (const siteId of unchangedSiteIds) {
+            const newt = newtBySiteId.get(siteId);
+
+            logger.debug(
+                `handleMessagingForUpdatedSiteResource: processing unchanged siteId=${siteId}`
+            );
+
+            if (!newt) {
+                logger.error(
+                    `handleMessagingForUpdatedSiteResource: missing newt for unchanged siteId=${siteId}`
+                );
+                throw new Error(
+                    "Newt not found for site during site resource update"
+                );
+            }
+
+            // Only update targets on newt if these items change
+            if (shouldUpdateTargets) {
+                logger.debug(
+                    `handleMessagingForUpdatedSiteResource: updating targets for siteId=${siteId} newtId=${newt.newtId}`
+                );
+                await updateTargets(
+                    newt.newtId,
+                    {
+                        oldTargets: oldTargets ? oldTargets : [],
+                        newTargets: newTargets ? newTargets : []
+                    },
+                    newt.version
+                );
+            }
+
+            for (const client of mergedAllClients) {
+                // does this client have access to another resource on this site that has the same destination still? if so we dont want to remove it from their olm yet
+                if (!existingSiteResource.destination) {
+                    logger.debug(
+                        `handleMessagingForUpdatedSiteResource: skipping peerData update for clientId=${client.clientId} siteId=${siteId} because existing destination is empty`
+                    );
+                    continue;
+                }
+
+                // we need to do this because the client only knows about peers not resources so we need to make sure that we dont remove it if there is still a another resource
+                const oldDestinationStillInUseBySite =
+                    oldDestinationStillInUseClientSitePairs.has(
+                        `${client.clientId}:${siteId}`
+                    );
+
+                // we also need to update the remote subnets on the olms for each client that has access to this site
+                peerDataUpdateBatch.push({
+                    clientId: client.clientId,
+                    siteId,
+                    remoteSubnets: destinationChanged
+                        ? {
+                              oldRemoteSubnets: !oldDestinationStillInUseBySite
+                                  ? generateRemoteSubnets([
+                                        existingSiteResource
+                                    ])
+                                  : [],
+                              newRemoteSubnets: generateRemoteSubnets([
+                                  updatedSiteResource
+                              ])
+                          }
+                        : undefined,
+                    aliases:
+                        aliasChanged || fullDomainChanged // the full domain is sent down as an alias
+                            ? {
+                                  oldAliases: generateAliasConfig([
+                                      existingSiteResource
+                                  ]),
+                                  newAliases: generateAliasConfig([
+                                      updatedSiteResource
+                                  ])
+                              }
+                            : undefined
+                });
+            }
+        }
+
+        logger.debug(
+            `handleMessagingForUpdatedSiteResource: dispatching updatePeerDataBatch count=${peerDataUpdateBatch.length}`
+        );
+
+        updatePeerDataBatch(peerDataUpdateBatch);
+    } else {
+        logger.debug(
+            "handleMessagingForUpdatedSiteResource: no unchanged-site update required because no relevant fields changed"
+        );
+    }
+
+    logger.debug(
+        `handleMessagingForUpdatedSiteResource: DONE siteResourceId=${updatedSiteResource.siteResourceId}`
+    );
+}
+
 export async function rebuildClientAssociationsFromClient(
     client: Client
 ): Promise<void> {
@@ -1776,7 +2369,7 @@ async function handleMessagesForClientResources(
                             )
                         );
 
-                    // Only remove remote subnet if no other resource uses the same destination
+                    // Only remove remote subnet if no other resource uses the same destination on the same site
                     const remoteSubnetsToRemove =
                         destinationStillInUse.length > 0
                             ? []

server/lib/rebuildQueue.ts

@@ -13,11 +13,15 @@ export interface RebuildJobHandlers {
 export interface RebuildQueueManager {
     enqueue(job: RebuildJob): Promise<void>;
     startProcessing(handlers: RebuildJobHandlers): void;
+    isQueued(job: RebuildJob): Promise<boolean>;
 }
 
 class NoopRebuildQueue implements RebuildQueueManager {
     async enqueue(_job: RebuildJob): Promise<void> {}
     startProcessing(_handlers: RebuildJobHandlers): void {}
+    async isQueued(_job: RebuildJob): Promise<boolean> {
+        return false;
+    }
 }
 
 export const rebuildQueue: RebuildQueueManager = new NoopRebuildQueue();

server/middlewares/verifyAccessTokenAccess.ts

@@ -119,8 +119,7 @@ export async function verifyAccessTokenAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifyAdmin.ts

@@ -56,8 +56,7 @@ export async function verifyAdmin(
             return next(
                 createHttpError(
                     HttpCode.FORBIDDEN,
-                    "Failed organization access policy check: " +
-                        (policyCheck.error || "Unknown error")
+                    "" + (policyCheck.error || "Unknown error")
                 )
             );
         }

server/middlewares/verifyApiKeyAccess.ts

@@ -113,8 +113,7 @@ export async function verifyApiKeyAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifyClientAccess.ts

@@ -107,8 +107,7 @@ export async function verifyClientAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }
@@ -129,10 +128,7 @@ export async function verifyClientAccess(
                       .where(
                           and(
                               eq(roleClients.clientId, client.clientId),
-                              inArray(
-                                  roleClients.roleId,
-                                  req.userOrgRoleIds!
-                              )
+                              inArray(roleClients.roleId, req.userOrgRoleIds!)
                           )
                       )
                       .limit(1)

server/middlewares/verifyDomainAccess.ts

@@ -88,8 +88,7 @@ export async function verifyDomainAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifyOrgAccess.ts

@@ -7,6 +7,7 @@ import HttpCode from "@server/types/HttpCode";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
 import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 import { getFirstString } from "@server/lib/requestParams";
+import logger from "@server/logger";
 
 export async function verifyOrgAccess(
     req: Request,
@@ -59,8 +60,7 @@ export async function verifyOrgAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifyResourceAccess.ts

@@ -105,8 +105,7 @@ export async function verifyResourceAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifyResourcePolicyAccess.ts

@@ -102,8 +102,7 @@ export async function verifyResourcePolicyAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifyRoleAccess.ts

@@ -132,8 +132,7 @@ export async function verifyRoleAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifySetResourceClients.ts

@@ -45,8 +45,7 @@ export async function verifySetResourceClients(
             return next(
                 createHttpError(
                     HttpCode.FORBIDDEN,
-                    "Failed organization access policy check: " +
-                        (policyCheck.error || "Unknown error")
+                    "" + (policyCheck.error || "Unknown error")
                 )
             );
         }

server/middlewares/verifySetResourceUsers.ts

@@ -40,8 +40,7 @@ export async function verifySetResourceUsers(
             return next(
                 createHttpError(
                     HttpCode.FORBIDDEN,
-                    "Failed organization access policy check: " +
-                        (policyCheck.error || "Unknown error")
+                    "" + (policyCheck.error || "Unknown error")
                 )
             );
         }

server/middlewares/verifySiteAccess.ts

@@ -115,8 +115,7 @@ export async function verifySiteAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifySiteProvisioningKeyAccess.ts

@@ -115,8 +115,7 @@ export async function verifySiteProvisioningKeyAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifySiteResourceAccess.ts

@@ -103,8 +103,7 @@ export async function verifySiteResourceAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifyTargetAccess.ts

@@ -122,8 +122,7 @@ export async function verifyTargetAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/middlewares/verifyUserAccess.ts

@@ -59,8 +59,7 @@ export async function verifyUserAccess(
                 return next(
                     createHttpError(
                         HttpCode.FORBIDDEN,
-                        "Failed organization access policy check: " +
-                            (policyCheck.error || "Unknown error")
+                        "" + (policyCheck.error || "Unknown error")
                     )
                 );
             }

server/private/lib/acmeCertSync.ts

@@ -693,9 +693,9 @@ async function syncAcmeCerts(acmeJsonPath: string): Promise<void> {
             );
             continue;
         }
-        logger.debug(
-            `acmeCertSync: found ${resolverData.Certificates.length} certificate(s) for resolver "${resolver}"`
-        );
+        // logger.debug(
+        //     `acmeCertSync: found ${resolverData.Certificates.length} certificate(s) for resolver "${resolver}"`
+        // );
         for (const cert of resolverData.Certificates) {
             allCerts.push(cert);
         }

server/private/lib/checkOrgAccessPolicy.ts

@@ -21,6 +21,49 @@ import {
 } from "@server/lib/checkOrgAccessPolicy";
 import { UserType } from "@server/types/UserTypes";
 
+function formatMaxSessionLengthRequirement(
+    maxSessionLengthHours: number
+): string {
+    if (maxSessionLengthHours < 24) {
+        return `This organization requires you to log in every ${maxSessionLengthHours} hours.`;
+    }
+
+    const maxDays = Math.round(maxSessionLengthHours / 24);
+    return `This organization requires you to log in every ${maxDays} days.`;
+}
+
+function buildOrgAccessPolicyError(
+    policies: CheckOrgAccessPolicyResult["policies"]
+): string | undefined {
+    if (!policies) {
+        return undefined;
+    }
+
+    const errors: string[] = [];
+
+    if (policies.requiredTwoFactor === false) {
+        errors.push(
+            "This organization requires two-factor authentication. Enable two-factor authentication on your account to continue."
+        );
+    }
+
+    if (policies.maxSessionLength?.compliant === false) {
+        errors.push(
+            `Your session has expired. ${formatMaxSessionLengthRequirement(
+                policies.maxSessionLength.maxSessionLengthHours
+            )}`
+        );
+    }
+
+    if (policies.passwordAge?.compliant === false) {
+        errors.push(
+            `Your password has expired. This organization requires you to change your password every ${policies.passwordAge.maxPasswordAgeDays} days.`
+        );
+    }
+
+    return errors.length > 0 ? errors.join(" ") : undefined;
+}
+
 export function enforceResourceSessionLength(
     resourceSession: ResourceSession,
     org: Org
@@ -36,13 +79,17 @@ export function enforceResourceSessionLength(
             if (sessionAgeMs > maxSessionLengthMs) {
                 return {
                     valid: false,
-                    error: `Resource session has expired due to organization policy (max session length: ${maxSessionLengthHours} hours)`
+                    error: `Your resource session has expired. ${formatMaxSessionLengthRequirement(
+                        maxSessionLengthHours
+                    )}`
                 };
             }
         } else {
             return {
                 valid: false,
-                error: `Resource session is invalid due to organization policy (max session length: ${maxSessionLengthHours} hours)`
+                error: `Your resource session is invalid. ${formatMaxSessionLengthRequirement(
+                    maxSessionLengthHours
+                )}`
             };
         }
     }
@@ -60,14 +107,20 @@ export async function checkOrgAccessPolicy(
     if (!orgId) {
         return {
             allowed: false,
-            error: "Organization ID is required"
+            error: "Unable to verify organization access. Organization information is missing."
         };
     }
     if (!userId) {
-        return { allowed: false, error: "User ID is required" };
+        return {
+            allowed: false,
+            error: "Unable to verify organization access. User information is missing."
+        };
     }
     if (!sessionId) {
-        return { allowed: false, error: "Session ID is required" };
+        return {
+            allowed: false,
+            error: "Your session is invalid. Please log in again."
+        };
     }
 
     if (build === "enterprise") {
@@ -89,7 +142,10 @@ export async function checkOrgAccessPolicy(
             .where(eq(orgs.orgId, orgId));
         props.org = orgQuery;
         if (!props.org) {
-            return { allowed: false, error: "Organization not found" };
+            return {
+                allowed: false,
+                error: "This organization could not be found."
+            };
         }
     }
 
@@ -100,7 +156,10 @@ export async function checkOrgAccessPolicy(
             .where(eq(users.userId, userId));
         props.user = userQuery;
         if (!props.user) {
-            return { allowed: false, error: "User not found" };
+            return {
+                allowed: false,
+                error: "Your account could not be found."
+            };
         }
     }
 
@@ -111,14 +170,17 @@ export async function checkOrgAccessPolicy(
             .where(eq(sessions.sessionId, sessionId));
         props.session = sessionQuery;
         if (!props.session) {
-            return { allowed: false, error: "Session not found" };
+            return {
+                allowed: false,
+                error: "Your session has expired. Please log in again."
+            };
         }
     }
 
     if (props.session.userId !== props.user.userId) {
         return {
             allowed: false,
-            error: "Session does not belong to the user"
+            error: "Your session is invalid. Please log in again."
         };
     }
 
@@ -187,8 +249,14 @@ export async function checkOrgAccessPolicy(
         allowed = false;
     }
 
+    const policyError = buildOrgAccessPolicyError(policies);
+
     return {
         allowed,
-        policies
+        policies,
+        error: allowed
+            ? undefined
+            : (policyError ??
+              "You do not meet this organization's security requirements.")
     };
 }

server/private/lib/lock.ts

@@ -11,14 +11,31 @@
  * This file is not licensed under the AGPLv3.
  */
 
-import { config } from "@server/lib/config";
 import logger from "@server/logger";
 import { redis } from "#private/lib/redis";
 import { v4 as uuidv4 } from "uuid";
 
 const instanceId = uuidv4();
 
+type LocalLockRecord = {
+    owner: string;
+    expiresAt: number;
+};
+
+const localLocks = new Map<string, LocalLockRecord>();
+
 export class LockManager {
+    private clearExpiredLocalLock(lockKey: string): void {
+        const current = localLocks.get(lockKey);
+        if (current && current.expiresAt <= Date.now()) {
+            localLocks.delete(lockKey);
+        }
+    }
+
+    private getLocalOwnerToken(): string {
+        return `${instanceId}:`;
+    }
+
     /**
      * Acquire a distributed lock using Redis SET with NX and PX options
      * @param lockKey - Unique identifier for the lock
@@ -32,12 +49,34 @@ export class LockManager {
         retryDelayMs: number = 100
     ): Promise<boolean> {
         if (!redis || !redis.status || redis.status !== "ready") {
-            return true;
+            for (let attempt = 0; attempt < maxRetries; attempt++) {
+                this.clearExpiredLocalLock(lockKey);
+
+                const existing = localLocks.get(lockKey);
+                if (!existing) {
+                    localLocks.set(lockKey, {
+                        owner: this.getLocalOwnerToken(),
+                        expiresAt: Date.now() + ttlMs
+                    });
+                    return true;
+                }
+
+                if (existing.owner === this.getLocalOwnerToken()) {
+                    existing.expiresAt = Date.now() + ttlMs;
+                    localLocks.set(lockKey, existing);
+                    return true;
+                }
+
+                if (attempt < maxRetries - 1) {
+                    const delay = retryDelayMs * Math.pow(2, attempt);
+                    await new Promise((resolve) => setTimeout(resolve, delay));
+                }
+            }
+
+            return false;
         }
 
-        const lockValue = `${
-            instanceId
-        }:${Date.now()}`;
+        const lockValue = `${instanceId}:${Date.now()}`;
         const redisKey = `lock:${lockKey}`;
 
         for (let attempt = 0; attempt < maxRetries; attempt++) {
@@ -53,11 +92,7 @@ export class LockManager {
                 );
 
                 if (result === "OK") {
-                    logger.debug(
-                        `Lock acquired: ${lockKey} by ${
-                            instanceId
-                        }`
-                    );
+                    logger.debug(`Lock acquired: ${lockKey} by ${instanceId}`);
                     return true;
                 }
 
@@ -65,17 +100,11 @@ export class LockManager {
                 const existingValue = await redis.get(redisKey);
                 if (
                     existingValue &&
-                    existingValue.startsWith(
-                        `${instanceId}:`
-                    )
+                    existingValue.startsWith(`${instanceId}:`)
                 ) {
                     // Extend the lock TTL since it's the same worker
                     await redis.pexpire(redisKey, ttlMs);
-                    logger.debug(
-                        `Lock extended: ${lockKey} by ${
-                            instanceId
-                        }`
-                    );
+                    logger.debug(`Lock extended: ${lockKey} by ${instanceId}`);
                     return true;
                 }
 
@@ -88,7 +117,10 @@ export class LockManager {
                     await new Promise((resolve) => setTimeout(resolve, delay));
                 }
             } catch (error) {
-                logger.error(`Failed to acquire lock ${lockKey} (attempt ${attempt + 1}/${maxRetries}):`, error);
+                logger.error(
+                    `Failed to acquire lock ${lockKey} (attempt ${attempt + 1}/${maxRetries}):`,
+                    error
+                );
                 // On error, still retry if we have attempts left
                 if (attempt < maxRetries - 1) {
                     const delay = retryDelayMs * Math.pow(2, attempt);
@@ -109,6 +141,11 @@ export class LockManager {
      */
     async releaseLock(lockKey: string): Promise<void> {
         if (!redis || !redis.status || redis.status !== "ready") {
+            this.clearExpiredLocalLock(lockKey);
+            const existing = localLocks.get(lockKey);
+            if (existing && existing.owner === this.getLocalOwnerToken()) {
+                localLocks.delete(lockKey);
+            }
             return;
         }
 
@@ -136,11 +173,7 @@ export class LockManager {
             )) as number;
 
             if (result === 1) {
-                logger.debug(
-                    `Lock released: ${lockKey} by ${
-                        instanceId
-                    }`
-                );
+                logger.debug(`Lock released: ${lockKey} by ${instanceId}`);
             } else {
                 logger.warn(
                     `Lock not released - not owned by worker: ${lockKey} by ${
@@ -159,6 +192,7 @@ export class LockManager {
      */
     async forceReleaseLock(lockKey: string): Promise<void> {
         if (!redis || !redis.status || redis.status !== "ready") {
+            localLocks.delete(lockKey);
             return;
         }
 
@@ -186,7 +220,20 @@ export class LockManager {
         owner?: string;
     }> {
         if (!redis || !redis.status || redis.status !== "ready") {
-            return { exists: false, ownedByMe: true, ttl: 0 };
+            this.clearExpiredLocalLock(lockKey);
+            const existing = localLocks.get(lockKey);
+
+            if (!existing) {
+                return { exists: false, ownedByMe: false, ttl: 0 };
+            }
+
+            const ttl = Math.max(0, existing.expiresAt - Date.now());
+            return {
+                exists: true,
+                ownedByMe: existing.owner === this.getLocalOwnerToken(),
+                ttl,
+                owner: existing.owner.split(":")[0]
+            };
         }
 
         const redisKey = `lock:${lockKey}`;
@@ -198,11 +245,7 @@ export class LockManager {
             ]);
 
             const exists = value !== null;
-            const ownedByMe =
-                exists &&
-                value!.startsWith(
-                    `${instanceId}:`
-                );
+            const ownedByMe = exists && value!.startsWith(`${instanceId}:`);
             const owner = exists ? value!.split(":")[0] : undefined;
 
             return {
@@ -225,6 +268,15 @@ export class LockManager {
      */
     async extendLock(lockKey: string, ttlMs: number): Promise<boolean> {
         if (!redis || !redis.status || redis.status !== "ready") {
+            this.clearExpiredLocalLock(lockKey);
+            const existing = localLocks.get(lockKey);
+
+            if (!existing || existing.owner !== this.getLocalOwnerToken()) {
+                return false;
+            }
+
+            existing.expiresAt = Date.now() + ttlMs;
+            localLocks.set(lockKey, existing);
             return true;
         }
 
@@ -255,9 +307,7 @@ export class LockManager {
 
             if (result === 1) {
                 logger.debug(
-                    `Lock extended: ${lockKey} by ${
-                        instanceId
-                    } for ${ttlMs}ms`
+                    `Lock extended: ${lockKey} by ${instanceId} for ${ttlMs}ms`
                 );
                 return true;
             }
@@ -282,12 +332,13 @@ export class LockManager {
         maxRetries: number = 5,
         baseDelayMs: number = 100
     ): Promise<boolean> {
-        if (!redis || !redis.status || redis.status !== "ready") {
-            return true;
-        }
-
         for (let attempt = 0; attempt <= maxRetries; attempt++) {
-            const acquired = await this.acquireLock(lockKey, ttlMs);
+            const acquired = await this.acquireLock(
+                lockKey,
+                ttlMs,
+                1,
+                baseDelayMs
+            );
 
             if (acquired) {
                 return true;
@@ -319,10 +370,6 @@ export class LockManager {
         fn: () => Promise<T>,
         ttlMs: number = 30000
     ): Promise<T> {
-        if (!redis || !redis.status || redis.status !== "ready") {
-            return await fn();
-        }
-
         const acquired = await this.acquireLock(lockKey, ttlMs);
 
         if (!acquired) {
@@ -346,7 +393,21 @@ export class LockManager {
         locksOwnedByMe: number;
     }> {
         if (!redis || !redis.status || redis.status !== "ready") {
-            return { activeLocksCount: 0, locksOwnedByMe: 0 };
+            const now = Date.now();
+            for (const [key, value] of localLocks.entries()) {
+                if (value.expiresAt <= now) {
+                    localLocks.delete(key);
+                }
+            }
+
+            let locksOwnedByMe = 0;
+            for (const value of localLocks.values()) {
+                if (value.owner === this.getLocalOwnerToken()) {
+                    locksOwnedByMe++;
+                }
+            }
+
+            return { activeLocksCount: localLocks.size, locksOwnedByMe };
         }
 
         try {
@@ -356,11 +417,7 @@ export class LockManager {
             if (keys.length > 0) {
                 const values = await redis.mget(...keys);
                 locksOwnedByMe = values.filter(
-                    (value) =>
-                        value &&
-                        value.startsWith(
-                            `${instanceId}:`
-                        )
+                    (value) => value && value.startsWith(`${instanceId}:`)
                 ).length;
             }
 

server/private/lib/rebuildQueue.ts

@@ -46,6 +46,17 @@ const POLL_INTERVAL_MS = 500;
 class RedisRebuildQueue {
     private processingStarted = false;
 
+    async isQueued(job: RebuildJob): Promise<boolean> {
+        if (!redis || redis.status !== "ready") return false;
+        const dedupeKey = `${job.type}:${job.id}`;
+        try {
+            const member = await redis.sismember(QUEUED_SET_KEY, dedupeKey);
+            return member === 1;
+        } catch {
+            return false;
+        }
+    }
+
     async enqueue(job: RebuildJob): Promise<void> {
         if (!redis || redis.status !== "ready") {
             logger.warn(

server/routers/auth/changePassword.ts

@@ -10,9 +10,8 @@ import { hashPassword, verifyPassword } from "@server/auth/password";
 import { verifyTotpCode } from "@server/auth/totp";
 import logger from "@server/logger";
 import { unauthorized } from "@server/auth/unauthorizedResponse";
-import { invalidateAllSessions } from "@server/auth/sessions/app";
-import { sessions, resourceSessions } from "@server/db";
-import { and, eq, ne, inArray } from "drizzle-orm";
+import { invalidateAllSessionsExceptCurrent } from "@server/auth/sessions/app";
+import { eq } from "drizzle-orm";
 import { passwordSchema } from "@server/auth/passwordSchema";
 import { UserType } from "@server/types/UserTypes";
 import { sendEmail } from "@server/emails";
@@ -31,48 +30,6 @@ export type ChangePasswordResponse = {
     codeRequested?: boolean;
 };
 
-async function invalidateAllSessionsExceptCurrent(
-    userId: string,
-    currentSessionId: string
-): Promise<void> {
-    try {
-        await db.transaction(async (trx) => {
-            // Get all user sessions except the current one
-            const userSessions = await trx
-                .select()
-                .from(sessions)
-                .where(
-                    and(
-                        eq(sessions.userId, userId),
-                        ne(sessions.sessionId, currentSessionId)
-                    )
-                );
-
-            // Delete resource sessions for the sessions we're invalidating
-            if (userSessions.length > 0) {
-                await trx.delete(resourceSessions).where(
-                    inArray(
-                        resourceSessions.userSessionId,
-                        userSessions.map((s) => s.sessionId)
-                    )
-                );
-            }
-
-            // Delete the user sessions (except current)
-            await trx
-                .delete(sessions)
-                .where(
-                    and(
-                        eq(sessions.userId, userId),
-                        ne(sessions.sessionId, currentSessionId)
-                    )
-                );
-        });
-    } catch (e) {
-        logger.error("Failed to invalidate user sessions except current", e);
-    }
-}
-
 export async function changePassword(
     req: Request,
     res: Response,

server/routers/auth/verifyTotp.ts

@@ -15,6 +15,10 @@ import TwoFactorAuthNotification from "@server/emails/templates/TwoFactorAuthNot
 import config from "@server/lib/config";
 import { UserType } from "@server/types/UserTypes";
 import { generateBackupCodes } from "@server/lib/totp";
+import {
+    invalidateAllSessions,
+    invalidateAllSessionsExceptCurrent
+} from "@server/auth/sessions/app";
 import { verifySession } from "@server/auth/sessions/verifySession";
 import { unauthorized } from "@server/auth/unauthorizedResponse";
 
@@ -168,6 +172,15 @@ export async function verifyTotp(
             );
         }
 
+        if (existingSession) {
+            await invalidateAllSessionsExceptCurrent(
+                user.userId,
+                existingSession.sessionId
+            );
+        } else {
+            await invalidateAllSessions(user.userId);
+        }
+
         sendEmail(
             TwoFactorAuthNotification({
                 email: user.email!,

server/routers/client/targets.ts

@@ -438,6 +438,70 @@ export async function removePeerDataBatch(
     await sendToClientsBatch(payloads);
 }
 
+export async function updatePeerDataBatch(
+    entries: {
+        clientId: number;
+        siteId: number;
+        remoteSubnets:
+            | {
+                  oldRemoteSubnets: string[];
+                  newRemoteSubnets: string[];
+              }
+            | undefined;
+        aliases:
+            | {
+                  oldAliases: Alias[];
+                  newAliases: Alias[];
+              }
+            | undefined;
+        olmId?: string;
+        version?: string | null;
+    }[]
+) {
+    if (entries.length === 0) {
+        return;
+    }
+
+    const resolvedTargets = await resolveOlmTargets(entries);
+
+    if (resolvedTargets.length === 0) {
+        return;
+    }
+
+    const payloads = entries
+        .map((entry) => {
+            const resolved = resolvedTargets.find(
+                (target) => target.clientId === entry.clientId
+            );
+            if (!resolved) {
+                return null;
+            }
+
+            return {
+                clientId: resolved.olmId,
+                message: {
+                    type: `olm/wg/peer/data/update`,
+                    data: {
+                        siteId: entry.siteId,
+                        ...entry.remoteSubnets,
+                        ...entry.aliases
+                    }
+                },
+                options: {
+                    incrementConfigVersion: true,
+                    compress: canCompress(resolved.version, "olm")
+                }
+            };
+        })
+        .filter((entry) => entry !== null);
+
+    if (payloads.length === 0) {
+        return;
+    }
+
+    await sendToClientsBatch(payloads);
+}
+
 export async function updatePeerData(
     clientId: number,
     siteId: number,

server/routers/newt/handleNewtGetConfigMessage.ts

@@ -9,6 +9,7 @@ import { buildClientConfigurationForNewtClient } from "./buildConfiguration";
 import { convertTargetsIfNecessary } from "../client/targets";
 import { canCompress } from "@server/lib/clientVersionChecks";
 import config from "@server/lib/config";
+import { waitForSiteRebuildIdle } from "@server/lib/rebuildClientAssociations";
 
 export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
     const { message, client, sendToClient } = context;
@@ -61,6 +62,8 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
         return;
     }
 
+    await waitForSiteRebuildIdle(siteId);
+
     // update the endpoint and the public key
     const [site] = await db
         .update(sites)

server/routers/newt/handleNewtPingMessage.ts

@@ -49,20 +49,22 @@ export const handleNewtPingMessage: MessageHandler = async (context) => {
             `Newt ping with outdated config version: ${message.configVersion} (current: ${configVersion})`
         );
 
-        const [site] = await db
-            .select()
-            .from(sites)
-            .where(eq(sites.siteId, newt.siteId))
-            .limit(1);
+        // TODO: IMPLEMENT THE SYNC ON THE NEWT SIDE AND COMMENT THIS BACK IN
 
-        if (!site) {
-            logger.warn(
-                `Newt ping message: site with ID ${newt.siteId} not found`
-            );
-            return;
-        }
+        // const [site] = await db
+        //     .select()
+        //     .from(sites)
+        //     .where(eq(sites.siteId, newt.siteId))
+        //     .limit(1);
 
-        await sendNewtSyncMessage(newt, site);
+        // if (!site) {
+        //     logger.warn(
+        //         `Newt ping message: site with ID ${newt.siteId} not found`
+        //     );
+        //     return;
+        // }
+
+        // await sendNewtSyncMessage(newt, site);
     }
 
     return {

server/routers/olm/handleOlmRegisterMessage.ts

@@ -21,6 +21,7 @@ import { build } from "@server/build";
 import { canCompress } from "@server/lib/clientVersionChecks";
 import config from "@server/lib/config";
 import cache from "#dynamic/lib/cache"; // not using regional here because we need this in the register message handler before we know where the client is
+import { waitForClientRebuildIdle } from "@server/lib/rebuildClientAssociations";
 
 const HOLEPUNCH_STALE_CHAIN_THRESHOLD = 18;
 const HOLEPUNCH_STALE_CHAIN_TTL_SECONDS = 1800;
@@ -385,6 +386,8 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => {
     }
 
     // NOTE: its important that the client here is the old client and the public key is the new key
+    await waitForClientRebuildIdle(olm.clientId);
+
     const siteConfigurations = await buildSiteConfigurationForOlmClient(
         client,
         publicKey,

server/routers/resource/deleteResource.ts

@@ -1,13 +1,4 @@
-import { eq, inArray } from "drizzle-orm";
-import {
-    db,
-    newts,
-    resourcePolicies,
-    resources,
-    sites,
-    targetHealthCheck,
-    targets
-} from "@server/db";
+import { db } from "@server/db";
 import response from "@server/lib/response";
 import logger from "@server/logger";
 import { OpenAPITags, registry } from "@server/openApi";
@@ -16,9 +7,11 @@ import { NextFunction, Request, Response } from "express";
 import createHttpError from "http-errors";
 import { z } from "zod";
 import { fromError } from "zod-validation-error";
-import { removeTargets } from "../newt/targets";
+import {
+    performDeleteResource,
+    runResourceDeleteSideEffects
+} from "@server/lib/deleteResource";
 
-// Define Zod schema for request parameters validation
 const deleteResourceSchema = z.strictObject({
     resourceId: z.coerce.number().int().positive()
 });
@@ -67,27 +60,13 @@ export async function deleteResource(
 
         const { resourceId } = parsedParams.data;
 
-        const targetsToBeRemoved = await db
-            .select()
-            .from(targets)
-            .where(eq(targets.resourceId, resourceId));
+        let deleteResult = null;
 
-        const healthChecksToBeRemoved = await db
-            .select()
-            .from(targetHealthCheck)
-            .where(
-                inArray(
-                    targetHealthCheck.targetId,
-                    targetsToBeRemoved.map((t) => t.targetId)
-                )
-            );
+        await db.transaction(async (trx) => {
+            deleteResult = await performDeleteResource(resourceId, trx);
+        });
 
-        const [deletedResource] = await db
-            .delete(resources)
-            .where(eq(resources.resourceId, resourceId))
-            .returning();
-
-        if (!deletedResource) {
+        if (!deleteResult) {
             return next(
                 createHttpError(
                     HttpCode.NOT_FOUND,
@@ -96,54 +75,7 @@ export async function deleteResource(
             );
         }
 
-        for (const target of targetsToBeRemoved) {
-            const [site] = await db
-                .select()
-                .from(sites)
-                .where(eq(sites.siteId, target.siteId))
-                .limit(1);
-
-            if (!site) {
-                return next(
-                    createHttpError(
-                        HttpCode.NOT_FOUND,
-                        `Site with ID ${target.siteId} not found`
-                    )
-                );
-            }
-
-            if (site.pubKey) {
-                if (site.type == "newt") {
-                    // get the newt on the site by querying the newt table for siteId
-                    const [newt] = await db
-                        .select()
-                        .from(newts)
-                        .where(eq(newts.siteId, site.siteId))
-                        .limit(1);
-
-                    await removeTargets(
-                        newt.newtId,
-                        // [target],
-                        [], // deleting the target from newt causes issues because we cant unbind the port. this needs to be fixed in newt before we can do this
-                        healthChecksToBeRemoved,
-                        deletedResource.mode === "udp" ? "udp" : "tcp",
-                        newt.version
-                    );
-                }
-            }
-        }
-
-        // Also delete default resource policy
-        if (deletedResource.defaultResourcePolicyId) {
-            await db
-                .delete(resourcePolicies)
-                .where(
-                    eq(
-                        resourcePolicies.resourcePolicyId,
-                        deletedResource.defaultResourcePolicyId
-                    )
-                );
-        }
+        await runResourceDeleteSideEffects(deleteResult);
 
         return response(res, {
             data: null,
@@ -154,6 +86,9 @@ export async function deleteResource(
         });
     } catch (error) {
         logger.error(error);
+        if (createHttpError.isHttpError(error)) {
+            return next(error);
+        }
         return next(
             createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
         );

server/routers/resource/getExchangeToken.ts

@@ -80,8 +80,7 @@ export async function getExchangeToken(
             return next(
                 createHttpError(
                     HttpCode.FORBIDDEN,
-                    "Failed organization access policy check: " +
-                        (hasAccess.error || "Unknown error")
+                    "" + (hasAccess.error || "Unknown error")
                 )
             );
         }

server/routers/site/deleteSite.ts

@@ -14,18 +14,41 @@ import { OpenAPITags, registry } from "@server/openApi";
 import { cleanupSiteAssociations } from "@server/lib/rebuildClientAssociations";
 import { usageService } from "@server/lib/billing/usageService";
 import { FeatureId } from "@server/lib/billing";
+import { ActionsEnum, checkUserActionPermission } from "@server/auth/actions";
+import {
+    deleteAssociatedResourcesForSite,
+    exceedsSiteAssociatedResourceDeleteLimit,
+    getAssociatedResourceCountForSite,
+    runDeleteSiteAssociatedResourcesSideEffects,
+    MAX_SITE_ASSOCIATED_RESOURCES_FOR_BULK_DELETE,
+    type DeleteSiteAssociatedResourcesSideEffects
+} from "@server/lib/deleteSiteAssociatedResources";
 
 const deleteSiteSchema = z.strictObject({
     siteId: z.coerce.number().int().positive()
 });
 
+const deleteSiteQuerySchema = z.strictObject({
+    deleteResources: z
+        .enum(["true", "false"])
+        .transform((v) => v === "true")
+        .optional()
+        .catch(false)
+        .openapi({
+            type: "boolean",
+            description:
+                "When true, also deletes all public and private resources associated with this site"
+        })
+});
+
 registry.registerPath({
     method: "delete",
     path: "/site/{siteId}",
     description: "Delete a site and all its associated data.",
     tags: [OpenAPITags.Site],
     request: {
-        params: deleteSiteSchema
+        params: deleteSiteSchema,
+        query: deleteSiteQuerySchema
     },
     responses: {
         200: {
@@ -61,7 +84,18 @@ export async function deleteSite(
             );
         }
 
+        const parsedQuery = deleteSiteQuerySchema.safeParse(req.query);
+        if (!parsedQuery.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedQuery.error).toString()
+                )
+            );
+        }
+
         const { siteId } = parsedParams.data;
+        const { deleteResources } = parsedQuery.data;
 
         const [site] = await db
             .select()
@@ -78,20 +112,67 @@ export async function deleteSite(
             );
         }
 
+        if (deleteResources) {
+            const canDeletePublic = await checkUserActionPermission(
+                ActionsEnum.deleteResource,
+                req
+            );
+            const canDeletePrivate = await checkUserActionPermission(
+                ActionsEnum.deleteSiteResource,
+                req
+            );
+
+            if (!canDeletePublic || !canDeletePrivate) {
+                return next(
+                    createHttpError(
+                        HttpCode.FORBIDDEN,
+                        "User does not have permission to delete associated resources"
+                    )
+                );
+            }
+
+            const associatedResourceCount =
+                await getAssociatedResourceCountForSite(siteId, site.orgId);
+
+            if (
+                exceedsSiteAssociatedResourceDeleteLimit(
+                    associatedResourceCount
+                )
+            ) {
+                return next(
+                    createHttpError(
+                        HttpCode.BAD_REQUEST,
+                        `Cannot delete site and associated resources when the site has more than ${MAX_SITE_ASSOCIATED_RESOURCES_FOR_BULK_DELETE} resources`
+                    )
+                );
+            }
+        }
+
         const [deletedNewt] = await db
             .select()
             .from(newts)
             .where(eq(newts.siteId, siteId))
             .limit(1);
 
+        let resourceSideEffects: DeleteSiteAssociatedResourcesSideEffects = {
+            resources: [],
+            siteResources: []
+        };
+
         await db.transaction(async (trx) => {
+            if (deleteResources) {
+                resourceSideEffects = await deleteAssociatedResourcesForSite(
+                    siteId,
+                    site.orgId,
+                    trx
+                );
+            }
+
             if (site.type == "wireguard") {
                 if (site.pubKey) {
                     await deletePeer(site.exitNodeId!, site.pubKey);
                 }
             } else if (site.type == "newt") {
-                // Clean up all client associations and send peer/proxy removal
-                // messages in a single efficient pass before deleting the row.
                 await cleanupSiteAssociations(site, trx);
             }
 
@@ -99,13 +180,17 @@ export async function deleteSite(
             await usageService.add(site.orgId, FeatureId.SITES, -1, trx);
         });
 
-        // Send termination message outside of transaction to prevent blocking
+        if (deleteResources) {
+            await runDeleteSiteAssociatedResourcesSideEffects(
+                resourceSideEffects
+            );
+        }
+
         if (deletedNewt) {
             const payload = {
                 type: `newt/wg/terminate`,
                 data: {}
             };
-            // Don't await this to prevent blocking the response
             sendToClient(deletedNewt.newtId, payload).catch((error) => {
                 logger.error(
                     "Failed to send termination message to newt:",
@@ -123,6 +208,9 @@ export async function deleteSite(
         });
     } catch (error) {
         logger.error(error);
+        if (createHttpError.isHttpError(error)) {
+            return next(error);
+        }
         return next(
             createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
         );

server/routers/siteResource/deleteSiteResource.ts

@@ -1,15 +1,17 @@
 import { Request, Response, NextFunction } from "express";
 import { z } from "zod";
-import { db, newts, primaryDb, sites } from "@server/db";
-import { siteResources } from "@server/db";
+import { db, siteResources } from "@server/db";
 import response from "@server/lib/response";
 import HttpCode from "@server/types/HttpCode";
 import createHttpError from "http-errors";
-import { eq, and } from "drizzle-orm";
+import { eq } from "drizzle-orm";
 import { fromError } from "zod-validation-error";
 import logger from "@server/logger";
 import { OpenAPITags, registry } from "@server/openApi";
-import { rebuildClientAssociationsFromSiteResource } from "@server/lib/rebuildClientAssociations";
+import {
+    performDeleteSiteResource,
+    runSiteResourceDeleteSideEffects
+} from "@server/lib/deleteSiteResource";
 
 const deleteSiteResourceParamsSchema = z.strictObject({
     siteResourceId: z.coerce.number().int().positive()
@@ -65,11 +67,10 @@ export async function deleteSiteResource(
 
         const { siteResourceId } = parsedParams.data;
 
-        // Check if site resource exists
         const [existingSiteResource] = await db
             .select()
             .from(siteResources)
-            .where(and(eq(siteResources.siteResourceId, siteResourceId)))
+            .where(eq(siteResources.siteResourceId, siteResourceId))
             .limit(1);
 
         if (!existingSiteResource) {
@@ -78,26 +79,22 @@ export async function deleteSiteResource(
             );
         }
 
-        // Delete the site resource
-        const [removedSiteResource] = await db
-            .delete(siteResources)
-            .where(eq(siteResources.siteResourceId, siteResourceId))
-            .returning();
+        let removedSiteResource = null;
 
-        // Run in the background after the response is sent. Wrapped in its
-        // own transaction so it always executes on the primary — avoiding any
-        // replica-lag issues while still allowing the HTTP response to return
-        // early.
-        rebuildClientAssociationsFromSiteResource(removedSiteResource).catch(
-            (err) => {
-                logger.error(
-                    `Error rebuilding client associations for site resource ${removedSiteResource!.siteResourceId}:`,
-                    err
-                );
-            }
-        );
+        await db.transaction(async (trx) => {
+            removedSiteResource = await performDeleteSiteResource(
+                siteResourceId,
+                trx
+            );
+        });
 
-        logger.info(`Deleted site resource ${siteResourceId}`);
+        if (!removedSiteResource) {
+            return next(
+                createHttpError(HttpCode.NOT_FOUND, "Site resource not found")
+            );
+        }
+
+        runSiteResourceDeleteSideEffects(removedSiteResource);
 
         return response(res, {
             data: { message: "Site resource deleted successfully" },

server/routers/siteResource/updateSiteResource.ts

@@ -1,8 +1,6 @@
 import {
     clientSiteResources,
-    clientSiteResourcesAssociationsCache,
     db,
-    newts,
     orgs,
     roles,
     roleSiteResources,
@@ -10,10 +8,7 @@ import {
     SiteResource,
     siteResources,
     sites,
-    networks,
-    Transaction,
-    userSiteResources,
-    primaryDb
+    userSiteResources
 } from "@server/db";
 import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
 import { TierFeature, tierMatrix } from "@server/lib/billing/tierMatrix";
@@ -21,17 +16,11 @@ import { validateAndConstructDomain } from "@server/lib/domainUtils";
 import response from "@server/lib/response";
 import { eq, and, ne, inArray } from "drizzle-orm";
 import { OpenAPITags, registry } from "@server/openApi";
-import { updatePeerData, updateTargets } from "@server/routers/client/targets";
+import { isIpInCidr, portRangeStringSchema } from "@server/lib/ip";
 import {
-    generateAliasConfig,
-    generateRemoteSubnets,
-    generateSubnetProxyTargetV2,
-    isIpInCidr,
-    portRangeStringSchema
-} from "@server/lib/ip";
-import {
-    getClientSiteResourceAccess,
-    rebuildClientAssociationsFromSiteResource
+    handleMessagingForUpdatedSiteResource,
+    rebuildClientAssociationsFromSiteResource,
+    waitForSiteResourceRebuildIdle
 } from "@server/lib/rebuildClientAssociations";
 import logger from "@server/logger";
 import HttpCode from "@server/types/HttpCode";
@@ -390,7 +379,7 @@ export async function updateSiteResource(
             );
         }
 
-        const existingSiteIds = existingSiteResource.networkId
+        const existingSiteNetworks = existingSiteResource.networkId
             ? await db
                   .select()
                   .from(siteNetworks)
@@ -398,7 +387,7 @@ export async function updateSiteResource(
                       eq(siteNetworks.networkId, existingSiteResource.networkId)
                   )
             : [];
-        const existingSiteIdSet = new Set(existingSiteIds.map((s) => s.siteId));
+        const existingSiteIds = existingSiteNetworks.map((sn) => sn.siteId);
 
         let fullDomain: string | null = null;
         let finalSubdomain: string | null = null;
@@ -464,6 +453,7 @@ export async function updateSiteResource(
         }
 
         let updatedSiteResource: SiteResource | undefined;
+        let updatedSiteIds: number[] = [];
         await db.transaction(async (trx) => {
             // Update the site resource
             const sshPamSet =
@@ -534,6 +524,7 @@ export async function updateSiteResource(
                     siteId: siteId,
                     networkId: updatedSiteResource.networkId!
                 });
+                updatedSiteIds.push(siteId);
             }
 
             await trx
@@ -605,27 +596,27 @@ export async function updateSiteResource(
             throw new Error("No updated resource found after update");
         }
 
-        rebuildClientAssociationsFromSiteResource(updatedSiteResource).catch(
-            (e) => {
-                logger.error(
-                    `Failed to rebuild client associations for site resource ${siteResourceId}. Error: ${e}`
-                );
-            }
-        );
+        const finalUpdatedSiteResource = updatedSiteResource;
 
-        handleMessagingForUpdatedSiteResource(
-            existingSiteResource,
-            updatedSiteResource,
-            Array.from(existingSiteIdSet).map((siteId: number) => ({
-                // we already added to the new sites above in the rebuild function so we only need to update the ones that did not change
-                siteId,
-                orgId: existingSiteResource.orgId
-            }))
-        ).catch((e) => {
-            logger.error(
-                `Failed to handle messaging for updated site resource ${siteResourceId}. Error: ${e}`
-            );
-        });
+        rebuildClientAssociationsFromSiteResource(finalUpdatedSiteResource)
+            .then(() =>
+                waitForSiteResourceRebuildIdle(
+                    finalUpdatedSiteResource.siteResourceId
+                )
+            )
+            .then(() =>
+                handleMessagingForUpdatedSiteResource(
+                    existingSiteResource,
+                    finalUpdatedSiteResource,
+                    existingSiteIds,
+                    updatedSiteIds
+                )
+            )
+            .catch((e) => {
+                logger.error(
+                    `Failed to rebuild and handle messaging for site resource ${siteResourceId}. Error: ${e}`
+                );
+            });
 
         return response(res, {
             data: updatedSiteResource,
@@ -644,211 +635,3 @@ export async function updateSiteResource(
         );
     }
 }
-
-export async function handleMessagingForUpdatedSiteResource(
-    existingSiteResource: SiteResource | undefined,
-    updatedSiteResource: SiteResource,
-    sites: { siteId: number; orgId: string }[]
-) {
-    const trx = primaryDb;
-    logger.debug(
-        "handleMessagingForUpdatedSiteResource: existingSiteResource is: ",
-        existingSiteResource
-    );
-    logger.debug(
-        "handleMessagingForUpdatedSiteResource: updatedSiteResource is: ",
-        updatedSiteResource
-    );
-
-    const { sitesList, mergedAllClients, mergedAllClientIds } =
-        await getClientSiteResourceAccess(
-            existingSiteResource || updatedSiteResource,
-            trx
-        );
-
-    const siteIds = sites.map((site) => site.siteId);
-
-    // after everything is rebuilt above we still need to update the targets and remote subnets if the destination changed
-    const destinationChanged =
-        existingSiteResource &&
-        existingSiteResource.destination !== updatedSiteResource.destination;
-    const destinationPortChanged =
-        existingSiteResource &&
-        existingSiteResource.destinationPort !==
-            updatedSiteResource.destinationPort;
-    const aliasChanged =
-        existingSiteResource &&
-        existingSiteResource.alias !== updatedSiteResource.alias;
-    const fullDomainChanged =
-        existingSiteResource &&
-        existingSiteResource.fullDomain !== updatedSiteResource.fullDomain;
-    const sslChanged =
-        existingSiteResource &&
-        existingSiteResource.ssl !== updatedSiteResource.ssl;
-    const portRangesChanged =
-        existingSiteResource &&
-        (existingSiteResource.tcpPortRangeString !==
-            updatedSiteResource.tcpPortRangeString ||
-            existingSiteResource.udpPortRangeString !==
-                updatedSiteResource.udpPortRangeString ||
-            existingSiteResource.disableIcmp !==
-                updatedSiteResource.disableIcmp);
-
-    // if the existingSiteResource is undefined (new resource) we don't need to do anything here, the rebuild above handled it all
-
-    if (
-        destinationChanged ||
-        aliasChanged ||
-        fullDomainChanged ||
-        sslChanged ||
-        portRangesChanged ||
-        destinationPortChanged
-    ) {
-        const newtsForSites =
-            siteIds.length > 0
-                ? await trx
-                      .select()
-                      .from(newts)
-                      .where(inArray(newts.siteId, siteIds))
-                : [];
-        const newtBySiteId = new Map(
-            newtsForSites.map((newt) => [newt.siteId, newt])
-        );
-
-        const oldDestinationStillInUseClientSitePairs = new Set<string>();
-        if (
-            existingSiteResource?.destination &&
-            siteIds.length > 0 &&
-            mergedAllClientIds.length > 0
-        ) {
-            const oldDestinationStillInUseRows = await trx
-                .select({
-                    clientId: clientSiteResourcesAssociationsCache.clientId,
-                    siteId: siteNetworks.siteId
-                })
-                .from(siteResources)
-                .innerJoin(
-                    clientSiteResourcesAssociationsCache,
-                    eq(
-                        clientSiteResourcesAssociationsCache.siteResourceId,
-                        siteResources.siteResourceId
-                    )
-                )
-                .innerJoin(
-                    siteNetworks,
-                    eq(siteNetworks.networkId, siteResources.networkId)
-                )
-                .where(
-                    and(
-                        inArray(
-                            clientSiteResourcesAssociationsCache.clientId,
-                            mergedAllClientIds
-                        ),
-                        inArray(siteNetworks.siteId, siteIds),
-                        eq(
-                            siteResources.destination,
-                            existingSiteResource.destination
-                        ),
-                        ne(
-                            siteResources.siteResourceId,
-                            existingSiteResource.siteResourceId
-                        )
-                    )
-                );
-
-            for (const row of oldDestinationStillInUseRows) {
-                oldDestinationStillInUseClientSitePairs.add(
-                    `${row.clientId}:${row.siteId}`
-                );
-            }
-        }
-
-        const shouldUpdateTargets =
-            destinationChanged ||
-            sslChanged ||
-            portRangesChanged ||
-            fullDomainChanged ||
-            destinationPortChanged;
-        const oldTargets = shouldUpdateTargets
-            ? await generateSubnetProxyTargetV2(
-                  existingSiteResource,
-                  mergedAllClients
-              )
-            : [];
-        const newTargets = shouldUpdateTargets
-            ? await generateSubnetProxyTargetV2(
-                  updatedSiteResource,
-                  mergedAllClients
-              )
-            : [];
-
-        for (const site of sites) {
-            const newt = newtBySiteId.get(site.siteId);
-
-            if (!newt) {
-                throw new Error(
-                    "Newt not found for site during site resource update"
-                );
-            }
-
-            // Only update targets on newt if these items change
-            if (shouldUpdateTargets) {
-                await updateTargets(
-                    newt.newtId,
-                    {
-                        oldTargets: oldTargets ? oldTargets : [],
-                        newTargets: newTargets ? newTargets : []
-                    },
-                    newt.version
-                );
-            }
-
-            const olmJobs: Promise<void>[] = [];
-            for (const client of mergedAllClients) {
-                // does this client have access to another resource on this site that has the same destination still? if so we dont want to remove it from their olm yet
-                if (!existingSiteResource.destination) {
-                    continue;
-                }
-
-                const oldDestinationStillInUseByASite =
-                    oldDestinationStillInUseClientSitePairs.has(
-                        `${client.clientId}:${site.siteId}`
-                    );
-
-                // we also need to update the remote subnets on the olms for each client that has access to this site
-                olmJobs.push(
-                    updatePeerData(
-                        // TODO: THIS SHOULD BE UPDATED TO WORK I A BATCH
-                        client.clientId,
-                        site.siteId,
-                        destinationChanged
-                            ? {
-                                  oldRemoteSubnets:
-                                      !oldDestinationStillInUseByASite
-                                          ? generateRemoteSubnets([
-                                                existingSiteResource
-                                            ])
-                                          : [],
-                                  newRemoteSubnets: generateRemoteSubnets([
-                                      updatedSiteResource
-                                  ])
-                              }
-                            : undefined,
-                        aliasChanged || fullDomainChanged // the full domain is sent down as an alias
-                            ? {
-                                  oldAliases: generateAliasConfig([
-                                      existingSiteResource
-                                  ]),
-                                  newAliases: generateAliasConfig([
-                                      updatedSiteResource
-                                  ])
-                              }
-                            : undefined
-                    )
-                );
-            }
-
-            await Promise.all(olmJobs);
-        }
-    }
-}

src/components/ProductUpdates.tsx

@@ -8,6 +8,7 @@ import {
     type ProductUpdate,
     productUpdatesQueries
 } from "@app/lib/queries";
+import { build } from "@server/build";
 import { useQueries } from "@tanstack/react-query";
 import {
     ArrowRight,
@@ -39,22 +40,42 @@ export default function ProductUpdates({
 }) {
     const { env } = useEnvContext();
 
+    const productUpdatesEnabled = env.app.notifications.product_updates;
+    const versionCheckEnabled =
+        env.app.notifications.new_releases && build !== "saas";
+
     const data = useQueries({
         queries: [
-            productUpdatesQueries.list(
-                env.app.notifications.product_updates,
-                env.app.version
-            ),
+            productUpdatesQueries.list(productUpdatesEnabled, env.app.version),
             productUpdatesQueries.latestVersion(
                 env.app.notifications.new_releases
             )
         ],
         combine(result) {
-            if (result[0].isLoading || result[1].isLoading) return null;
-            return {
-                updates: result[0].data?.data ?? [],
-                latestVersion: result[1].data
-            };
+            const [updatesQuery, versionQuery] = result;
+
+            const updatesSettled =
+                !productUpdatesEnabled ||
+                updatesQuery.isFetched ||
+                updatesQuery.isError;
+            const versionSettled =
+                !versionCheckEnabled ||
+                versionQuery.isFetched ||
+                versionQuery.isError;
+
+            if (!updatesSettled || !versionSettled) return null;
+
+            const updates = updatesQuery.isError
+                ? []
+                : Array.isArray(updatesQuery.data?.data)
+                  ? updatesQuery.data.data
+                  : [];
+
+            const latestVersion = versionQuery.isError
+                ? undefined
+                : versionQuery.data;
+
+            return { updates, latestVersion };
         }
     });
     const t = useTranslations();
@@ -76,19 +97,30 @@ export default function ProductUpdates({
 
     if (!data) return null;
 
-    const latestVersion = data?.latestVersion?.data?.pangolin.latestVersion;
+    const versionResponse = data.latestVersion?.data;
+    const latestVersion = versionResponse?.pangolin?.latestVersion;
     const currentVersion = env.app.version;
 
-    const showNewVersionPopup = Boolean(
+    let showNewVersionPopup = false;
+    if (
         latestVersion &&
-            valid(latestVersion) &&
-            valid(currentVersion) &&
-            ignoredVersionUpdate !== latestVersion &&
-            gt(latestVersion, currentVersion)
-    );
+        valid(latestVersion) &&
+        valid(currentVersion) &&
+        ignoredVersionUpdate !== latestVersion
+    ) {
+        try {
+            showNewVersionPopup = gt(latestVersion, currentVersion);
+        } catch {
+            showNewVersionPopup = false;
+        }
+    }
+
+    const readUpdateIds = Array.isArray(productUpdatesRead)
+        ? productUpdatesRead
+        : [];
 
     const filteredUpdates = data.updates.filter(
-        (update) => !productUpdatesRead.includes(update.id)
+        (update) => !readUpdateIds.includes(update.id)
     );
 
     if (filteredUpdates.length === 0 && !showNewVersionPopup) {
@@ -133,17 +165,14 @@ export default function ProductUpdates({
                             show={filteredUpdates.length > 0}
                             onDimissAll={() =>
                                 setProductUpdatesRead([
-                                    ...productUpdatesRead,
+                                    ...readUpdateIds,
                                     ...filteredUpdates.map(
                                         (update) => update.id
                                     )
                                 ])
                             }
                             onDimiss={(id) =>
-                                setProductUpdatesRead([
-                                    ...productUpdatesRead,
-                                    id
-                                ])
+                                setProductUpdatesRead([...readUpdateIds, id])
                             }
                         />
                     </div>
@@ -151,11 +180,9 @@ export default function ProductUpdates({
             </div>
 
             <NewVersionAvailable
-                version={data.latestVersion?.data}
+                version={versionResponse}
                 onDimiss={() => {
-                    setIgnoredVersionUpdate(
-                        data.latestVersion?.data?.pangolin.latestVersion ?? null
-                    );
+                    setIgnoredVersionUpdate(latestVersion ?? null);
                 }}
                 show={showNewVersionPopup}
             />
@@ -346,6 +373,10 @@ function NewVersionAvailable({
         }
     }, [show]);
 
+    if (!version?.pangolin?.latestVersion) {
+        return null;
+    }
+
     return (
         <Transition show={open}>
             {version && (

src/components/RoleForm.tsx

@@ -61,7 +61,7 @@ export function parseUnixGroups(value: string | undefined): string[] {
     if (!value?.trim()) return [];
 
     return value
-        .split(/[,\s\n]+/)
+        .split(/\r?\n/)
         .map((group) => group.trim())
         .filter(Boolean);
 }
@@ -69,18 +69,10 @@ export function parseUnixGroups(value: string | undefined): string[] {
 export function parseSudoCommands(value: string | undefined): string[] {
     if (!value?.trim()) return [];
 
-    const commands: string[] = [];
-    for (const segment of value.split(/[,\n]+/)) {
-        const trimmed = segment.trim();
-        if (!trimmed) continue;
-
-        for (const part of trimmed.split(/ (?=\/)/)) {
-            const command = part.trim();
-            if (command) commands.push(command);
-        }
-    }
-
-    return commands;
+    return value
+        .split(/\r?\n/)
+        .map((command) => command.trim())
+        .filter(Boolean);
 }
 
 function hasOnlyAbsoluteSudoCommands(value: string | undefined): boolean {

src/components/SitesTable.tsx

@@ -19,6 +19,7 @@ import {
     DropdownMenu,
     DropdownMenuContent,
     DropdownMenuItem,
+    DropdownMenuSeparator,
     DropdownMenuTrigger
 } from "@app/components/ui/dropdown-menu";
 import { InfoPopup } from "@app/components/ui/info-popup";
@@ -104,6 +105,7 @@ export default function SitesTable({
     } = useNavigationContext();
 
     const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
+    const [deleteWithResources, setDeleteWithResources] = useState(false);
     const [selectedSite, setSelectedSite] = useState<SiteRow | null>(null);
     const [resourcesDialogSite, setResourcesDialogSite] =
         useState<SiteRow | null>(null);
@@ -157,10 +159,12 @@ export default function SitesTable({
         });
     }
 
-    function deleteSite(siteId: number) {
+    function deleteSite(siteId: number, withResources: boolean) {
         startTransition(async () => {
             await api
-                .delete(`/site/${siteId}`)
+                .delete(`/site/${siteId}`, {
+                    params: { deleteResources: withResources }
+                })
                 .catch((e) => {
                     console.error(t("siteErrorDelete"), e);
                     toast({
@@ -521,16 +525,33 @@ export default function SitesTable({
                                             )}
                                         </DropdownMenuItem>
                                     </Link>
+                                    <DropdownMenuSeparator />
                                     <DropdownMenuItem
                                         onClick={() => {
                                             setSelectedSite(siteRow);
+                                            setDeleteWithResources(false);
                                             setIsDeleteModalOpen(true);
                                         }}
                                     >
                                         <span className="text-red-500">
-                                            {t("delete")}
+                                            {t("sitesTableDeleteSite")}
                                         </span>
                                     </DropdownMenuItem>
+                                    {siteRow.resourceCount <= 250 && (
+                                        <DropdownMenuItem
+                                            onClick={() => {
+                                                setSelectedSite(siteRow);
+                                                setDeleteWithResources(true);
+                                                setIsDeleteModalOpen(true);
+                                            }}
+                                        >
+                                            <span className="text-red-500">
+                                                {t(
+                                                    "sitesTableDeleteSiteAndResources"
+                                                )}
+                                            </span>
+                                        </DropdownMenuItem>
+                                    )}
                                 </DropdownMenuContent>
                             </DropdownMenu>
                             <Link
@@ -639,19 +660,38 @@ export default function SitesTable({
                     setOpen={(val) => {
                         setIsDeleteModalOpen(val);
                         setSelectedSite(null);
+                        setDeleteWithResources(false);
                     }}
                     dialog={
                         <div className="space-y-2">
-                            <p>{t("siteQuestionRemove")}</p>
-                            <p>{t("siteMessageRemove")}</p>
+                            <p>
+                                {deleteWithResources
+                                    ? t("siteQuestionRemoveAndResources")
+                                    : t("siteQuestionRemove")}
+                            </p>
+                            <p>
+                                {deleteWithResources
+                                    ? t("siteMessageRemoveAndResources")
+                                    : t("siteMessageRemove")}
+                            </p>
                         </div>
                     }
-                    buttonText={t("siteConfirmDelete")}
+                    buttonText={
+                        deleteWithResources
+                            ? t("siteConfirmDeleteAndResources")
+                            : t("siteConfirmDelete")
+                    }
                     onConfirm={async () =>
-                        startTransition(() => deleteSite(selectedSite!.id))
+                        startTransition(() =>
+                            deleteSite(selectedSite!.id, deleteWithResources)
+                        )
                     }
                     string={selectedSite.name}
-                    title={t("siteDelete")}
+                    title={
+                        deleteWithResources
+                            ? t("siteDeleteAndResources")
+                            : t("siteDelete")
+                    }
                 />
             )}