Properly lock the ip selection through writes to db

server/lib/blueprints/clientResources.ts

@@ -364,8 +364,14 @@ export async function updateClientResources(
             });
         } else {
             let aliasAddress: string | null = null;
+            let releaseAliasLock: (() => Promise<void>) | null = null;
             if (resourceData.mode === "host" || resourceData.mode === "http") {
-                aliasAddress = await getNextAvailableAliasAddress(orgId, trx);
+                const { value, release } = await getNextAvailableAliasAddress(
+                    orgId,
+                    trx
+                );
+                aliasAddress = value;
+                releaseAliasLock = release;
             }
 
             let domainInfo:
@@ -427,6 +433,8 @@ export async function updateClientResources(
                 })
                 .returning();
 
+            await releaseAliasLock?.();
+
             const siteResourceId = newResource.siteResourceId;
 
             for (const site of allSites) {

server/lib/calculateUserClientsForOrgs.ts

@@ -331,16 +331,8 @@ export async function calculateUserClientsForOrgs(
                     ];
 
                 // Get next available subnet
-                const newSubnet = await getNextAvailableClientSubnet(
-                    orgId,
-                    transaction
-                );
-                if (!newSubnet) {
-                    logger.warn(
-                        `Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): no available subnet found`
-                    );
-                    continue;
-                }
+                const { value: newSubnet, release: releaseSubnetLock } =
+                    await getNextAvailableClientSubnet(orgId, transaction);
 
                 const subnet = newSubnet.split("/")[0];
                 const updatedSubnet = `${subnet}/${org.subnet.split("/")[1]}`;
@@ -370,6 +362,7 @@ export async function calculateUserClientsForOrgs(
                     .insert(clients)
                     .values(newClientData)
                     .returning();
+                await releaseSubnetLock();
                 existingClientCache.set(
                     getOrgOlmKey(orgId, olm.olmId),
                     newClient

server/lib/ip.ts

@@ -327,127 +327,145 @@ export function doCidrsOverlap(cidr1: string, cidr2: string): boolean {
 export async function getNextAvailableClientSubnet(
     orgId: string,
     transaction: Transaction | typeof db = db
-): Promise<string> {
-    return await lockManager.withLock(
-        `client-subnet-allocation:${orgId}`,
-        async () => {
-            const [org] = await transaction
-                .select()
-                .from(orgs)
-                .where(eq(orgs.orgId, orgId));
+): Promise<{ value: string; release: () => Promise<void> }> {
+    const lockKey = `client-subnet-allocation:${orgId}`;
+    const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
+    if (!acquired) {
+        throw new Error(`Failed to acquire lock: ${lockKey}`);
+    }
+    const release = () => lockManager.releaseLock(lockKey);
 
-            if (!org) {
-                throw new Error(`Organization with ID ${orgId} not found`);
-            }
+    try {
+        const [org] = await transaction
+            .select()
+            .from(orgs)
+            .where(eq(orgs.orgId, orgId));
 
-            if (!org.subnet) {
-                throw new Error(
-                    `Organization with ID ${orgId} has no subnet defined`
-                );
-            }
-
-            const existingAddressesSites = await transaction
-                .select({
-                    address: sites.address
-                })
-                .from(sites)
-                .where(and(isNotNull(sites.address), eq(sites.orgId, orgId)));
-
-            const existingAddressesClients = await transaction
-                .select({
-                    address: clients.subnet
-                })
-                .from(clients)
-                .where(
-                    and(isNotNull(clients.subnet), eq(clients.orgId, orgId))
-                );
-
-            const addresses = [
-                ...existingAddressesSites.map(
-                    (site) => `${site.address?.split("/")[0]}/32`
-                ), // we are overriding the 32 so that we pick individual addresses in the subnet of the org for the site and the client even though they are stored with the /block_size of the org
-                ...existingAddressesClients.map(
-                    (client) => `${client.address.split("/")}/32`
-                )
-            ].filter((address) => address !== null) as string[];
-
-            const subnet = findNextAvailableCidr(addresses, 32, org.subnet); // pick the sites address in the org
-            if (!subnet) {
-                throw new Error("No available subnets remaining in space");
-            }
-
-            return subnet;
+        if (!org) {
+            throw new Error(`Organization with ID ${orgId} not found`);
         }
-    );
+
+        if (!org.subnet) {
+            throw new Error(
+                `Organization with ID ${orgId} has no subnet defined`
+            );
+        }
+
+        const existingAddressesSites = await transaction
+            .select({
+                address: sites.address
+            })
+            .from(sites)
+            .where(and(isNotNull(sites.address), eq(sites.orgId, orgId)));
+
+        const existingAddressesClients = await transaction
+            .select({
+                address: clients.subnet
+            })
+            .from(clients)
+            .where(and(isNotNull(clients.subnet), eq(clients.orgId, orgId)));
+
+        const addresses = [
+            ...existingAddressesSites.map(
+                (site) => `${site.address?.split("/")[0]}/32`
+            ), // we are overriding the 32 so that we pick individual addresses in the subnet of the org for the site and the client even though they are stored with the /block_size of the org
+            ...existingAddressesClients.map(
+                (client) => `${client.address.split("/")[0]}/32`
+            )
+        ].filter((address) => address !== null) as string[];
+
+        const subnet = findNextAvailableCidr(addresses, 32, org.subnet); // pick the sites address in the org
+        if (!subnet) {
+            throw new Error("No available subnets remaining in space");
+        }
+
+        return { value: subnet, release };
+    } catch (e) {
+        await release();
+        throw e;
+    }
 }
 
 export async function getNextAvailableAliasAddress(
     orgId: string,
     trx: Transaction | typeof db = db
-): Promise<string> {
-    return await lockManager.withLock(
-        `alias-address-allocation:${orgId}`,
-        async () => {
-            const [org] = await trx
-                .select()
-                .from(orgs)
-                .where(eq(orgs.orgId, orgId));
+): Promise<{ value: string; release: () => Promise<void> }> {
+    const lockKey = `alias-address-allocation:${orgId}`;
+    const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
+    if (!acquired) {
+        throw new Error(`Failed to acquire lock: ${lockKey}`);
+    }
+    const release = () => lockManager.releaseLock(lockKey);
 
-            if (!org) {
-                throw new Error(`Organization with ID ${orgId} not found`);
-            }
+    try {
+        const [org] = await trx
+            .select()
+            .from(orgs)
+            .where(eq(orgs.orgId, orgId));
 
-            if (!org.subnet) {
-                throw new Error(
-                    `Organization with ID ${orgId} has no subnet defined`
-                );
-            }
-
-            if (!org.utilitySubnet) {
-                throw new Error(
-                    `Organization with ID ${orgId} has no utility subnet defined`
-                );
-            }
-
-            const existingAddresses = await trx
-                .select({
-                    aliasAddress: siteResources.aliasAddress
-                })
-                .from(siteResources)
-                .where(
-                    and(
-                        isNotNull(siteResources.aliasAddress),
-                        eq(siteResources.orgId, orgId)
-                    )
-                );
-
-            const addresses = [
-                ...existingAddresses.map(
-                    (site) => `${site.aliasAddress?.split("/")[0]}/32`
-                ),
-                // reserve a /29 for the dns server and other stuff
-                `${org.utilitySubnet.split("/")[0]}/29`
-            ].filter((address) => address !== null) as string[];
-
-            let subnet = findNextAvailableCidr(
-                addresses,
-                32,
-                org.utilitySubnet
-            );
-            if (!subnet) {
-                throw new Error("No available subnets remaining in space");
-            }
-
-            // remove the cidr
-            subnet = subnet.split("/")[0];
-
-            return subnet;
+        if (!org) {
+            throw new Error(`Organization with ID ${orgId} not found`);
         }
-    );
+
+        if (!org.subnet) {
+            throw new Error(
+                `Organization with ID ${orgId} has no subnet defined`
+            );
+        }
+
+        if (!org.utilitySubnet) {
+            throw new Error(
+                `Organization with ID ${orgId} has no utility subnet defined`
+            );
+        }
+
+        const existingAddresses = await trx
+            .select({
+                aliasAddress: siteResources.aliasAddress
+            })
+            .from(siteResources)
+            .where(
+                and(
+                    isNotNull(siteResources.aliasAddress),
+                    eq(siteResources.orgId, orgId)
+                )
+            );
+
+        const addresses = [
+            ...existingAddresses.map(
+                (site) => `${site.aliasAddress?.split("/")[0]}/32`
+            ),
+            // reserve a /29 for the dns server and other stuff
+            `${org.utilitySubnet.split("/")[0]}/29`
+        ].filter((address) => address !== null) as string[];
+
+        let subnet = findNextAvailableCidr(addresses, 32, org.utilitySubnet);
+        if (!subnet) {
+            throw new Error("No available subnets remaining in space");
+        }
+
+        // remove the cidr
+        subnet = subnet.split("/")[0];
+
+        return { value: subnet, release };
+    } catch (e) {
+        await release();
+        throw e;
+    }
 }
 
-export async function getNextAvailableOrgSubnet(): Promise<string> {
-    return await lockManager.withLock("org-subnet-allocation", async () => {
+export async function getNextAvailableOrgSubnet(): Promise<{
+    value: string;
+    release: () => Promise<void>;
+}> {
+    const lockKey = "org-subnet-allocation";
+    const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
+    if (!acquired) {
+        throw new Error(`Failed to acquire lock: ${lockKey}`);
+    }
+    const release = () => lockManager.releaseLock(lockKey);
+
+    try {
         const existingAddresses = await db
             .select({
                 subnet: orgs.subnet
@@ -466,8 +484,11 @@ export async function getNextAvailableOrgSubnet(): Promise<string> {
             throw new Error("No available subnets remaining in space");
         }
 
-        return subnet;
-    });
+        return { value: subnet, release };
+    } catch (e) {
+        await release();
+        throw e;
+    }
 }
 
 export function generateRemoteSubnets(