Properly lock the ip selection through writes to db

This commit is contained in:
Owen
2026-05-27 21:06:34 -07:00
parent cd9e56fdb7
commit 64c901d91f
8 changed files with 598 additions and 551 deletions
+9 -1
View File
@@ -364,8 +364,14 @@ export async function updateClientResources(
}); });
} else { } else {
let aliasAddress: string | null = null; let aliasAddress: string | null = null;
let releaseAliasLock: (() => Promise<void>) | null = null;
if (resourceData.mode === "host" || resourceData.mode === "http") { if (resourceData.mode === "host" || resourceData.mode === "http") {
aliasAddress = await getNextAvailableAliasAddress(orgId, trx); const { value, release } = await getNextAvailableAliasAddress(
orgId,
trx
);
aliasAddress = value;
releaseAliasLock = release;
} }
let domainInfo: let domainInfo:
@@ -427,6 +433,8 @@ export async function updateClientResources(
}) })
.returning(); .returning();
await releaseAliasLock?.();
const siteResourceId = newResource.siteResourceId; const siteResourceId = newResource.siteResourceId;
for (const site of allSites) { for (const site of allSites) {
+3 -10
View File
@@ -331,16 +331,8 @@ export async function calculateUserClientsForOrgs(
]; ];
// Get next available subnet // Get next available subnet
const newSubnet = await getNextAvailableClientSubnet( const { value: newSubnet, release: releaseSubnetLock } =
orgId, await getNextAvailableClientSubnet(orgId, transaction);
transaction
);
if (!newSubnet) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): no available subnet found`
);
continue;
}
const subnet = newSubnet.split("/")[0]; const subnet = newSubnet.split("/")[0];
const updatedSubnet = `${subnet}/${org.subnet.split("/")[1]}`; const updatedSubnet = `${subnet}/${org.subnet.split("/")[1]}`;
@@ -370,6 +362,7 @@ export async function calculateUserClientsForOrgs(
.insert(clients) .insert(clients)
.values(newClientData) .values(newClientData)
.returning(); .returning();
await releaseSubnetLock();
existingClientCache.set( existingClientCache.set(
getOrgOlmKey(orgId, olm.olmId), getOrgOlmKey(orgId, olm.olmId),
newClient newClient
+46 -25
View File
@@ -327,10 +327,15 @@ export function doCidrsOverlap(cidr1: string, cidr2: string): boolean {
export async function getNextAvailableClientSubnet( export async function getNextAvailableClientSubnet(
orgId: string, orgId: string,
transaction: Transaction | typeof db = db transaction: Transaction | typeof db = db
): Promise<string> { ): Promise<{ value: string; release: () => Promise<void> }> {
return await lockManager.withLock( const lockKey = `client-subnet-allocation:${orgId}`;
`client-subnet-allocation:${orgId}`, const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
async () => { if (!acquired) {
throw new Error(`Failed to acquire lock: ${lockKey}`);
}
const release = () => lockManager.releaseLock(lockKey);
try {
const [org] = await transaction const [org] = await transaction
.select() .select()
.from(orgs) .from(orgs)
@@ -358,16 +363,14 @@ export async function getNextAvailableClientSubnet(
address: clients.subnet address: clients.subnet
}) })
.from(clients) .from(clients)
.where( .where(and(isNotNull(clients.subnet), eq(clients.orgId, orgId)));
and(isNotNull(clients.subnet), eq(clients.orgId, orgId))
);
const addresses = [ const addresses = [
...existingAddressesSites.map( ...existingAddressesSites.map(
(site) => `${site.address?.split("/")[0]}/32` (site) => `${site.address?.split("/")[0]}/32`
), // we are overriding the 32 so that we pick individual addresses in the subnet of the org for the site and the client even though they are stored with the /block_size of the org ), // we are overriding the 32 so that we pick individual addresses in the subnet of the org for the site and the client even though they are stored with the /block_size of the org
...existingAddressesClients.map( ...existingAddressesClients.map(
(client) => `${client.address.split("/")}/32` (client) => `${client.address.split("/")[0]}/32`
) )
].filter((address) => address !== null) as string[]; ].filter((address) => address !== null) as string[];
@@ -376,18 +379,25 @@ export async function getNextAvailableClientSubnet(
throw new Error("No available subnets remaining in space"); throw new Error("No available subnets remaining in space");
} }
return subnet; return { value: subnet, release };
} catch (e) {
await release();
throw e;
} }
);
} }
export async function getNextAvailableAliasAddress( export async function getNextAvailableAliasAddress(
orgId: string, orgId: string,
trx: Transaction | typeof db = db trx: Transaction | typeof db = db
): Promise<string> { ): Promise<{ value: string; release: () => Promise<void> }> {
return await lockManager.withLock( const lockKey = `alias-address-allocation:${orgId}`;
`alias-address-allocation:${orgId}`, const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
async () => { if (!acquired) {
throw new Error(`Failed to acquire lock: ${lockKey}`);
}
const release = () => lockManager.releaseLock(lockKey);
try {
const [org] = await trx const [org] = await trx
.select() .select()
.from(orgs) .from(orgs)
@@ -429,11 +439,7 @@ export async function getNextAvailableAliasAddress(
`${org.utilitySubnet.split("/")[0]}/29` `${org.utilitySubnet.split("/")[0]}/29`
].filter((address) => address !== null) as string[]; ].filter((address) => address !== null) as string[];
let subnet = findNextAvailableCidr( let subnet = findNextAvailableCidr(addresses, 32, org.utilitySubnet);
addresses,
32,
org.utilitySubnet
);
if (!subnet) { if (!subnet) {
throw new Error("No available subnets remaining in space"); throw new Error("No available subnets remaining in space");
} }
@@ -441,13 +447,25 @@ export async function getNextAvailableAliasAddress(
// remove the cidr // remove the cidr
subnet = subnet.split("/")[0]; subnet = subnet.split("/")[0];
return subnet; return { value: subnet, release };
} catch (e) {
await release();
throw e;
} }
);
} }
export async function getNextAvailableOrgSubnet(): Promise<string> { export async function getNextAvailableOrgSubnet(): Promise<{
return await lockManager.withLock("org-subnet-allocation", async () => { value: string;
release: () => Promise<void>;
}> {
const lockKey = "org-subnet-allocation";
const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
if (!acquired) {
throw new Error(`Failed to acquire lock: ${lockKey}`);
}
const release = () => lockManager.releaseLock(lockKey);
try {
const existingAddresses = await db const existingAddresses = await db
.select({ .select({
subnet: orgs.subnet subnet: orgs.subnet
@@ -466,8 +484,11 @@ export async function getNextAvailableOrgSubnet(): Promise<string> {
throw new Error("No available subnets remaining in space"); throw new Error("No available subnets remaining in space");
} }
return subnet; return { value: subnet, release };
}); } catch (e) {
await release();
throw e;
}
} }
export function generateRemoteSubnets( export function generateRemoteSubnets(
+3 -1
View File
@@ -51,7 +51,9 @@ export async function pickClientDefaults(
const olmId = generateId(15); const olmId = generateId(15);
const secret = generateId(48); const secret = generateId(48);
const newSubnet = await getNextAvailableClientSubnet(orgId); const { value: newSubnet, release } =
await getNextAvailableClientSubnet(orgId);
await release(); // release immediately — this endpoint only previews the next available value
if (!newSubnet) { if (!newSubnet) {
return next( return next(
createHttpError( createHttpError(
+9 -11
View File
@@ -203,17 +203,10 @@ export async function registerNewt(
let newSiteId: number | undefined; let newSiteId: number | undefined;
const { value: newClientAddress, release: releaseSubnetLock } =
await getNextAvailableClientSubnet(orgId);
try {
await db.transaction(async (trx) => { await db.transaction(async (trx) => {
const newClientAddress = await getNextAvailableClientSubnet(orgId);
if (!newClientAddress) {
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"No available subnet found"
)
);
}
let clientAddress = newClientAddress.split("/")[0]; let clientAddress = newClientAddress.split("/")[0];
clientAddress = `${clientAddress}/${org.subnet!.split("/")[1]}`; // we want the block size of the whole org clientAddress = `${clientAddress}/${org.subnet!.split("/")[1]}`; // we want the block size of the whole org
@@ -227,7 +220,9 @@ export async function registerNewt(
address: clientAddress, address: clientAddress,
type: "newt", type: "newt",
dockerSocketEnabled: true, dockerSocketEnabled: true,
status: keyRecord.approveNewSites ? "approved" : "pending" status: keyRecord.approveNewSites
? "approved"
: "pending"
}) })
.returning(); .returning();
@@ -281,6 +276,9 @@ export async function registerNewt(
await usageService.add(orgId, FeatureId.SITES, 1, trx); await usageService.add(orgId, FeatureId.SITES, 1, trx);
}); });
} finally {
await releaseSubnetLock();
}
logger.info( logger.info(
`Provisioned new site (ID: ${newSiteId}) and newt (ID: ${newtId}) for org ${orgId} via provisioning key ${provisioningKeyId}` `Provisioned new site (ID: ${newSiteId}) and newt (ID: ${newtId}) for org ${orgId} via provisioning key ${provisioningKeyId}`
+22 -18
View File
@@ -174,6 +174,7 @@ export async function createSite(
} }
let updatedAddress = null; let updatedAddress = null;
let releaseSubnetLock: (() => Promise<void>) | null = null;
if (address) { if (address) {
if (!org.subnet) { if (!org.subnet) {
return next( return next(
@@ -244,19 +245,14 @@ export async function createSite(
); );
} }
} else { } else {
const newClientAddress = await getNextAvailableClientSubnet(orgId); const { value: newClientAddress, release } =
if (!newClientAddress) { await getNextAvailableClientSubnet(orgId);
return next( releaseSubnetLock = release;
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"No available address found"
)
);
}
updatedAddress = newClientAddress.split("/")[0]; updatedAddress = newClientAddress.split("/")[0];
} }
let newSite: Site | undefined;
try {
if (subnet && exitNodeId) { if (subnet && exitNodeId) {
//make sure the subnet is in the range of the exit node if provided //make sure the subnet is in the range of the exit node if provided
const [exitNode] = await db const [exitNode] = await db
@@ -266,7 +262,10 @@ export async function createSite(
if (!exitNode) { if (!exitNode) {
return next( return next(
createHttpError(HttpCode.NOT_FOUND, "Exit node not found") createHttpError(
HttpCode.NOT_FOUND,
"Exit node not found"
)
); );
} }
@@ -321,7 +320,9 @@ export async function createSite(
const existingSite = await db const existingSite = await db
.select() .select()
.from(sites) .from(sites)
.where(and(eq(sites.niceId, niceId), eq(sites.orgId, orgId))) .where(
and(eq(sites.niceId, niceId), eq(sites.orgId, orgId))
)
.limit(1); .limit(1);
if (existingSite.length > 0) { if (existingSite.length > 0) {
@@ -334,7 +335,6 @@ export async function createSite(
} }
} }
let newSite: Site | undefined;
await db.transaction(async (trx) => { await db.transaction(async (trx) => {
if (type == "newt") { if (type == "newt") {
[newSite] = await trx [newSite] = await trx
@@ -378,10 +378,8 @@ export async function createSite(
); );
} }
const { exitNode, hasAccess } = await verifyExitNodeOrgAccess( const { exitNode, hasAccess } =
exitNodeId, await verifyExitNodeOrgAccess(exitNodeId, orgId);
orgId
);
if (!exitNode) { if (!exitNode) {
logger.warn("Exit node not found"); logger.warn("Exit node not found");
@@ -448,7 +446,10 @@ export async function createSite(
if (adminRole.length === 0) { if (adminRole.length === 0) {
return next( return next(
createHttpError(HttpCode.NOT_FOUND, `Admin role not found`) createHttpError(
HttpCode.NOT_FOUND,
`Admin role not found`
)
); );
} }
@@ -505,6 +506,9 @@ export async function createSite(
await usageService.add(orgId, FeatureId.SITES, 1, trx); await usageService.add(orgId, FeatureId.SITES, 1, trx);
}); });
} finally {
await releaseSubnetLock?.();
}
if (!newSite) { if (!newSite) {
return next( return next(
+3 -1
View File
@@ -119,7 +119,9 @@ export async function pickSiteDefaults(
); );
} }
const newClientAddress = await getNextAvailableClientSubnet(orgId); const { value: newClientAddress, release: releaseSubnetLock } =
await getNextAvailableClientSubnet(orgId);
await releaseSubnetLock(); // release immediately — this endpoint only previews the next available value
if (!newClientAddress) { if (!newClientAddress) {
return next( return next(
createHttpError( createHttpError(
@@ -397,11 +397,16 @@ export async function createSiteResource(
} }
let aliasAddress: string | null = null; let aliasAddress: string | null = null;
let releaseAliasLock: (() => Promise<void>) | null = null;
if (mode === "host" || mode === "http") { if (mode === "host" || mode === "http") {
aliasAddress = await getNextAvailableAliasAddress(orgId); const { value, release } =
await getNextAvailableAliasAddress(orgId);
aliasAddress = value;
releaseAliasLock = release;
} }
let newSiteResource: SiteResource | undefined; let newSiteResource: SiteResource | undefined;
try {
await db.transaction(async (trx) => { await db.transaction(async (trx) => {
const [network] = await trx const [network] = await trx
.insert(networks) .insert(networks)
@@ -445,7 +450,9 @@ export async function createSiteResource(
aliasAddress, aliasAddress,
tcpPortRangeString: tcpPortRangeStringAdjusted, tcpPortRangeString: tcpPortRangeStringAdjusted,
udpPortRangeString: udpPortRangeString:
mode == "http" || mode == "ssh" ? "" : udpPortRangeString, mode == "http" || mode == "ssh"
? ""
: udpPortRangeString,
disableIcmp: disableIcmp:
disableIcmp || disableIcmp ||
(mode == "http" || mode == "ssh" ? true : false), // default to true for http resources, otherwise false (mode == "http" || mode == "ssh" ? true : false), // default to true for http resources, otherwise false
@@ -484,7 +491,10 @@ export async function createSiteResource(
if (!adminRole) { if (!adminRole) {
return next( return next(
createHttpError(HttpCode.NOT_FOUND, `Admin role not found`) createHttpError(
HttpCode.NOT_FOUND,
`Admin role not found`
)
); );
} }
@@ -497,7 +507,10 @@ export async function createSiteResource(
await trx await trx
.insert(roleSiteResources) .insert(roleSiteResources)
.values( .values(
roleIds.map((roleId) => ({ roleId, siteResourceId })) roleIds.map((roleId) => ({
roleId,
siteResourceId
}))
); );
} }
@@ -505,7 +518,10 @@ export async function createSiteResource(
await trx await trx
.insert(userSiteResources) .insert(userSiteResources)
.values( .values(
userIds.map((userId) => ({ userId, siteResourceId })) userIds.map((userId) => ({
userId,
siteResourceId
}))
); );
} }
@@ -535,6 +551,9 @@ export async function createSiteResource(
} }
} }
}); });
} finally {
await releaseAliasLock?.();
}
if (!newSiteResource) { if (!newSiteResource) {
return next( return next(