Merge branch 'dev' into refactor/batch-status-requests

This commit is contained in:
Fred KISSIE
2026-07-20 18:38:00 +01:00
202 changed files with 6263 additions and 884 deletions
-6
View File
@@ -34,12 +34,6 @@ import {
rebuildClientAssociationsFromSiteResource,
waitForSiteResourceRebuildIdle
} from "../rebuildClientAssociations";
import { build } from "@server/build";
import HttpCode from "@server/types/HttpCode";
import createHttpError from "http-errors";
import next from "next";
import { LimitId } from "../billing";
import { usageService } from "../billing/usageService";
type ApplyBlueprintArgs = {
orgId: string;
+62 -10
View File
@@ -26,9 +26,6 @@ import { createCertificate } from "#dynamic/routers/certificates/createCertifica
import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
import { tierMatrix } from "../billing/tierMatrix";
import { build } from "@server/build";
import HttpCode from "@server/types/HttpCode";
import createHttpError from "http-errors";
import next from "next";
import { LimitId } from "../billing";
import { usageService } from "../billing/usageService";
@@ -201,17 +198,19 @@ export async function updatePrivateResources(
}
}
let resourceStatusFromSite: "approved" | "pending" = "approved";
if (siteId && allSites.length === 0) {
// only add if there are not provided sites
// Use the provided siteId directly, but verify it belongs to the org
const [siteSingle] = await trx
.select({ siteId: sites.siteId })
.select({ siteId: sites.siteId, status: sites.status })
.from(sites)
.where(and(eq(sites.siteId, siteId), eq(sites.orgId, orgId)))
.limit(1);
if (siteSingle) {
allSites.push(siteSingle);
}
resourceStatusFromSite = siteSingle.status ?? "approved";
}
if (allSites.length === 0) {
@@ -220,6 +219,13 @@ export async function updatePrivateResources(
);
}
const resourceEnabled =
resourceData.enabled == undefined || resourceData.enabled == null
? true
: resourceStatusFromSite === "pending"
? false
: resourceData.enabled;
if (existingResource) {
let domainInfo:
| { subdomain: string | null; domainId: string }
@@ -233,6 +239,31 @@ export async function updatePrivateResources(
);
}
if (resourceData.alias) {
const [aliasConflict] = await trx
.select({
siteResourceId: siteResources.siteResourceId
})
.from(siteResources)
.where(
and(
eq(siteResources.orgId, orgId),
eq(siteResources.alias, resourceData.alias),
ne(
siteResources.siteResourceId,
existingResource.siteResourceId
)
)
)
.limit(1);
if (aliasConflict) {
throw new Error(
`Alias ${resourceData.alias} already in use by another site resource in org ${orgId}`
);
}
}
// Update existing resource
const [updatedResource] = await trx
.update(siteResources)
@@ -243,8 +274,7 @@ export async function updatePrivateResources(
scheme: resourceData.scheme,
destination: resourceData.destination,
destinationPort: resourceData["destination-port"],
enabled: true, // hardcoded for now
// enabled: resourceData.enabled ?? true,
enabled: resourceEnabled,
alias: resourceData.alias || null,
disableIcmp:
resourceData["disable-icmp"] ||
@@ -263,7 +293,8 @@ export async function updatePrivateResources(
pamMode: resourceData["auth-daemon"]?.pam || "passthrough",
authDaemonMode:
resourceData["auth-daemon"]?.mode || "native",
authDaemonPort: resourceData["auth-daemon"]?.port || 22123
authDaemonPort: resourceData["auth-daemon"]?.port || 22123,
status: resourceStatusFromSite
})
.where(
eq(
@@ -474,6 +505,27 @@ export async function updatePrivateResources(
);
}
if (resourceData.alias) {
const [aliasConflict] = await trx
.select({
siteResourceId: siteResources.siteResourceId
})
.from(siteResources)
.where(
and(
eq(siteResources.orgId, orgId),
eq(siteResources.alias, resourceData.alias)
)
)
.limit(1);
if (aliasConflict) {
throw new Error(
`Alias ${resourceData.alias} already in use by another site resource in org ${orgId}`
);
}
}
const [network] = await trx
.insert(networks)
.values({
@@ -496,8 +548,7 @@ export async function updatePrivateResources(
scheme: resourceData.scheme,
destination: resourceData.destination,
destinationPort: resourceData["destination-port"],
enabled: true, // hardcoded for now
// enabled: resourceData.enabled ?? true,
enabled: resourceEnabled,
alias: resourceData.alias || null,
aliasAddress: aliasAddress,
disableIcmp:
@@ -517,7 +568,8 @@ export async function updatePrivateResources(
pamMode: resourceData["auth-daemon"]?.pam || "passthrough",
authDaemonMode:
resourceData["auth-daemon"]?.mode || "native",
authDaemonPort: resourceData["auth-daemon"]?.port || 22123
authDaemonPort: resourceData["auth-daemon"]?.port || 22123,
status: resourceStatusFromSite
})
.returning();
+37 -15
View File
@@ -25,6 +25,7 @@ import {
rolePolicies,
roleResources,
roles,
Site,
sites,
Target,
TargetHealthCheck,
@@ -74,19 +75,40 @@ export async function updatePublicResources(
)) {
const targetsToUpdate: Target[] = [];
const healthchecksToUpdate: TargetHealthCheck[] = [];
let resource: Resource;
let resourceStatusFromSite: "approved" | "pending" = "approved";
let providedSite: Partial<Site> | undefined;
if (siteId) {
// Use the provided siteId directly, but verify it belongs to the org
[providedSite] = await trx
.select({
siteId: sites.siteId,
type: sites.type,
status: sites.status
})
.from(sites)
.where(and(eq(sites.siteId, siteId), eq(sites.orgId, orgId)))
.limit(1);
resourceStatusFromSite = providedSite?.status ?? "approved";
}
async function createTarget( // reusable function to create a target
resourceId: number,
targetData: TargetData
) {
const targetSiteId = targetData.site;
let site;
let site: Partial<Site> | undefined;
if (targetSiteId) {
// Look up site by niceId
[site] = await trx
.select({ siteId: sites.siteId, type: sites.type })
.select({
siteId: sites.siteId,
type: sites.type,
status: sites.status
})
.from(sites)
.where(
and(
@@ -95,15 +117,9 @@ export async function updatePublicResources(
)
)
.limit(1);
} else if (siteId) {
} else if (siteId && providedSite) {
// Use the provided siteId directly, but verify it belongs to the org
[site] = await trx
.select({ siteId: sites.siteId, type: sites.type })
.from(sites)
.where(
and(eq(sites.siteId, siteId), eq(sites.orgId, orgId))
)
.limit(1);
site = providedSite;
} else {
throw new Error(`Target site is required`);
}
@@ -139,7 +155,7 @@ export async function updatePublicResources(
.insert(targets)
.values({
resourceId: resourceId,
siteId: site.siteId,
siteId: site.siteId!,
ip: targetData.hostname,
mode: resourceData.mode as Target["mode"],
method: targetData.method,
@@ -172,7 +188,7 @@ export async function updatePublicResources(
.insert(targetHealthCheck)
.values({
name: `${targetData.hostname}:${targetData.port}`,
siteId: site.siteId,
siteId: site.siteId!,
targetId: newTarget.targetId,
orgId: orgId,
hcEnabled: healthcheckData?.enabled || false,
@@ -230,7 +246,10 @@ export async function updatePublicResources(
const resourceEnabled =
resourceData.enabled == undefined || resourceData.enabled == null
? true
: resourceData.enabled;
: resourceStatusFromSite === "pending"
? false
: resourceData.enabled;
const resourceSsl =
resourceData.ssl == undefined || resourceData.ssl == null
? true
@@ -406,7 +425,8 @@ export async function updatePublicResources(
? (resourceData["proxy-protocol-version"] ??
1)
: 1,
resourcePolicyId: sharedPolicy.resourcePolicyId
resourcePolicyId: sharedPolicy.resourcePolicyId,
status: resourceStatusFromSite
})
.where(
eq(
@@ -590,7 +610,8 @@ export async function updatePublicResources(
authDaemonPort:
resourceData["auth-daemon"]?.port || 22123,
resourcePolicyId: null,
defaultResourcePolicyId: inlinePolicyId
defaultResourcePolicyId: inlinePolicyId,
status: resourceStatusFromSite
})
.where(
eq(
@@ -1131,6 +1152,7 @@ export async function updatePublicResources(
.values({
orgId,
niceId: resourceNiceId,
status: resourceStatusFromSite,
name: resourceData.name || "Unnamed Resource",
mode: resourceData.mode,
proxyPort: ["http", "ssh", "rdp", "vnc"].includes(
+1 -1
View File
@@ -470,7 +470,7 @@ export const PrivateResourceSchema = z
// proxyPort: z.int().positive().optional(),
"destination-port": z.int().positive().optional(),
destination: z.string().min(1).optional(),
// enabled: z.boolean().default(true),
enabled: z.boolean().default(true),
"tcp-ports": portRangeStringSchema.optional().default("*"),
"udp-ports": portRangeStringSchema.optional().default("*"),
"disable-icmp": z.boolean().optional().default(false),
+1 -1
View File
@@ -2,7 +2,7 @@ import path from "path";
import { fileURLToPath } from "url";
// This is a placeholder value replaced by the build process
export const APP_VERSION = "1.20.0";
export const APP_VERSION = "1.21.0";
export const __FILENAME = fileURLToPath(import.meta.url);
export const __DIRNAME = path.dirname(__FILENAME);
+3 -5
View File
@@ -14,8 +14,6 @@ import {
} from "@server/db";
import logger from "@server/logger";
import { removeTargets } from "@server/routers/newt/targets";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
export type DeleteResourceResult = {
deletedResource: Resource;
@@ -117,10 +115,10 @@ export async function runResourceDeleteSideEffects(
.limit(1);
if (!site) {
throw createHttpError(
HttpCode.NOT_FOUND,
`Site with ID ${target.siteId} not found`
logger.debug(
`Site with ID ${target.siteId} not found during resource delete side effects; skipping target removal`
);
continue;
}
if (site.pubKey && site.type === "newt") {
+82 -3
View File
@@ -1,6 +1,7 @@
import { and, eq, sql } from "drizzle-orm";
import { and, eq, inArray, sql } from "drizzle-orm";
import {
db,
resources,
siteNetworks,
siteResources,
targets,
@@ -97,6 +98,64 @@ export function exceedsSiteAssociatedResourceDeleteLimit(
return resourceCount > MAX_SITE_ASSOCIATED_RESOURCES_FOR_BULK_DELETE;
}
export async function getPendingResourceIdsForSite(
siteId: number,
trx: Transaction | typeof db = db
): Promise<number[]> {
const resourceIds = await getResourceIdsForSite(siteId, trx);
if (resourceIds.length === 0) {
return [];
}
const rows = await trx
.select({ resourceId: resources.resourceId })
.from(resources)
.where(
and(
inArray(resources.resourceId, resourceIds),
eq(resources.status, "pending")
)
);
return rows.map((row) => row.resourceId);
}
export async function getPendingSiteResourceIdsForSite(
siteId: number,
orgId: string,
trx: Transaction | typeof db = db
): Promise<number[]> {
const siteResourceIds = await getSiteResourceIdsForSite(siteId, orgId, trx);
if (siteResourceIds.length === 0) {
return [];
}
const rows = await trx
.select({ siteResourceId: siteResources.siteResourceId })
.from(siteResources)
.where(
and(
inArray(siteResources.siteResourceId, siteResourceIds),
eq(siteResources.status, "pending")
)
);
return rows.map((row) => row.siteResourceId);
}
export async function getPendingAssociatedResourceCountForSite(
siteId: number,
orgId: string,
trx: Transaction | typeof db = db
): Promise<number> {
const [resourceIds, siteResourceIds] = await Promise.all([
getPendingResourceIdsForSite(siteId, trx),
getPendingSiteResourceIdsForSite(siteId, orgId, trx)
]);
return resourceIds.length + siteResourceIds.length;
}
export async function deleteAssociatedResourcesForSite(
siteId: number,
orgId: string,
@@ -105,12 +164,32 @@ export async function deleteAssociatedResourcesForSite(
const resourceIds = await getResourceIdsForSite(siteId, trx);
const siteResourceIds = await getSiteResourceIdsForSite(siteId, orgId, trx);
const [resources, siteResourcesDeleted] = await Promise.all([
const [deletedResources, siteResourcesDeleted] = await Promise.all([
performDeleteResources(resourceIds, trx),
performDeleteSiteResources(siteResourceIds, trx)
]);
return { resources, siteResources: siteResourcesDeleted };
return { resources: deletedResources, siteResources: siteResourcesDeleted };
}
export async function deletePendingAssociatedResourcesForSite(
siteId: number,
orgId: string,
trx: Transaction | typeof db = db
): Promise<DeleteSiteAssociatedResourcesSideEffects> {
const resourceIds = await getPendingResourceIdsForSite(siteId, trx);
const siteResourceIds = await getPendingSiteResourceIdsForSite(
siteId,
orgId,
trx
);
const [deletedResources, siteResourcesDeleted] = await Promise.all([
performDeleteResources(resourceIds, trx),
performDeleteSiteResources(siteResourceIds, trx)
]);
return { resources: deletedResources, siteResources: siteResourcesDeleted };
}
export async function runDeleteSiteAssociatedResourcesSideEffects(
+9
View File
@@ -496,6 +496,7 @@ export function generateRemoteSubnets(
): string[] {
const remoteSubnets = allSiteResources
.filter((sr) => {
if (!sr.enabled) return false;
if (!sr.destination) return false;
if (sr.mode === "cidr") {
@@ -530,6 +531,7 @@ export function generateAliasConfig(allSiteResources: SiteResource[]): Alias[] {
return allSiteResources
.filter(
(sr) =>
sr.enabled &&
sr.aliasAddress &&
((sr.alias && (sr.mode == "host" || sr.mode == "ssh")) ||
(sr.fullDomain && sr.mode == "http"))
@@ -662,6 +664,13 @@ export async function generateSubnetProxyTargetV2(
subnet: string | null;
}[]
): Promise<SubnetProxyTargetV2[] | undefined> {
if (!siteResource.enabled) {
logger.debug(
`Site resource ${siteResource.siteResourceId} is disabled, skipping target generation.`
);
return;
}
if (clients.length === 0) {
logger.debug(
`No clients have access to site resource ${siteResource.siteResourceId}, skipping target generation.`
+34 -3
View File
@@ -15,10 +15,41 @@ function getSegmentRegex(patternPart: string): RegExp {
return regex;
}
// Decodes percent-encoding (so an encoded slash like `%2F` is treated as a
// real path separator, matching what most backends will do) and then
// resolves `.` / `..` segments, so a request like `/public%2F..%2Fadmin/`
// or `/public/../admin/` is matched as `/admin/`, not as a literal segment
// or a wildcard-swallowed sequence under `/public/*`.
function decodeAndResolvePath(p: string): string[] {
const rawParts = p.split("/").filter(Boolean);
const resolved: string[] = [];
for (const rawPart of rawParts) {
let part: string;
try {
part = decodeURIComponent(rawPart);
} catch {
part = rawPart;
}
// an encoded slash can turn one raw segment into several real ones
for (const segment of part.split("/").filter(Boolean)) {
if (segment === ".") {
continue;
} else if (segment === "..") {
resolved.pop();
} else {
resolved.push(segment);
}
}
}
return resolved;
}
export function isPathAllowed(pattern: string, path: string): boolean {
const normalize = (p: string) => p.split("/").filter(Boolean);
const patternParts = normalize(pattern);
const pathParts = normalize(path);
const patternParts = pattern.split("/").filter(Boolean);
const pathParts = decodeAndResolvePath(path);
function matchSegments(
patternIndex: number,
+30 -16
View File
@@ -1561,9 +1561,19 @@ export async function handleMessagingForUpdatedSiteResource(
updatedSiteResource.udpPortRangeString ||
existingSiteResource.disableIcmp !==
updatedSiteResource.disableIcmp);
// Toggling enabled on/off doesn't change any of the fields above, but it
// does change whether targets/peer data should exist at all, so it needs
// to drive the same old->new diff machinery: going enabled->disabled
// diffs "real data" against "nothing" (a remove), and disabled->enabled
// diffs "nothing" against "real data" (an add). generateSubnetProxyTargetV2/
// generateRemoteSubnets/generateAliasConfig already return nothing for a
// disabled resource, so no other changes are needed here.
const enabledChanged =
existingSiteResource &&
existingSiteResource.enabled !== updatedSiteResource.enabled;
logger.debug(
`handleMessagingForUpdatedSiteResource: change flags destinationChanged=${Boolean(destinationChanged)} destinationPortChanged=${Boolean(destinationPortChanged)} aliasChanged=${Boolean(aliasChanged)} fullDomainChanged=${Boolean(fullDomainChanged)} sslChanged=${Boolean(sslChanged)} portRangesChanged=${Boolean(portRangesChanged)}`
`handleMessagingForUpdatedSiteResource: change flags destinationChanged=${Boolean(destinationChanged)} destinationPortChanged=${Boolean(destinationPortChanged)} aliasChanged=${Boolean(aliasChanged)} fullDomainChanged=${Boolean(fullDomainChanged)} sslChanged=${Boolean(sslChanged)} portRangesChanged=${Boolean(portRangesChanged)} enabledChanged=${Boolean(enabledChanged)}`
);
// if the existingSiteResource is undefined (new resource) we don't need to do anything here, the rebuild above handled it all
@@ -1574,14 +1584,16 @@ export async function handleMessagingForUpdatedSiteResource(
fullDomainChanged ||
sslChanged ||
portRangesChanged ||
destinationPortChanged
destinationPortChanged ||
enabledChanged
) {
const shouldUpdateTargets =
destinationChanged ||
sslChanged ||
portRangesChanged ||
fullDomainChanged ||
destinationPortChanged;
destinationPortChanged ||
enabledChanged;
logger.debug(
`handleMessagingForUpdatedSiteResource: entering unchanged-site update path shouldUpdateTargets=${shouldUpdateTargets}`
@@ -1657,20 +1669,22 @@ export async function handleMessagingForUpdatedSiteResource(
peerDataUpdateBatch.push({
clientId: client.clientId,
siteId,
remoteSubnets: destinationChanged
? {
oldRemoteSubnets: !oldDestinationStillInUseBySite
? generateRemoteSubnets([
existingSiteResource
])
: [],
newRemoteSubnets: generateRemoteSubnets([
updatedSiteResource
])
}
: undefined,
remoteSubnets:
destinationChanged || enabledChanged
? {
oldRemoteSubnets:
!oldDestinationStillInUseBySite
? generateRemoteSubnets([
existingSiteResource
])
: [],
newRemoteSubnets: generateRemoteSubnets([
updatedSiteResource
])
}
: undefined,
aliases:
aliasChanged || fullDomainChanged // the full domain is sent down as an alias
aliasChanged || fullDomainChanged || enabledChanged // the full domain is sent down as an alias
? {
oldAliases: generateAliasConfig([
existingSiteResource
+49 -18
View File
@@ -8,26 +8,42 @@ const STATUS_HISTORY_CACHE_TTL = 60; // seconds
function statusHistoryCacheKey(
entityType: string,
entityId: number,
days: number
days: number,
tzOffsetMinutes: number
): string {
return `statusHistory:${entityType}:${entityId}:${days}`;
return `statusHistory:${entityType}:${entityId}:${days}:${tzOffsetMinutes}`;
}
// Returns the epoch seconds of the most recent local-calendar-day midnight,
// where "local" is defined by tzOffsetMinutes (minutes to ADD to UTC to get
// local time, e.g. Australia/Sydney standard time is 600). Defaults to 0
// (UTC) so callers that don't pass a timezone keep the original behavior.
function localMidnightSec(tzOffsetMinutes: number): number {
const localNow = new Date(Date.now() + tzOffsetMinutes * 60_000);
localNow.setUTCHours(0, 0, 0, 0);
return Math.floor(localNow.getTime() / 1000) - tzOffsetMinutes * 60;
}
export async function getCachedStatusHistory(
entityType: string,
entityId: number,
days: number
days: number,
tzOffsetMinutes: number = 0
): Promise<StatusHistoryResponse> {
const cacheKey = statusHistoryCacheKey(entityType, entityId, days);
const cacheKey = statusHistoryCacheKey(
entityType,
entityId,
days,
tzOffsetMinutes
);
const cached = await cache.get<StatusHistoryResponse>(cacheKey);
if (cached !== undefined) {
return cached;
}
// Anchor to UTC midnight so the query window aligns with stable calendar days
const utcToday = new Date();
utcToday.setUTCHours(0, 0, 0, 0);
const todayMidnightSec = Math.floor(utcToday.getTime() / 1000);
// Anchor to local midnight (UTC when tzOffsetMinutes is 0) so the query
// window aligns with stable calendar days for the requesting client
const todayMidnightSec = localMidnightSec(tzOffsetMinutes);
const startSec = todayMidnightSec - days * 86400;
const events = await logsDb
@@ -63,7 +79,8 @@ export async function getCachedStatusHistory(
const { buckets, totalDowntime } = computeBuckets(
events,
days,
priorStatus
priorStatus,
tzOffsetMinutes
);
const totalWindow = days * 86400;
const overallUptime =
@@ -99,11 +116,19 @@ export const statusHistoryQuerySchema = z
days: z
.string()
.optional()
.transform((v) => (v ? parseInt(v, 10) : 90))
.transform((v) => (v ? parseInt(v, 10) : 90)),
// Minutes to add to UTC to get the requesting client's local time
// (e.g. Australia/Sydney standard time is 600). Optional and
// defaults to 0 (UTC) so older clients keep the prior behavior.
tzOffsetMinutes: z
.string()
.optional()
.transform((v) => (v ? parseInt(v, 10) : 0))
})
.pipe(
z.object({
days: z.number().int().min(1).max(365)
days: z.number().int().min(1).max(365),
tzOffsetMinutes: z.number().int().min(-720).max(840)
})
);
@@ -133,15 +158,15 @@ export function computeBuckets(
id: number;
}[],
days: number,
priorStatus: string | null = null
priorStatus: string | null = null,
tzOffsetMinutes: number = 0
): { buckets: StatusHistoryDayBucket[]; totalDowntime: number } {
const nowSec = Math.floor(Date.now() / 1000);
// Anchor bucket boundaries to UTC midnight so dates are stable calendar days
// and don't drift as the cache expires and is recomputed
const utcToday = new Date();
utcToday.setUTCHours(0, 0, 0, 0);
const todayMidnightSec = Math.floor(utcToday.getTime() / 1000);
// Anchor bucket boundaries to local midnight (UTC when tzOffsetMinutes is
// 0) so dates are stable calendar days for the requesting client and
// don't drift as the cache expires and is recomputed
const todayMidnightSec = localMidnightSec(tzOffsetMinutes);
const buckets: StatusHistoryDayBucket[] = [];
let totalDowntime = 0;
@@ -237,7 +262,13 @@ export function computeBuckets(
)
: 100;
const dateStr = new Date(dayStartSec * 1000).toISOString().slice(0, 10);
// Shift by the client's offset before formatting so the label reflects
// their local calendar date rather than the UTC date of dayStartSec
const dateStr = new Date(
(dayStartSec + tzOffsetMinutes * 60) * 1000
)
.toISOString()
.slice(0, 10);
const hasAnyData = currentStatus !== null || dayEvents.length > 0;
+10
View File
@@ -2,6 +2,7 @@ import {
db,
Org,
orgs,
resourceAccessToken,
resources,
siteResources,
sites,
@@ -83,6 +84,15 @@ export async function removeUserFromOrg(
.delete(userOrgs)
.where(and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, org.orgId)));
await trx
.delete(resourceAccessToken)
.where(
and(
eq(resourceAccessToken.userId, userId),
eq(resourceAccessToken.orgId, org.orgId)
)
);
await trx.delete(userResources).where(
and(
eq(userResources.userId, userId),