Merge pull request #2780 from fosrl/dev

1.17.0

.github/CODEOWNERS

@@ -1 +0,0 @@
-* @oschwartz10612 @miloschwartz

install/config/crowdsec/traefik_config.yml

@@ -86,8 +86,6 @@ entryPoints:
     http:
       tls:
         certResolver: "letsencrypt"
-      middlewares:
-        - crowdsec@file
       encodedCharacters:
         allowEncodedSlash: true
         allowEncodedQuestionMark: true

messages/en-US.json

@@ -2116,7 +2116,6 @@
     "domainPickerFreeProvidedDomain": "Free Provided Domain",
     "domainPickerVerified": "Verified",
     "domainPickerUnverified": "Unverified",
-    "domainPickerManual": "Manual",
     "domainPickerInvalidSubdomainStructure": "This subdomain contains invalid characters or structure. It will be sanitized automatically when you save.",
     "domainPickerError": "Error",
     "domainPickerErrorLoadDomains": "Failed to load organization domains",

server/lib/traefik/getTraefikConfig.ts

@@ -479,7 +479,10 @@ export async function getTraefikConfig(
 
                         // TODO: HOW TO HANDLE ^^^^^^ BETTER
                         const anySitesOnline = targets.some(
-                            (target) => target.site.online
+                            (target) =>
+                            target.site.online ||
+                            target.site.type === "local" ||
+                            target.site.type === "wireguard"
                         );
 
                         return (
@@ -492,7 +495,7 @@ export async function getTraefikConfig(
                                     if (target.health == "unhealthy") {
                                         return false;
                                     }
-
+                                    
                                     // If any sites are online, exclude offline sites
                                     if (anySitesOnline && !target.site.online) {
                                         return false;
@@ -607,7 +610,10 @@ export async function getTraefikConfig(
                     servers: (() => {
                         // Check if any sites are online
                         const anySitesOnline = targets.some(
-                            (target) => target.site.online
+                            (target) =>
+                            target.site.online ||
+                            target.site.type === "local" ||
+                            target.site.type === "wireguard"
                         );
 
                         return targets
@@ -615,7 +621,7 @@ export async function getTraefikConfig(
                                 if (!target.enabled) {
                                     return false;
                                 }
-
+                                
                                 // If any sites are online, exclude offline sites
                                 if (anySitesOnline && !target.site.online) {
                                     return false;

server/private/lib/traefik/getTraefikConfig.ts

@@ -671,7 +671,10 @@ export async function getTraefikConfig(
 
                         // TODO: HOW TO HANDLE ^^^^^^ BETTER
                         const anySitesOnline = targets.some(
-                            (target) => target.site.online
+                            (target) =>
+                            target.site.online ||
+                            target.site.type === "local" ||
+                            target.site.type === "wireguard"
                         );
 
                         return (
@@ -799,7 +802,10 @@ export async function getTraefikConfig(
                     servers: (() => {
                         // Check if any sites are online
                         const anySitesOnline = targets.some(
-                            (target) => target.site.online
+                            (target) =>
+                            target.site.online ||
+                            target.site.type === "local" ||
+                            target.site.type === "wireguard"
                         );
 
                         return targets

server/routers/newt/pingAccumulator.ts

@@ -1,6 +1,6 @@
 import { db } from "@server/db";
 import { sites, clients, olms } from "@server/db";
-import { inArray } from "drizzle-orm";
+import { eq, inArray } from "drizzle-orm";
 import logger from "@server/logger";
 
 /**
@@ -21,7 +21,7 @@ import logger from "@server/logger";
  */
 
 const FLUSH_INTERVAL_MS = 10_000; // Flush every 10 seconds
-const MAX_RETRIES = 5;
+const MAX_RETRIES = 2;
 const BASE_DELAY_MS = 50;
 
 // ── Site (newt) pings ──────────────────────────────────────────────────
@@ -36,14 +36,6 @@ const pendingOlmArchiveResets: Set<string> = new Set();
 
 let flushTimer: NodeJS.Timeout | null = null;
 
-/**
- * Guard that prevents two flush cycles from running concurrently.
- * setInterval does not await async callbacks, so without this a slow flush
- * (e.g. due to DB latency) would overlap with the next scheduled cycle and
- * the two concurrent bulk UPDATEs would deadlock each other.
- */
-let isFlushing = false;
-
 // ── Public API ─────────────────────────────────────────────────────────
 
 /**
@@ -80,12 +72,6 @@ export function recordClientPing(
 
 /**
  * Flush all accumulated site pings to the database.
- *
- * Each batch of up to BATCH_SIZE rows is written with a **single** UPDATE
- * statement. We use the maximum timestamp across the batch so that `lastPing`
- * reflects the most recent ping seen for any site in the group. This avoids
- * the multi-statement transaction that previously created additional
- * row-lock ordering hazards.
  */
 async function flushSitePingsToDb(): Promise<void> {
     if (pendingSitePings.size === 0) {
@@ -97,35 +83,55 @@ async function flushSitePingsToDb(): Promise<void> {
     const pingsToFlush = new Map(pendingSitePings);
     pendingSitePings.clear();
 
-    const entries = Array.from(pingsToFlush.entries());
+    // Sort by siteId for consistent lock ordering (prevents deadlocks)
+    const sortedEntries = Array.from(pingsToFlush.entries()).sort(
+        ([a], [b]) => a - b
+    );
 
     const BATCH_SIZE = 50;
-    for (let i = 0; i < entries.length; i += BATCH_SIZE) {
-        const batch = entries.slice(i, i + BATCH_SIZE);
-
-        // Use the latest timestamp in the batch so that `lastPing` always
-        // moves forward. Using a single timestamp for the whole batch means
-        // we only ever need one UPDATE statement (no transaction).
-        const maxTimestamp = Math.max(...batch.map(([, ts]) => ts));
-        const siteIds = batch.map(([id]) => id);
+    for (let i = 0; i < sortedEntries.length; i += BATCH_SIZE) {
+        const batch = sortedEntries.slice(i, i + BATCH_SIZE);
 
         try {
             await withRetry(async () => {
-                await db
-                    .update(sites)
-                    .set({
-                        online: true,
-                        lastPing: maxTimestamp
-                    })
-                    .where(inArray(sites.siteId, siteIds));
+                // Group by timestamp for efficient bulk updates
+                const byTimestamp = new Map<number, number[]>();
+                for (const [siteId, timestamp] of batch) {
+                    const group = byTimestamp.get(timestamp) || [];
+                    group.push(siteId);
+                    byTimestamp.set(timestamp, group);
+                }
+
+                if (byTimestamp.size === 1) {
+                    const [timestamp, siteIds] = Array.from(
+                        byTimestamp.entries()
+                    )[0];
+                    await db
+                        .update(sites)
+                        .set({
+                            online: true,
+                            lastPing: timestamp
+                        })
+                        .where(inArray(sites.siteId, siteIds));
+                } else {
+                    await db.transaction(async (tx) => {
+                        for (const [timestamp, siteIds] of byTimestamp) {
+                            await tx
+                                .update(sites)
+                                .set({
+                                    online: true,
+                                    lastPing: timestamp
+                                })
+                                .where(inArray(sites.siteId, siteIds));
+                        }
+                    });
+                }
             }, "flushSitePingsToDb");
         } catch (error) {
             logger.error(
                 `Failed to flush site ping batch (${batch.length} sites), re-queuing for next cycle`,
                 { error }
             );
-            // Re-queue only if the preserved timestamp is newer than any
-            // update that may have landed since we snapshotted.
             for (const [siteId, timestamp] of batch) {
                 const existing = pendingSitePings.get(siteId);
                 if (!existing || existing < timestamp) {
@@ -138,8 +144,6 @@ async function flushSitePingsToDb(): Promise<void> {
 
 /**
  * Flush all accumulated client (OLM) pings to the database.
- *
- * Same single-UPDATE-per-batch approach as `flushSitePingsToDb`.
  */
 async function flushClientPingsToDb(): Promise<void> {
     if (pendingClientPings.size === 0 && pendingOlmArchiveResets.size === 0) {
@@ -155,25 +159,51 @@ async function flushClientPingsToDb(): Promise<void> {
 
     // ── Flush client pings ─────────────────────────────────────────────
     if (pingsToFlush.size > 0) {
-        const entries = Array.from(pingsToFlush.entries());
+        const sortedEntries = Array.from(pingsToFlush.entries()).sort(
+            ([a], [b]) => a - b
+        );
 
         const BATCH_SIZE = 50;
-        for (let i = 0; i < entries.length; i += BATCH_SIZE) {
-            const batch = entries.slice(i, i + BATCH_SIZE);
-
-            const maxTimestamp = Math.max(...batch.map(([, ts]) => ts));
-            const clientIds = batch.map(([id]) => id);
+        for (let i = 0; i < sortedEntries.length; i += BATCH_SIZE) {
+            const batch = sortedEntries.slice(i, i + BATCH_SIZE);
 
             try {
                 await withRetry(async () => {
-                    await db
-                        .update(clients)
-                        .set({
-                            lastPing: maxTimestamp,
-                            online: true,
-                            archived: false
-                        })
-                        .where(inArray(clients.clientId, clientIds));
+                    const byTimestamp = new Map<number, number[]>();
+                    for (const [clientId, timestamp] of batch) {
+                        const group = byTimestamp.get(timestamp) || [];
+                        group.push(clientId);
+                        byTimestamp.set(timestamp, group);
+                    }
+
+                    if (byTimestamp.size === 1) {
+                        const [timestamp, clientIds] = Array.from(
+                            byTimestamp.entries()
+                        )[0];
+                        await db
+                            .update(clients)
+                            .set({
+                                lastPing: timestamp,
+                                online: true,
+                                archived: false
+                            })
+                            .where(inArray(clients.clientId, clientIds));
+                    } else {
+                        await db.transaction(async (tx) => {
+                            for (const [timestamp, clientIds] of byTimestamp) {
+                                await tx
+                                    .update(clients)
+                                    .set({
+                                        lastPing: timestamp,
+                                        online: true,
+                                        archived: false
+                                    })
+                                    .where(
+                                        inArray(clients.clientId, clientIds)
+                                    );
+                            }
+                        });
+                    }
                 }, "flushClientPingsToDb");
             } catch (error) {
                 logger.error(
@@ -230,12 +260,7 @@ export async function flushPingsToDb(): Promise<void> {
 
 /**
  * Simple retry wrapper with exponential backoff for transient errors
- * (deadlocks, connection timeouts, unexpected disconnects).
- *
- * PostgreSQL deadlocks (40P01) are always safe to retry: the database
- * guarantees exactly one winner per deadlock pair, so the loser just needs
- * to try again. MAX_RETRIES is intentionally higher than typical connection
- * retry budgets to give deadlock victims enough chances to succeed.
+ * (connection timeouts, unexpected disconnects).
  */
 async function withRetry<T>(
     operation: () => Promise<T>,
@@ -252,8 +277,7 @@ async function withRetry<T>(
                 const jitter = Math.random() * baseDelay;
                 const delay = baseDelay + jitter;
                 logger.warn(
-                    `Transient DB error in ${context}, retrying attempt ${attempt}/${MAX_RETRIES} after ${delay.toFixed(0)}ms`,
-                    { code: error?.code ?? error?.cause?.code }
+                    `Transient DB error in ${context}, retrying attempt ${attempt}/${MAX_RETRIES} after ${delay.toFixed(0)}ms`
                 );
                 await new Promise((resolve) => setTimeout(resolve, delay));
                 continue;
@@ -264,14 +288,14 @@ async function withRetry<T>(
 }
 
 /**
- * Detect transient errors that are safe to retry.
+ * Detect transient connection errors that are safe to retry.
  */
 function isTransientError(error: any): boolean {
     if (!error) return false;
 
     const message = (error.message || "").toLowerCase();
     const causeMessage = (error.cause?.message || "").toLowerCase();
-    const code = error.code || error.cause?.code || "";
+    const code = error.code || "";
 
     // Connection timeout / terminated
     if (
@@ -284,17 +308,12 @@ function isTransientError(error: any): boolean {
         return true;
     }
 
-    // PostgreSQL deadlock detected — always safe to retry (one winner guaranteed)
+    // PostgreSQL deadlock
     if (code === "40P01" || message.includes("deadlock")) {
         return true;
     }
 
-    // PostgreSQL serialization failure
-    if (code === "40001") {
-        return true;
-    }
-
-    // ECONNRESET, ECONNREFUSED, EPIPE, ETIMEDOUT
+    // ECONNRESET, ECONNREFUSED, EPIPE
     if (
         code === "ECONNRESET" ||
         code === "ECONNREFUSED" ||
@@ -318,26 +337,12 @@ export function startPingAccumulator(): void {
     }
 
     flushTimer = setInterval(async () => {
-        // Skip this tick if the previous flush is still in progress.
-        // setInterval does not await async callbacks, so without this guard
-        // two flush cycles can run concurrently and deadlock each other on
-        // overlapping bulk UPDATE statements.
-        if (isFlushing) {
-            logger.debug(
-                "Ping accumulator: previous flush still in progress, skipping cycle"
-            );
-            return;
-        }
-
-        isFlushing = true;
         try {
             await flushPingsToDb();
         } catch (error) {
             logger.error("Unhandled error in ping accumulator flush", {
                 error
             });
-        } finally {
-            isFlushing = false;
         }
     }, FLUSH_INTERVAL_MS);
 
@@ -359,22 +364,7 @@ export async function stopPingAccumulator(): Promise<void> {
         flushTimer = null;
     }
 
-    // Final flush to persist any remaining pings.
-    // Wait for any in-progress flush to finish first so we don't race.
-    if (isFlushing) {
-        logger.debug(
-            "Ping accumulator: waiting for in-progress flush before stopping…"
-        );
-        await new Promise<void>((resolve) => {
-            const poll = setInterval(() => {
-                if (!isFlushing) {
-                    clearInterval(poll);
-                    resolve();
-                }
-            }, 50);
-        });
-    }
-
+    // Final flush to persist any remaining pings
     try {
         await flushPingsToDb();
     } catch (error) {
@@ -389,4 +379,4 @@ export async function stopPingAccumulator(): Promise<void> {
  */
 export function getPendingPingCount(): number {
     return pendingSitePings.size + pendingClientPings.size;
-}
+}

server/setup/scriptsPg/1.17.0.ts

@@ -235,9 +235,7 @@ export default async function migration() {
             for (const row of existingUserInviteRoles) {
                 await db.execute(sql`
                     INSERT INTO "userInviteRoles" ("inviteId", "roleId")
-                    SELECT ${row.inviteId}, ${row.roleId}
-                    WHERE EXISTS (SELECT 1 FROM "userInvites" WHERE "inviteId" = ${row.inviteId})
-                      AND EXISTS (SELECT 1 FROM "roles" WHERE "roleId" = ${row.roleId})
+                    VALUES (${row.inviteId}, ${row.roleId})
                     ON CONFLICT DO NOTHING
                 `);
             }
@@ -260,10 +258,7 @@ export default async function migration() {
             for (const row of existingUserOrgRoles) {
                 await db.execute(sql`
                     INSERT INTO "userOrgRoles" ("userId", "orgId", "roleId")
-                    SELECT ${row.userId}, ${row.orgId}, ${row.roleId}
-                    WHERE EXISTS (SELECT 1 FROM "user" WHERE "id" = ${row.userId})
-                      AND EXISTS (SELECT 1 FROM "orgs" WHERE "orgId" = ${row.orgId})
-                      AND EXISTS (SELECT 1 FROM "roles" WHERE "roleId" = ${row.roleId})
+                    VALUES (${row.userId}, ${row.orgId}, ${row.roleId})
                     ON CONFLICT DO NOTHING
                 `);
             }

server/setup/scriptsSqlite/1.17.0.ts

@@ -145,7 +145,7 @@ export default async function migration() {
             ).run();
 
             db.prepare(
-                `INSERT INTO '__new_userOrgs'("userId", "orgId", "isOwner", "autoProvisioned", "pamUsername") SELECT "userId", "orgId", "isOwner", "autoProvisioned", "pamUsername" FROM 'userOrgs' WHERE EXISTS (SELECT 1 FROM 'user' WHERE id = userOrgs.userId) AND EXISTS (SELECT 1 FROM 'orgs' WHERE orgId = userOrgs.orgId);`
+                `INSERT INTO '__new_userOrgs'("userId", "orgId", "isOwner", "autoProvisioned", "pamUsername") SELECT "userId", "orgId", "isOwner", "autoProvisioned", "pamUsername" FROM 'userOrgs';`
             ).run();
             db.prepare(`DROP TABLE 'userOrgs';`).run();
             db.prepare(
@@ -246,15 +246,12 @@ export default async function migration() {
         // Re-insert the preserved invite role assignments into the new userInviteRoles table
         if (existingUserInviteRoles.length > 0) {
             const insertUserInviteRole = db.prepare(
-                `INSERT OR IGNORE INTO 'userInviteRoles' ("inviteId", "roleId")
-                 SELECT ?, ?
-                 WHERE EXISTS (SELECT 1 FROM 'userInvites' WHERE inviteId = ?)
-                   AND EXISTS (SELECT 1 FROM 'roles' WHERE roleId = ?)`
+                `INSERT OR IGNORE INTO 'userInviteRoles' ("inviteId", "roleId") VALUES (?, ?)`
             );
 
             const insertAll = db.transaction(() => {
                 for (const row of existingUserInviteRoles) {
-                    insertUserInviteRole.run(row.inviteId, row.roleId, row.inviteId, row.roleId);
+                    insertUserInviteRole.run(row.inviteId, row.roleId);
                 }
             });
 
@@ -268,16 +265,12 @@ export default async function migration() {
         // Re-insert the preserved role assignments into the new userOrgRoles table
         if (existingUserOrgRoles.length > 0) {
             const insertUserOrgRole = db.prepare(
-                `INSERT OR IGNORE INTO 'userOrgRoles' ("userId", "orgId", "roleId")
-                 SELECT ?, ?, ?
-                 WHERE EXISTS (SELECT 1 FROM 'user' WHERE id = ?)
-                   AND EXISTS (SELECT 1 FROM 'orgs' WHERE orgId = ?)
-                   AND EXISTS (SELECT 1 FROM 'roles' WHERE roleId = ?)`
+                `INSERT OR IGNORE INTO 'userOrgRoles' ("userId", "orgId", "roleId") VALUES (?, ?, ?)`
             );
 
             const insertAll = db.transaction(() => {
                 for (const row of existingUserOrgRoles) {
-                    insertUserOrgRole.run(row.userId, row.orgId, row.roleId, row.userId, row.orgId, row.roleId);
+                    insertUserOrgRole.run(row.userId, row.orgId, row.roleId);
                 }
             });
 

src/app/[orgId]/settings/domains/[domainId]/page.tsx

@@ -10,7 +10,6 @@ import { authCookieHeader } from "@app/lib/api/cookies";
 import { GetDNSRecordsResponse } from "@server/routers/domain";
 import DNSRecordsTable from "@app/components/DNSRecordTable";
 import DomainCertForm from "@app/components/DomainCertForm";
-import { build } from "@server/build";
 
 interface DomainSettingsPageProps {
     params: Promise<{ domainId: string; orgId: string }>;
@@ -66,14 +65,12 @@ export default async function DomainSettingsPage({
                 )}
             </div>
             <div className="space-y-6">
-                {build != "oss" && env.flags.usePangolinDns ? (
-                    <DomainInfoCard
-                        failed={domain.failed}
-                        verified={domain.verified}
-                        type={domain.type}
-                        errorMessage={domain.errorMessage}
-                    />
-                ) : null}
+                <DomainInfoCard
+                    failed={domain.failed}
+                    verified={domain.verified}
+                    type={domain.type}
+                    errorMessage={domain.errorMessage}
+                />
 
                 <DNSRecordsTable records={dnsRecords} type={domain.type} />
 

src/components/CreateDomainForm.tsx

@@ -154,7 +154,7 @@ export default function CreateDomainForm({
 
     const punycodePreview = useMemo(() => {
         if (!baseDomain) return "";
-        const punycode = toPunycode(baseDomain.toLowerCase());
+        const punycode = toPunycode(baseDomain);
         return punycode !== baseDomain.toLowerCase() ? punycode : "";
     }, [baseDomain]);
 
@@ -239,24 +239,21 @@ export default function CreateDomainForm({
                             className="space-y-4"
                             id="create-domain-form"
                         >
-                            {build != "oss" && env.flags.usePangolinDns ? (
-                                <FormField
-                                    control={form.control}
-                                    name="type"
-                                    render={({ field }) => (
-                                        <FormItem>
-                                            <StrategySelect
-                                                options={domainOptions}
-                                                defaultValue={field.value}
-                                                onChange={field.onChange}
-                                                cols={1}
-                                            />
-                                            <FormMessage />
-                                        </FormItem>
-                                    )}
-                                />
-                            ) : null}
-
+                            <FormField
+                                control={form.control}
+                                name="type"
+                                render={({ field }) => (
+                                    <FormItem>
+                                        <StrategySelect
+                                            options={domainOptions}
+                                            defaultValue={field.value}
+                                            onChange={field.onChange}
+                                            cols={1}
+                                        />
+                                        <FormMessage />
+                                    </FormItem>
+                                )}
+                            />
                             <FormField
                                 control={form.control}
                                 name="baseDomain"

src/components/DomainPicker.tsx

@@ -509,11 +509,9 @@ export default function DomainPicker({
                                         <span className="truncate">
                                             {selectedBaseDomain.domain}
                                         </span>
-                                        {selectedBaseDomain.verified &&
-                                            selectedBaseDomain.domainType !==
-                                                "wildcard" && (
-                                                <CheckCircle2 className="h-3 w-3 text-green-500 shrink-0" />
-                                            )}
+                                        {selectedBaseDomain.verified && (
+                                            <CheckCircle2 className="h-3 w-3 text-green-500 shrink-0" />
+                                        )}
                                     </div>
                                 ) : (
                                     t("domainPickerSelectBaseDomain")
@@ -576,23 +574,14 @@ export default function DomainPicker({
                                                                     }
                                                                 </span>
                                                                 <span className="text-xs text-muted-foreground">
-                                                                    {orgDomain.type ===
-                                                                    "wildcard"
+                                                                    {orgDomain.type.toUpperCase()}{" "}
+                                                                    •{" "}
+                                                                    {orgDomain.verified
                                                                         ? t(
-                                                                              "domainPickerManual"
+                                                                              "domainPickerVerified"
                                                                           )
-                                                                        : (
-                                                                              <>
-                                                                                  {orgDomain.type.toUpperCase()}{" "}
-                                                                                  •{" "}
-                                                                                  {orgDomain.verified
-                                                                                      ? t(
-                                                                                            "domainPickerVerified"
-                                                                                        )
-                                                                                      : t(
-                                                                                            "domainPickerUnverified"
-                                                                                        )}
-                                                                              </>
+                                                                        : t(
+                                                                              "domainPickerUnverified"
                                                                           )}
                                                                 </span>
                                                             </div>

src/components/PendingSitesTable.tsx

@@ -333,8 +333,7 @@ export default function PendingSitesTable({
                         "jupiter",
                         "saturn",
                         "uranus",
-                        "neptune",
-                        "pluto"
+                        "neptune"
                     ].includes(originalRow.exitNodeName.toLowerCase());
 
                 if (isCloudNode) {

src/components/SitesTable.tsx

@@ -342,8 +342,7 @@ export default function SitesTable({
                         "jupiter",
                         "saturn",
                         "uranus",
-                        "neptune",
-                        "pluto"
+                        "neptune"
                     ].includes(originalRow.exitNodeName.toLowerCase());
 
                 if (isCloudNode) {