Fix cache separation for versions

.gitignore

@@ -46,7 +46,6 @@ public/branding
 server/db/index.ts
 server/build.ts
 postgres/
-dynamic/
 *.mmdb
 scratch/
 tsconfig.json

cli/commands/clearExitNodes.ts

@@ -23,7 +23,7 @@ export const clearExitNodes: CommandModule<
             // Delete all exit nodes
             const deletedCount = await db
                 .delete(exitNodes)
-                .where(eq(exitNodes.exitNodeId, exitNodes.exitNodeId))  .returning();; // delete all
+                .where(eq(exitNodes.exitNodeId, exitNodes.exitNodeId)).returning();; // delete all
 
             console.log(`Deleted ${deletedCount.length} exit node(s) from the database`);
 

config/ha-reference/README.md

@@ -0,0 +1,3 @@
+## Example Docker Reference HA Deployment
+
+This directory contains basic config for a highly available deployment of Pangolin with two nodes. For more information [refer to the docs](/self-host/clustering/understanding-clustering).

config/ha-reference/database/docker-compose.yml

@@ -0,0 +1,23 @@
+services:
+  postgres:
+    image: postgres:17
+    container_name: postgres
+    environment:
+      POSTGRES_DB: postgres # Default database name
+      POSTGRES_USER: postgres # Default user
+      POSTGRES_PASSWORD: password # Default password (change for production!)
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+    restart: always
+
+  redis:
+    image: redis:latest
+    container_name: redis
+    ports:
+      - "6379:6379"
+    restart: always
+
+volumes:
+  postgres_data:

config/ha-reference/node1/config/config.yml

@@ -0,0 +1,38 @@
+# To see all available options, please visit the docs:
+# https://docs.pangolin.net/
+
+gerbil:
+    start_port: 51820
+    base_endpoint: "<THIS_NODE_EXTERNAL_IP>"
+    exit_node_name: "node1"
+
+app:
+    dashboard_url: "https://pangolin.example.com"
+    log_level: "info"
+
+postgres:
+    connection_string: postgresql://<POSTGRES_USERNAME>:<POSTGRES_PASSWORD>@<POSTGRES_INTERNAL_HOST>:5432/postgres
+
+traefik:
+    site_types: ["newt"] # Wireguard and local sites are not support in clustering
+    file_mode: true # Pangolin will generate and save yaml files in a shared volume
+
+server:
+    secret: "<SECRET>"
+    cors:
+        origins: ["https://pangolin.example.com"]
+        methods: ["GET", "POST", "PUT", "DELETE", "PATCH"]
+        allowed_headers: ["X-CSRF-Token", "Content-Type"]
+        credentials: false
+    maxmind_db_path: "./config/GeoLite2-Country.mmdb" # Make sure to download and place into the config dir
+    maxmind_asn_path: "./config/GeoLite2-ASN.mmdb"
+
+flags:
+    require_email_verification: false
+    disable_signup_without_invite: true
+    disable_user_create_org: false
+    allow_raw_resources: false
+    enable_acme_cert_sync: false
+    disable_local_sites: true
+    disable_basic_wireguard_sites: true
+    disable_config_managed_domains: true

config/ha-reference/node1/config/dynamic/dynamic_config.yml

@@ -0,0 +1,67 @@
+http:
+  middlewares:
+    badger:
+      plugin:
+        badger:
+          disableForwardAuth: true
+
+  routers:
+    # Next.js router (handles everything except API and WebSocket paths)
+    next-router:
+      rule: "!PathPrefix(`/api/v1`)"
+      service: next-service
+      entryPoints:
+        - dashboard
+      middlewares:
+        - badger
+
+    # API router (handles /api/v1 paths)
+    api-router:
+      rule: "PathPrefix(`/api/v1`)"
+      service: api-service
+      entryPoints:
+        - dashboard
+      middlewares:
+        - badger
+
+    # WebSocket router
+    ws-router:
+      rule: "PathPrefix(`/`)"
+      service: api-service
+      entryPoints:
+        - dashboard
+      middlewares:
+        - badger
+
+  services:
+    next-service:
+      loadBalancer:
+        servers:
+          - url: "http://pangolin:3002"  # Next.js server
+
+    api-service:
+      loadBalancer:
+        servers:
+          - url: "http://pangolin:3000"  # API/WebSocket server
+
+tcp:
+  serversTransports:
+    pp-transport-v1:
+      proxyProtocol:
+        version: 1
+    pp-transport-v2:
+      proxyProtocol:
+        version: 2
+
+udp:
+  routers:
+    dns-router:
+      entryPoints:
+        - dns
+      service: dns-service
+
+  services:
+    dns-service:
+      loadBalancer:
+        servers:
+          - address: "pangolin:53"

config/ha-reference/node1/config/privateConfig.yml

@@ -0,0 +1,18 @@
+app:
+  region: "region1"
+  identity_provider_mode: "org"
+redis:
+  host: "<REDIS_INTERNAL_HOST>"
+  port: 6379
+flags:
+  enable_redis: true
+  use_pangolin_dns: true
+acme:
+  cert_mode: "pangolin"
+  contact_email: "<CONTACT_EMAIL>"
+  enable_acme_client: true
+dns:
+  enabled: true
+  nameserver_name: "ns.example.com"
+  cname_extension: "cname.example.com"
+  site_extension: "site.example.com" # Optional

config/ha-reference/node1/config/traefik/traefik_config.yml

@@ -0,0 +1,45 @@
+providers:
+  file:
+    directory: "/var/dynamic"
+    watch: true
+
+experimental:
+  plugins:
+    badger:
+      moduleName: "github.com/fosrl/badger"
+      version: "v1.7.0"
+
+log:
+  level: "INFO"
+  format: "common"
+  maxSize: 100
+  maxBackups: 3
+  maxAge: 3
+  compress: true
+
+entryPoints:
+  web:
+    address: ":80"
+  websecure:
+    address: ":443"
+    proxyProtocol: # We trust gerbil upstream
+      trustedIPs:
+        - 0.0.0.0/0
+        - ::1/128
+    transport:
+      respondingTimeouts:
+        readTimeout: "30m"
+    http:
+      encodedCharacters:
+        allowEncodedSlash: true
+        allowEncodedQuestionMark: true
+  dashboard:
+    address: ":3000"
+  dns:
+    address: ":53/udp"
+
+serversTransport:
+  insecureSkipVerify: true
+
+ping:
+  entryPoint: "web"

config/ha-reference/node1/docker-compose.yml

@@ -0,0 +1,62 @@
+name: pangolin
+services:
+  pangolin:
+    image: docker.io/fosrl/pangolin:ee-latest
+    container_name: pangolin
+    restart: unless-stopped
+    volumes:
+      - ./config:/app/config
+      - ./config/certificates:/var/certificates
+      - ./config/dynamic:/var/dynamic
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3001/api/v1/"]
+      interval: "10s"
+      timeout: "10s"
+      retries: 15
+
+  gerbil:
+    image: docker.io/fosrl/gerbil:latest
+    container_name: gerbil
+    restart: unless-stopped
+    depends_on:
+      pangolin:
+        condition: service_healthy
+    command:
+      - --reachableAt=http://<NODE1_INTERNAL_IP>:3004
+      - --generateAndSaveKeyTo=/var/config/key
+      - --remoteConfig=http://pangolin:3001/api/v1/
+      - --trusted-upstreams=<NODE1_EXTERNAL_IP>,<NODE2_EXTERNAL_IP>
+    volumes:
+      - ./config/:/var/config
+    cap_add:
+      - NET_ADMIN
+      - SYS_MODULE
+    ports:
+      - 51820:51820/udp # wireguard
+      - 21820:21820/udp # relay
+      - 53:53/udp # DNS
+      - 443:8443 # resources
+      - 80:80 # web
+      - 3004:3004 # gerbil api
+      - 3000:3000 # Pangolin UI
+
+  traefik:
+    image: docker.io/traefik:v3.7.11
+    container_name: traefik
+    restart: unless-stopped
+    network_mode: service:gerbil # Ports appear on the gerbil service
+    depends_on:
+      pangolin:
+        condition: service_healthy
+    command:
+      - --configFile=/etc/traefik/traefik_config.yml
+    volumes:
+      - ./config/traefik:/etc/traefik:ro
+      - ./config/traefik/logs:/var/log/traefik
+      - ./config/certificates:/var/certificates:ro
+      - ./config/dynamic:/var/dynamic:ro
+
+networks:
+  default:
+    driver: bridge
+    name: pangolin

config/ha-reference/node2/config/config.yml

@@ -0,0 +1,38 @@
+# To see all available options, please visit the docs:
+# https://docs.pangolin.net/
+
+gerbil:
+    start_port: 51820
+    base_endpoint: "<THIS_NODE_EXTERNAL_IP>"
+    exit_node_name: "node2"
+
+app:
+    dashboard_url: "https://pangolin.example.com"
+    log_level: "info"
+
+postgres:
+    connection_string: postgresql://<POSTGRES_USERNAME>:<POSTGRES_PASSWORD>@<POSTGRES_INTERNAL_HOST>:5432/postgres
+
+traefik:
+    site_types: ["newt"] # Wireguard and local sites are not support in clustering
+    file_mode: true # Pangolin will generate and save yaml files in a shared volume
+
+server:
+    secret: "<SECRET>"
+    cors:
+        origins: ["https://pangolin.example.com"]
+        methods: ["GET", "POST", "PUT", "DELETE", "PATCH"]
+        allowed_headers: ["X-CSRF-Token", "Content-Type"]
+        credentials: false
+    maxmind_db_path: "./config/GeoLite2-Country.mmdb" # Make sure to download and place into the config dir
+    maxmind_asn_path: "./config/GeoLite2-ASN.mmdb"
+
+flags:
+    require_email_verification: false
+    disable_signup_without_invite: true
+    disable_user_create_org: false
+    allow_raw_resources: false
+    enable_acme_cert_sync: false
+    disable_local_sites: true 
+    disable_basic_wireguard_sites: true
+    disable_config_managed_domains: true

config/ha-reference/node2/config/dynamic/dynamic_config.yml

@@ -0,0 +1,67 @@
+http:
+  middlewares:
+    badger:
+      plugin:
+        badger:
+          disableForwardAuth: true
+
+  routers:
+    # Next.js router (handles everything except API and WebSocket paths)
+    next-router:
+      rule: "!PathPrefix(`/api/v1`)"
+      service: next-service
+      entryPoints:
+        - dashboard
+      middlewares:
+        - badger
+
+    # API router (handles /api/v1 paths)
+    api-router:
+      rule: "PathPrefix(`/api/v1`)"
+      service: api-service
+      entryPoints:
+        - dashboard
+      middlewares:
+        - badger
+
+    # WebSocket router
+    ws-router:
+      rule: "PathPrefix(`/`)"
+      service: api-service
+      entryPoints:
+        - dashboard
+      middlewares:
+        - badger
+
+  services:
+    next-service:
+      loadBalancer:
+        servers:
+          - url: "http://pangolin:3002"  # Next.js server
+
+    api-service:
+      loadBalancer:
+        servers:
+          - url: "http://pangolin:3000"  # API/WebSocket server
+
+tcp:
+  serversTransports:
+    pp-transport-v1:
+      proxyProtocol:
+        version: 1
+    pp-transport-v2:
+      proxyProtocol:
+        version: 2
+
+udp:
+  routers:
+    dns-router:
+      entryPoints:
+        - dns
+      service: dns-service
+
+  services:
+    dns-service:
+      loadBalancer:
+        servers:
+          - address: "pangolin:53"

config/ha-reference/node2/config/privateConfig.yml

@@ -0,0 +1,16 @@
+app:
+  region: "region1"
+  identity_provider_mode: "org"
+redis:
+  host: "<REDIS_INTERNAL_HOST>"
+  port: 6379
+flags:
+  enable_redis: true
+  use_pangolin_dns: true
+acme:
+  cert_mode: "pangolin"
+dns:
+  enabled: true
+  nameserver_name: "ns.example.com"
+  cname_extension: "cname.example.com"
+  site_extension: "site.example.com" # Optional

config/ha-reference/node2/config/traefik/traefik_config.yml

@@ -0,0 +1,45 @@
+providers:
+  file:
+    directory: "/var/dynamic"
+    watch: true
+
+experimental:
+  plugins:
+    badger:
+      moduleName: "github.com/fosrl/badger"
+      version: "v1.7.0"
+
+log:
+  level: "INFO"
+  format: "common"
+  maxSize: 100
+  maxBackups: 3
+  maxAge: 3
+  compress: true
+
+entryPoints:
+  web:
+    address: ":80"
+  websecure:
+    address: ":443"
+    proxyProtocol: # We trust gerbil upstream
+      trustedIPs:
+        - 0.0.0.0/0
+        - ::1/128
+    transport:
+      respondingTimeouts:
+        readTimeout: "30m"
+    http:
+      encodedCharacters:
+        allowEncodedSlash: true
+        allowEncodedQuestionMark: true
+  dashboard:
+    address: ":3000"
+  dns:
+    address: ":53/udp"
+
+serversTransport:
+  insecureSkipVerify: true
+
+ping:
+  entryPoint: "web"

config/ha-reference/node2/docker-compose.yml

@@ -0,0 +1,62 @@
+name: pangolin
+services:
+  pangolin:
+    image: docker.io/fosrl/pangolin:ee-latest
+    container_name: pangolin
+    restart: unless-stopped
+    volumes:
+      - ./config:/app/config
+      - ./config/certificates:/var/certificates
+      - ./config/dynamic:/var/dynamic
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3001/api/v1/"]
+      interval: "10s"
+      timeout: "10s"
+      retries: 15
+
+  gerbil:
+    image: docker.io/fosrl/gerbil:latest
+    container_name: gerbil
+    restart: unless-stopped
+    depends_on:
+      pangolin:
+        condition: service_healthy
+    command:
+      - --reachableAt=http://<NODE1_INTERNAL_IP>:3004
+      - --generateAndSaveKeyTo=/var/config/key
+      - --remoteConfig=http://pangolin:3001/api/v1/
+      - --trusted-upstreams=<NODE1_EXTERNAL_IP>,<NODE2_EXTERNAL_IP>
+    volumes:
+      - ./config/:/var/config
+    cap_add:
+      - NET_ADMIN
+      - SYS_MODULE
+    ports:
+      - 51820:51820/udp # wireguard
+      - 21820:21820/udp # relay
+      - 53:53/udp # DNS
+      - 443:8443 # resources
+      - 80:80 # web
+      - 3004:3004 # gerbil api
+      - 3000:3000 # Pangolin UI 
+
+  traefik:
+    image: docker.io/traefik:v3.7.11
+    container_name: traefik
+    restart: unless-stopped
+    network_mode: service:gerbil # Ports appear on the gerbil service
+    depends_on:
+      pangolin:
+        condition: service_healthy
+    command:
+      - --configFile=/etc/traefik/traefik_config.yml
+    volumes:
+      - ./config/traefik:/etc/traefik:ro
+      - ./config/traefik/logs:/var/log/traefik
+      - ./config/certificates:/var/certificates:ro
+      - ./config/dynamic:/var/dynamic:ro
+
+networks:
+  default:
+    driver: bridge
+    name: pangolin

dynamic/cert_config.yml

@@ -0,0 +1,2 @@
+tls:
+  certificates: []

dynamic/router_config.yml

@@ -0,0 +1 @@
+{}

messages/en-US.json

@@ -80,7 +80,7 @@
     "siteManageSites": "Manage Sites",
     "siteDescription": "Create and manage sites to enable connectivity to private networks",
     "sitesBannerTitle": "Connect Any Network",
-    "sitesBannerDescription": "A site is a connection to a remote network that allows Pangolin to provide access to resources, whether public or private, to users anywhere. Install the site network connector (Newt) anywhere you can run a binary or container to establish the connection.",
+    "sitesBannerDescription": "A site is a connection to a remote network that allows Pangolin to provide access to resources, whether public or private, to users anywhere. Install the site network connector anywhere you can run a binary or container to establish the connection.",
     "sitesBannerButtonText": "Install Site Connector",
     "approvalsBannerTitle": "Approve or Deny Device Access",
     "approvalsBannerDescription": "Review and approve or deny device access requests from users. When device approvals are required, users must get admin approval before their devices can connect to your organization's resources.",
@@ -152,7 +152,8 @@
     "siteResourcesHowToAccess": "How to access",
     "siteResourcesTargetsOnSite": "Targets on this site",
     "siteSetting": "{siteName} Settings",
-    "siteNewtTunnel": "Newt Site (Recommended)",
+    "siteNewtTunnel": "Pangolin Site (Recommended)",
+    "pangolinSite": "Pangolin Site",
     "siteNewtTunnelDescription": "Easiest way to create an entrypoint into any network. No extra setup.",
     "siteWg": "Basic WireGuard",
     "siteWgDescription": "Use any WireGuard client to establish a tunnel. Manual NAT setup required.",
@@ -226,9 +227,9 @@
     "never": "Never",
     "shareErrorSelectResource": "Please select a resource",
     "proxyResourceTitle": "Manage Public Resources",
-    "proxyResourceDescription": "Create and manage resources that are publicly accessible through a web browser",
+    "proxyResourceDescription": "Create and manage resources that are publicly accessible via a proxy",
     "publicResourcesBannerTitle": "Web-based Public Access",
-    "publicResourcesBannerDescription": "Public resources are proxies accessible to anyone on the internet through a web browser and include identity and context-aware access policies. Unlike private resources, they do not require client-side software.",
+    "publicResourcesBannerDescription": "Public resources are proxies accessible to anyone on the internet, like a website or API, and include identity and context-aware access policies. Unlike private resources, they do not require any client-side software to access.",
     "clientResourceTitle": "Manage Private Resources",
     "clientResourceDescription": "Create and manage resources that are only accessible through a connected client",
     "privateResourcesBannerTitle": "Zero-Trust Private Access",
@@ -465,6 +466,8 @@
     "apiKeysDelete": "Delete API Key",
     "apiKeysManage": "Manage API Keys",
     "apiKeysDescription": "API keys are used to authenticate with the integration API",
+    "orgsManage": "Manage Organizations",
+    "orgsDescription": "View and manage all organizations on this instance",
     "provisioningKeysTitle": "Provisioning Key",
     "provisioningKeysManage": "Manage Provisioning Keys",
     "provisioningKeysDescription": "Provisioning keys are used to authenticate automated site provisioning for your organization.",
@@ -520,12 +523,12 @@
     "pendingSitesBannerDescription": "Sites that connect using a provisioning key appear here for review.",
     "pendingSitesBannerButtonText": "Learn More",
     "apiKeysSettings": "{apiKeyName} Settings",
-    "userTitle": "Manage All Users",
+    "userTitle": "Manage Users",
-    "userDescription": "View and manage all users in the system",
+    "userDescription": "View and manage all users in this instance",
     "userAbount": "About User Management",
     "userAbountDescription": "This table displays all base user objects in the system. Each user may belong to multiple organizations. Removing a user from an organization does not delete their base user object. They will remain in the system. To completely remove a user from the system, you must delete their base user object using the delete action in this table.",
     "userServer": "Server Users",
-    "userSearch": "Search server users...",
+    "userSearch": "Search users...",
     "userErrorDelete": "Error deleting user",
     "userDeleteConfirm": "Confirm Delete User",
     "userDeleteServer": "Delete User from Server",
@@ -594,7 +597,7 @@
     "licensePricingPage": "For the most up-to-date pricing and discounts, please visit the ",
     "invite": "Invitations",
     "inviteRegenerate": "Regenerate Invitation",
-    "inviteRegenerateDescription": "Revoke previous invitation and create a new one",
+    "inviteRegenerateDescription": "Create a new invite link for this user. The previous invitation will be revoked.",
     "inviteRemove": "Remove Invitation",
     "inviteRemoveError": "Failed to remove invitation",
     "inviteRemoveErrorDescription": "An error occurred while removing the invitation.",
@@ -674,11 +677,11 @@
     "accessUserCreateDescription": "Follow the steps below to create a new user",
     "userSeeAll": "See All Users",
     "userTypeTitle": "User Type",
-    "userTypeDescription": "Determine how you want to create the user",
+    "userTypeDescription": "Select the identity provider to use for this user",
     "userSettings": "User Information",
-    "userSettingsDescription": "Enter the details for the new user",
+    "userSettingsDescription": "Enter the general details for the new user",
     "inviteEmailSent": "Send invite email to user",
-    "inviteValid": "Invite Valid For (days)",
+    "inviteValid": "Invite Valid For",
     "selectDuration": "Select duration",
     "selectResource": "Select Resource",
     "filterByResource": "Filter By Resource",
@@ -716,6 +719,7 @@
     "nameOptional": "Name (Optional)",
     "accessControls": "Access Controls",
     "userDescription2": "Manage the settings on this user",
+    "userGeneralSettingsDescription": "Manage this user's roles and settings in the organization",
     "accessRoleErrorAdd": "Failed to add user to role",
     "accessRoleErrorAddDescription": "An error occurred while adding user to the role.",
     "userSaved": "User saved",
@@ -1214,7 +1218,7 @@
     "orgPoliciesEdit": "Edit Organization Policy",
     "org": "Organization",
     "orgSelect": "Select organization",
-    "orgSearch": "Search org",
+    "orgSearch": "Search organizations...",
     "orgNotFound": "No org found.",
     "roleMappingPathOptional": "Role Mapping Path (Optional)",
     "orgMappingPathOptional": "Organization Mapping Path (Optional)",
@@ -1353,7 +1357,7 @@
     "siteLabelsDescription": "Manage labels associated with this site.",
     "labelsNotFound": "No labels found.",
     "labelsEmptyCreateHint": "Start typing above to create a label.",
-    "labelSearch": "Search labels",
+    "labelSearch": "Search labels...",
     "labelSearchOrCreate": "Search or create a label",
     "accessLabelFilterCount": "{count, plural, one {# label} other {# labels}}",
     "labelOverflowCount": "+{count, plural, one {# label} other {# labels}}",
@@ -1424,6 +1428,24 @@
     "logoutError": "Error logging out",
     "signingAs": "Signed in as",
     "serverAdmin": "Server Admin",
+    "promoteServerAdmin": "Promote to Server admin",
+    "promoteServerAdminTitle": "Promote to Server Admin",
+    "promoteServerAdminQuestion": "Are you sure you want to promote {selectedUser} to server admin?",
+    "promoteServerAdminMessage": "Server admins have the highest privileges and can manage the server.",
+    "promoteServerAdminWarning": "This can be undone at any time by demoting the user.",
+    "promoteServerAdminConfirm": "Promote to Server Admin",
+    "promoteServerAdminSuccess": "User Promoted",
+    "promoteServerAdminSuccessDescription": "{selectedUser} is now a server admin.",
+    "promoteServerAdminError": "Failed to promote user",
+    "demoteServerAdmin": "Demote from Server admin",
+    "demoteServerAdminTitle": "Demote from Server Admin",
+    "demoteServerAdminQuestion": "Are you sure you want to demote {selectedUser} from server admin?",
+    "demoteServerAdminMessage": "{selectedUser} will lose all server admin privileges.",
+    "demoteServerAdminWarning": "This can be undone at any time by promoting the user.",
+    "demoteServerAdminConfirm": "Demote from server admin",
+    "demoteServerAdminSuccess": "User demoted",
+    "demoteServerAdminSuccessDescription": "{selectedUser} is no longer a server admin.",
+    "demoteServerAdminError": "Failed to demote user",
     "managedSelfhosted": "Managed Self-Hosted",
     "otpEnable": "Enable Two-factor",
     "otpDisable": "Disable Two-factor",
@@ -2095,9 +2117,10 @@
     "resourceBudgetSettings": "Budget",
     "resourceBudgetSettingsDescription": "Configure how this AI gateway restricts usage based on spending or token limits",
     "sidebarApiKeys": "API Keys",
+    "sidebarOrgs": "Organizations",
     "sidebarProvisioning": "Provisioning",
     "sidebarSettings": "Settings",
-    "sidebarAllUsers": "All Users",
+    "sidebarAllUsers": "Users",
     "sidebarIdentityProviders": "Identity Providers",
     "sidebarLicense": "License",
     "sidebarClients": "Clients",
@@ -2898,7 +2921,7 @@
     "editInternalResourceDialogAlias": "Alias",
     "editInternalResourceDialogAliasDescription": "An optional internal DNS alias for this resource.",
     "createInternalResourceDialogNoSitesAvailable": "No Sites Available",
-    "createInternalResourceDialogNoSitesAvailableDescription": "You need to have at least one Newt site with a subnet configured to create private resources.",
+    "createInternalResourceDialogNoSitesAvailableDescription": "You need to have at least one site with a subnet configured to create private resources.",
     "createInternalResourceDialogClose": "Close",
     "createInternalResourceDialogCreateClientResource": "Create Private Resource",
     "createInternalResourceDialogCreateClientResourceDescription": "Create a new resource that will only be accessible to clients connected to the organization",
@@ -3477,7 +3500,8 @@
     },
     "priority": "Priority",
     "priorityDescription": "Higher priority routes are evaluated first. Priority = 100 means automatic ordering (system decides). Use another number to enforce manual priority.",
-    "instanceName": "Instance Name",
+    "instanceName": "Server ID",
+    "clearInstanceName": "Reset Server Association",
     "pathMatchModalTitle": "Configure Path Matching",
     "pathMatchModalDescription": "Set up how incoming requests should be matched based on their path.",
     "pathMatchType": "Match Type",
@@ -3767,6 +3791,7 @@
     "noData": "No Data",
     "machineClients": "Machine Clients",
     "install": "Install",
+    "downloadInstaller": "Download Installer",
     "run": "Run",
     "envFile": "Environment File",
     "serviceFile": "Service File",
@@ -3974,11 +3999,12 @@
     "disconnected": "Disconnected",
     "approvalsEmptyStateTitle": "Device Approvals Not Enabled",
     "approvalsEmptyStateDescription": "Enable device approvals for roles to require admin approval before users can connect new devices.",
+    "approvalsEmptyStateHowToTitle": "How to Enable",
     "approvalsEmptyStateStep1Title": "Go to Roles",
     "approvalsEmptyStateStep1Description": "Navigate to your organization's roles settings to configure device approvals.",
     "approvalsEmptyStateStep2Title": "Enable Device Approvals",
     "approvalsEmptyStateStep2Description": "Edit a role and enable the 'Require Device Approvals' option. Users with this role will need admin approval for new devices.",
-    "approvalsEmptyStatePreviewDescription": "Preview: When enabled, pending device requests will appear here for review",
+    "approvalsEmptyStatePreviewDescription": "When enabled, pending device requests will appear here for review.",
     "approvalsEmptyStateButtonText": "Manage Roles",
     "domainErrorTitle": "We are having trouble verifying your domain",
     "idpAdminAutoProvisionPoliciesTabHint": "Configure role mapping and organization policies on the <policiesTabLink>Auto Provision Settings</policiesTabLink> tab.",
@@ -4253,6 +4279,11 @@
     "resourceLauncherViewAsAdmin": "View as Admin",
     "resourceLauncherResourceDetailsDescription": "Connection information and status for this resource.",
     "resourceLauncherResourceDetails": "Resource Details",
+    "resourceLauncherSitesDescription": "The resource is accessible via the following sites.",
+    "resourceLauncherViewSiteAsAdmin": "View Site as Admin",
+    "resourceLauncherFilterBySite": "Filter by Site",
+    "resourceLauncherSshCommand": "SSH Command",
+    "resourceLauncherSshCommandDescription": "Use the Pangolin CLI to open an SSH session to this resource.",
     "resourceLauncherAuthMethodsDescription": "Authentication methods enabled for this resource.",
     "resourceLauncherPrivateClientRequired": "Connect with a client on your device to access this resource privately.",
     "resourceLauncherPrivateClientRequiredTitle": "Client Connection Required",
@@ -4363,5 +4394,6 @@
     "rdpUnicodeKeyboardMode": "Unicode keyboard mode",
     "sessionToolbarShow": "Show toolbar",
     "sessionToolbarHide": "Hide toolbar",
-    "actionUpdateSiteApprovals": "Update Site Approvals"
+    "actionUpdateSiteApprovals": "Update Site Approvals",
+    "check": "Check"
 }

next.config.ts

@@ -34,6 +34,11 @@ const nextConfig: NextConfig = {
                 source: "/:orgId/settings/resources/client/:path*",
                 destination: "/:orgId/settings/resources/private/:path*",
                 permanent: true
+            },
+            {
+                source: "/:orgId/settings/access/users/:userId/access-controls",
+                destination: "/:orgId/settings/access/users/:userId/general",
+                permanent: false
             }
         ];
     }

package-lock.json

@@ -50,6 +50,7 @@
                 "@xterm/addon-fit": "^0.11.0",
                 "@xterm/addon-web-links": "^0.12.0",
                 "@xterm/xterm": "^6.0.0",
+                "acme-client": "^5.4.0",
                 "arctic": "3.7.0",
                 "axios": "1.20.0",
                 "better-sqlite3": "11.9.1",
@@ -61,6 +62,7 @@
                 "cors": "2.8.6",
                 "crypto-js": "4.2.0",
                 "d3": "7.9.0",
+                "dns-packet": "^5.6.1",
                 "drizzle-orm": "0.45.2",
                 "express": "5.2.1",
                 "express-rate-limit": "8.7.0",
@@ -124,6 +126,7 @@
                 "@types/cors": "2.8.19",
                 "@types/crypto-js": "4.2.2",
                 "@types/d3": "7.4.3",
+                "@types/dns-packet": "^5.6.5",
                 "@types/express": "5.0.6",
                 "@types/express-session": "1.19.0",
                 "@types/jmespath": "0.15.2",
@@ -2445,6 +2448,12 @@
                 "@jridgewell/sourcemap-codec": "^1.4.14"
             }
         },
+        "node_modules/@leichtgewicht/ip-codec": {
+            "version": "2.0.5",
+            "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz",
+            "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==",
+            "license": "MIT"
+        },
         "node_modules/@levischuck/tiny-cbor": {
             "version": "0.2.11",
             "resolved": "https://registry.npmjs.org/@levischuck/tiny-cbor/-/tiny-cbor-0.2.11.tgz",
@@ -6764,6 +6773,16 @@
                 "@types/d3-selection": "*"
             }
         },
+        "node_modules/@types/dns-packet": {
+            "version": "5.6.5",
+            "resolved": "https://registry.npmjs.org/@types/dns-packet/-/dns-packet-5.6.5.tgz",
+            "integrity": "sha512-qXOC7XLOEe43ehtWJCMnQXvgcIpv6rPmQ1jXT98Ad8A3TB1Ue50jsCbSSSyuazScEuZ/Q026vHbrOTVkmwA+7Q==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@types/node": "*"
+            }
+        },
         "node_modules/@types/esrecurse": {
             "version": "4.3.1",
             "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz",
@@ -7435,6 +7454,22 @@
                 "node": ">= 0.6"
             }
         },
+        "node_modules/acme-client": {
+            "version": "5.4.0",
+            "resolved": "https://registry.npmjs.org/acme-client/-/acme-client-5.4.0.tgz",
+            "integrity": "sha512-mORqg60S8iML6XSmVjqjGHJkINrCGLMj2QvDmFzI9vIlv1RGlyjmw3nrzaINJjkNsYXC41XhhD5pfy7CtuGcbA==",
+            "license": "MIT",
+            "dependencies": {
+                "@peculiar/x509": "^1.11.0",
+                "asn1js": "^3.0.5",
+                "axios": "^1.7.2",
+                "debug": "^4.3.5",
+                "node-forge": "^1.3.1"
+            },
+            "engines": {
+                "node": ">= 16"
+            }
+        },
         "node_modules/acorn": {
             "version": "8.16.0",
             "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz",
@@ -9261,6 +9296,18 @@
                 "node": ">=8"
             }
         },
+        "node_modules/dns-packet": {
+            "version": "5.6.1",
+            "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz",
+            "integrity": "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==",
+            "license": "MIT",
+            "dependencies": {
+                "@leichtgewicht/ip-codec": "^2.0.1"
+            },
+            "engines": {
+                "node": ">=6"
+            }
+        },
         "node_modules/doctrine": {
             "version": "2.1.0",
             "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
@@ -13140,6 +13187,15 @@
                 "semver": "bin/semver.js"
             }
         },
+        "node_modules/node-forge": {
+            "version": "1.4.0",
+            "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz",
+            "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==",
+            "license": "(BSD-3-Clause OR GPL-2.0)",
+            "engines": {
+                "node": ">= 6.13.0"
+            }
+        },
         "node_modules/node-releases": {
             "version": "2.0.54",
             "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz",

package.json

@@ -73,6 +73,7 @@
         "@xterm/addon-fit": "^0.11.0",
         "@xterm/addon-web-links": "^0.12.0",
         "@xterm/xterm": "^6.0.0",
+        "acme-client": "^5.4.0",
         "arctic": "3.7.0",
         "axios": "1.20.0",
         "better-sqlite3": "11.9.1",
@@ -84,6 +85,7 @@
         "cors": "2.8.6",
         "crypto-js": "4.2.0",
         "d3": "7.9.0",
+        "dns-packet": "^5.6.1",
         "drizzle-orm": "0.45.2",
         "express": "5.2.1",
         "express-rate-limit": "8.7.0",
@@ -96,6 +98,7 @@
         "jmespath": "0.16.0",
         "js-yaml": "5.4.1",
         "jsonwebtoken": "9.0.3",
+        "lru-cache": "11.5.2",
         "lucide-react": "1.38.0",
         "maxmind": "5.0.7",
         "moment": "2.30.1",
@@ -103,7 +106,6 @@
         "next-intl": "4.14.1",
         "next-themes": "0.4.6",
         "nextjs-toploader": "3.9.17",
-        "node-cache": "5.1.2",
         "nodemailer": "9.1.0",
         "oslo": "1.2.1",
         "pg": "8.23.0",
@@ -147,6 +149,7 @@
         "@types/cors": "2.8.19",
         "@types/crypto-js": "4.2.2",
         "@types/d3": "7.4.3",
+        "@types/dns-packet": "^5.6.5",
         "@types/express": "5.0.6",
         "@types/express-session": "1.19.0",
         "@types/jmespath": "0.15.2",

server/certificates.ts

@@ -0,0 +1,8 @@
+export async function startCertificateManager() {
+    // No-op: ACME certificate generation/management is only available in
+    // builds that include the private/enterprise feature set.
+}
+
+export async function stopCertificateManager() {
+    // No-op counterpart to startCertificateManager.
+}

server/db/pg/schema/schema.ts

@@ -682,6 +682,8 @@ export const newts = pgTable(
         secretHash: varchar("secretHash").notNull(),
         dateCreated: varchar("dateCreated").notNull(),
         version: varchar("version"),
+        agent: varchar("agent"), // either newt or cli
+        agentVersion: varchar("agentVersion"),
         siteId: integer("siteId").references(() => sites.siteId, {
             onDelete: "cascade"
         })

server/db/sqlite/schema/schema.ts

@@ -703,6 +703,8 @@ export const newts = sqliteTable(
         secretHash: text("secretHash").notNull(),
         dateCreated: text("dateCreated").notNull(),
         version: text("version"),
+        agent: text("agent"), // either newt or cli
+        agentVersion: text("agentVersion"),
         siteId: integer("siteId").references(() => sites.siteId, {
             onDelete: "cascade"
         })

server/dns.ts

@@ -0,0 +1,8 @@
+export async function startDnsServer() {
+    // No-op: the authoritative DNS server is only available in builds
+    // that include the private/enterprise feature set.
+}
+
+export async function stopDnsServer() {
+    // No-op counterpart to startDnsServer.
+}

server/index.ts

@@ -25,6 +25,8 @@ import { setHostMeta } from "@server/lib/hostMeta";
 import { TraefikConfigManager } from "@server/lib/traefik/TraefikConfigManager";
 import { initCleanup } from "#dynamic/cleanup";
 import { startSchedulers } from "#dynamic/startSchedulers";
+import { startDnsServer } from "#dynamic/dns";
+import { startCertificateManager } from "#dynamic/certificates";
 import license from "#dynamic/license/license";
 import { fetchServerIp } from "@server/lib/serverIpService";
 import { initAiModelCatalog } from "@server/lib/aiModelCatalog";
@@ -45,6 +47,10 @@ async function startServers() {
 
     startSchedulers();
 
+    await startDnsServer();
+
+    await startCertificateManager();
+
     // Start all servers
     const apiServer = createApiServer();
     const internalServer = createInternalServer();

server/lib/cache.ts

@@ -1,13 +1,7 @@
-import NodeCache from "node-cache";
 import logger from "@server/logger";
+import { createLocalCache } from "@server/lib/createLocalCache";
 
-// Create local cache with maxKeys limit to prevent memory leaks
+export const localCache = createLocalCache();
-// With ~10k requests/day and 5min TTL, 10k keys should be more than sufficient
-export const localCache = new NodeCache({
-    stdTTL: 3600,
-    checkperiod: 120,
-    maxKeys: 10000
-});
 
 // Log cache statistics periodically for monitoring
 // setInterval(() => {

server/lib/createLocalCache.ts

@@ -0,0 +1,79 @@
+import { LRUCache } from "lru-cache";
+
+const DEFAULT_MAX_KEYS = 10000;
+const DEFAULT_TTL_MS = 3600 * 1000;
+
+export type LocalCache = {
+    get<T>(key: string): T | undefined;
+    set(key: string, value: unknown, ttlSeconds?: number): boolean;
+    del(key: string | string[]): number;
+    has(key: string): boolean;
+    keys(): string[];
+    flushAll(): void;
+    getStats(): { keys: number };
+    getTtl(key: string): number | undefined;
+};
+
+export function createLocalCache(max = DEFAULT_MAX_KEYS): LocalCache {
+    const lru = new LRUCache<string, {}>({
+        max,
+        ttl: DEFAULT_TTL_MS,
+        updateAgeOnGet: false
+    });
+
+    return {
+        get<T>(key: string): T | undefined {
+            return lru.get(key) as T | undefined;
+        },
+
+        set(key: string, value: unknown, ttlSeconds?: number): boolean {
+            const stored = value as {};
+            if (ttlSeconds === undefined) {
+                lru.set(key, stored);
+            } else if (ttlSeconds === 0) {
+                lru.set(key, stored, { ttl: 0 });
+            } else {
+                lru.set(key, stored, { ttl: ttlSeconds * 1000 });
+            }
+            return true;
+        },
+
+        del(key: string | string[]): number {
+            const keys = Array.isArray(key) ? key : [key];
+            let deleted = 0;
+            for (const k of keys) {
+                if (lru.delete(k)) {
+                    deleted++;
+                }
+            }
+            return deleted;
+        },
+
+        has(key: string): boolean {
+            return lru.has(key);
+        },
+
+        keys(): string[] {
+            return [...lru.keys()];
+        },
+
+        flushAll(): void {
+            lru.clear();
+        },
+
+        getStats(): { keys: number } {
+            return { keys: lru.size };
+        },
+
+        getTtl(key: string): number | undefined {
+            if (!lru.has(key)) {
+                return undefined;
+            }
+            const remaining = lru.getRemainingTTL(key);
+            if (!Number.isFinite(remaining)) {
+                return 0;
+            }
+            return Date.now() + remaining;
+        }
+    };
+}

server/lib/dbRetry.ts

@@ -71,7 +71,7 @@ export async function withRetry<T>(
                 const jitter = Math.random() * baseDelay;
                 const delay = baseDelay + jitter;
                 logger.warn(
-                    `Transient DB error in ${context}, retrying attempt ${attempt}/${maxRetries} after ${delay.toFixed(0)}ms`,
+                    `Transient DB issue in ${context}, retrying attempt ${attempt}/${maxRetries} after ${delay.toFixed(0)}ms`,
                     { code: error?.code ?? error?.cause?.code }
                 );
                 await new Promise((resolve) => setTimeout(resolve, delay));

server/lib/dns/generateDomains.ts

@@ -0,0 +1,3 @@
+export function createCname(domainId: string, baseDomain: string) {}
+
+export function createNs() {}

server/lib/exitNodes/exitNodeCheckIn.ts

@@ -0,0 +1,14 @@
+// Tracks, per process lifetime, whether a given exit node has ever checked in
+// (called /gerbil/get-config) since this Pangolin instance started. This lets
+// callers distinguish "gerbil hasn't come up yet" (expected briefly after a
+// restart, since gerbil depends on pangolin's container starting first) from
+// "gerbil was reachable and now isn't" (a real problem worth an error log).
+const checkedInExitNodeIds = new Set<number>();
+
+export function markExitNodeCheckedIn(exitNodeId: number): void {
+    checkedInExitNodeIds.add(exitNodeId);
+}
+
+export function hasExitNodeCheckedIn(exitNodeId: number): boolean {
+    return checkedInExitNodeIds.has(exitNodeId);
+}

server/lib/exitNodes/exitNodeComms.ts

@@ -1,6 +1,7 @@
 import axios from "axios";
 import logger from "@server/logger";
 import { ExitNode } from "@server/db";
+import { hasExitNodeCheckedIn } from "./exitNodeCheckIn";
 
 interface ExitNodeRequest {
     remoteType?: string;
@@ -72,13 +73,19 @@ export async function sendToExitNode(
 
         return response.data;
     } catch (error) {
-        if (axios.isAxiosError(error)) {
+        const message = axios.isAxiosError(error)
-            logger.error(
+            ? `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${exitNode.reachableAt} (status: ${error.response?.status}): ${error.message}`
-                `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${exitNode.reachableAt} (status: ${error.response?.status}): ${error.message}`
+            : `Error making ${method} request for exit node at ${exitNode.reachableAt}: ${error}`;
-            );
+
+        // The exit node (gerbil) may still be starting up and not yet
+        // reachable. Until it has checked in at least once, log this at a
+        // lower level since it's expected; once it has checked in, a
+        // connection failure is a real problem.
+        if (hasExitNodeCheckedIn(exitNode.exitNodeId)) {
+            logger.error(message);
         } else {
-            logger.error(
+            logger.warn(
-                `Error making ${method} request for exit node at ${exitNode.reachableAt}: ${error}`
+                `${message} (exit node has not checked in yet since startup, this is expected briefly)`
             );
         }
         throw error;

server/lib/exitNodes/index.ts

@@ -1,5 +1,6 @@
 export * from "./exitNodes";
 export * from "./exitNodeComms";
+export * from "./exitNodeCheckIn";
 export * from "./subnet";
 export * from "./getCurrentExitNodeId";
 export * from "./calculateExitNodeWeight";

server/lib/readConfigFile.ts

@@ -348,8 +348,8 @@ export const configSchema = z
                     .optional()
                     .pipe(z.string())
                     .transform((url) => url.toLowerCase()),
-                subnet_group: z.string().optional().default("100.89.137.0/20"),
+                subnet_group: z.string().optional().default("100.89.137.0/18"),
-                block_size: z.number().positive().gt(0).optional().default(24),
+                block_size: z.number().positive().gt(0).optional().default(22),
                 site_block_size: z
                     .number()
                     .positive()
@@ -493,23 +493,6 @@ export const configSchema = z
                     .prefault({})
             })
             .optional()
-            .prefault({}),
-        dns: z
-            .object({
-                nameservers: z
-                    .array(z.string().optional().optional())
-                    .optional()
-                    .default([
-                        "ns1.pangolin.net",
-                        "ns2.pangolin.net",
-                        "ns3.pangolin.net"
-                    ]),
-                cname_extension: z
-                    .string()
-                    .optional()
-                    .default("cname.pangolin.net")
-            })
-            .optional()
             .prefault({})
     })
     .refine(

server/lib/retry.ts

@@ -0,0 +1,72 @@
+import logger from "@server/logger";
+
+export async function withRetry<T>(
+    fn: () => Promise<T>,
+    options: {
+        retries?: number;
+        baseDelayMs?: number;
+        label?: string;
+        // Called with each caught error to decide whether it's worth
+        // retrying. Defaults to retrying everything (existing behavior) -
+        // pass this to exclude errors that are known to be permanent (e.g.
+        // an upstream rate limit or validation rejection) rather than
+        // transient, so they fail fast instead of wasting retry attempts.
+        shouldRetry?: (error: unknown) => boolean;
+    } = {}
+): Promise<T> {
+    const {
+        retries = 3,
+        baseDelayMs = 250,
+        label = "operation",
+        shouldRetry = () => true
+    } = options;
+
+    let attempt = 0;
+    while (true) {
+        try {
+            return await fn();
+        } catch (error) {
+            attempt++;
+            if (attempt > retries || !shouldRetry(error)) {
+                throw error;
+            }
+
+            // Exponential backoff with jitter so retries don't all land at once.
+            const delay =
+                baseDelayMs * 2 ** (attempt - 1) * (0.5 + Math.random());
+
+            logger.warn(
+                `${label} failed (attempt ${attempt}/${retries + 1}), retrying in ${delay.toFixed(0)}ms`,
+                error
+            );
+
+            await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+    }
+}
+
+// Bounds an operation that has no timeout of its own (e.g. acme-client's
+// axios instance never sets one, so a stalled TCP connection to the ACME
+// server hangs forever instead of erroring). Without this, a single hung
+// call can leave its caller's promise permanently unsettled - fatal for
+// code that gates future work on that promise resolving, like the
+// scheduler's runExclusive() waiting on a batch's Promise.all.
+export async function withTimeout<T>(
+    promise: Promise<T>,
+    ms: number,
+    label = "operation"
+): Promise<T> {
+    let timer: NodeJS.Timeout;
+    const timeout = new Promise<never>((_, reject) => {
+        timer = setTimeout(
+            () => reject(new Error(`${label} timed out after ${ms}ms`)),
+            ms
+        );
+    });
+
+    try {
+        return await Promise.race([promise, timeout]);
+    } finally {
+        clearTimeout(timer!);
+    }
+}

server/lib/traefik/TraefikConfigManager.ts

@@ -6,7 +6,10 @@ import * as yaml from "js-yaml";
 import axios from "axios";
 import { db, exitNodes } from "@server/db";
 import { eq } from "drizzle-orm";
-import { getCurrentExitNodeId } from "@server/lib/exitNodes";
+import {
+    getCurrentExitNodeId,
+    hasExitNodeCheckedIn
+} from "@server/lib/exitNodes";
 import { getTraefikConfig } from "#dynamic/lib/traefik";
 import { getValidCertificatesForDomains } from "@server/lib/certificates";
 import { sendToExitNode } from "#dynamic/lib/exitNodes";
@@ -341,10 +344,6 @@ export class TraefikConfigManager {
 
             const { domains, traefikConfig } = getTraefikConfig;
 
-            // Add static domains from config
-            // const staticDomains = [config.getRawConfig().app.dashboard_url];
-            // staticDomains.forEach((domain) => domains.add(domain));
-
             // Log if domains changed
             if (
                 this.lastActiveDomains.size !== domains.size ||
@@ -358,7 +357,7 @@ export class TraefikConfigManager {
                 this.lastActiveDomains = new Set(domains);
             }
 
-            if (process.env.USE_PANGOLIN_DNS === "true" && build != "oss") {
+            if (process.env.CERT_MODE === "pangolin" && build != "oss") {
                 // Scan current local certificate state
                 this.lastLocalCertificateState =
                     await this.scanLocalCertificateState();
@@ -439,13 +438,13 @@ export class TraefikConfigManager {
                     // Always ensure all existing certificates (including wildcards) are in the config
                     await this.updateDynamicConfigFromLocalCerts(domains);
                 } else {
-                    const timeSinceLastFetch = this.lastCertificateFetch
+                    // const timeSinceLastFetch = this.lastCertificateFetch
-                        ? Math.round(
+                    //     ? Math.round(
-                              (Date.now() -
+                    //           (Date.now() -
-                                  this.lastCertificateFetch.getTime()) /
+                    //               this.lastCertificateFetch.getTime()) /
-                                  (1000 * 60)
+                    //               (1000 * 60)
-                          )
+                    //       )
-                        : 0;
+                    //     : 0;
 
                     // logger.debug(
                     //     `Skipping certificate fetch - no changes detected and within 24-hour window (last fetch: ${timeSinceLastFetch} minutes ago)`
@@ -466,32 +465,51 @@ export class TraefikConfigManager {
             await this.writeTraefikDynamicConfig(traefikConfig);
 
             // Send domains to SNI proxy
+            let exitNodeForSni: typeof exitNodes.$inferSelect | undefined;
             try {
-                let exitNode;
                 if (config.getRawConfig().gerbil.exit_node_name) {
                     const exitNodeName =
                         config.getRawConfig().gerbil.exit_node_name!;
-                    [exitNode] = await db
+                    [exitNodeForSni] = await db
                         .select()
                         .from(exitNodes)
                         .where(eq(exitNodes.name, exitNodeName))
                         .limit(1);
                 } else {
-                    [exitNode] = await db.select().from(exitNodes).limit(1);
+                    [exitNodeForSni] = await db
+                        .select()
+                        .from(exitNodes)
+                        .limit(1);
                 }
-                if (exitNode) {
+                if (exitNodeForSni) {
-                    await sendToExitNode(exitNode, {
+                    await sendToExitNode(exitNodeForSni, {
                         localPath: "/update-local-snis",
                         method: "POST",
-                        data: { fullDomains: Array.from(domains) }
+                        data: {
+                            fullDomains: [
+                                ...Array.from(domains),
+                                ...config.getRawConfig().traefik.static_domains
+                            ]
+                        }
                     });
                 } else {
-                    logger.error(
+                    logger.warn(
                         "No exit node found. Has gerbil registered yet?"
                     );
                 }
             } catch (err) {
-                logger.error("Failed to post domains to SNI proxy:", err);
+                // sendToExitNode already logs the underlying connection
+                // error at the appropriate level (warn before the exit node
+                // has checked in since startup, error after), so avoid
+                // double-logging it as an error here.
+                if (
+                    exitNodeForSni &&
+                    !hasExitNodeCheckedIn(exitNodeForSni.exitNodeId)
+                ) {
+                    logger.warn("Failed to post domains to SNI proxy:", err);
+                } else {
+                    logger.error("Failed to post domains to SNI proxy:", err);
+                }
             }
 
             // Update active domains tracking

server/lib/traefik/rule.ts

@@ -1,7 +1,5 @@
-/**
+// Build the Host()/HostRegexp() Traefik rule for a resource's domain.
- * Build the Host()/HostRegexp() Traefik rule for a resource's domain.
+// Wildcard resources match any single subdomain via HostRegexp.
- * Wildcard resources match any single subdomain via HostRegexp.
- */
 export function buildHostRule(
     fullDomain: string,
     wildcard?: boolean | null
@@ -14,10 +12,8 @@ export function buildHostRule(
     return `Host(\`${fullDomain}\`)`;
 }
 
-/**
+// Append a path-matching clause to a Traefik rule based on the resource's
- * Append a path-matching clause to a Traefik rule based on the resource's
+// configured path and pathMatchType.
- * configured path and pathMatchType.
- */
 export function appendPathMatch(
     rule: string,
     path: string | null | undefined,
@@ -40,10 +36,8 @@ export function appendPathMatch(
     return rule;
 }
 
-/**
+// Compute the router priority for a resource, favoring an explicit override
- * Compute the router priority for a resource, favoring an explicit override
+// and otherwise deriving it from the path match specificity.
- * and otherwise deriving it from the path match specificity.
- */
 export function computeRoutePriority(
     priority: number | null | undefined,
     path: string | null | undefined,

server/private/certificates.ts

@@ -0,0 +1,17 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+export {
+    startCertificateManager,
+    stopCertificateManager
+} from "./lib/certificates";

server/private/cleanup.ts

@@ -20,6 +20,8 @@ import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth"
 import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator";
 import { shutdownUsageRecorder } from "@server/lib/aiBudgetEnforcement";
 import { shutdownAiSessionLogger } from "@server/routers/aiGateway/logAiSession";
+import { stopDnsServer } from "./dns";
+import { stopCertificateManager } from "./certificates";
 
 async function cleanup() {
     await stopPingAccumulator();
@@ -31,6 +33,8 @@ async function cleanup() {
     await rateLimitService.cleanup();
     await wsCleanup();
     await logStreamingManager.shutdown();
+    await stopDnsServer();
+    await stopCertificateManager();
 
     process.exit(0);
 }

server/private/dns.ts

@@ -0,0 +1,44 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { AuthoritativeDNSServer } from "#private/lib/dns";
+import { privateConfig } from "#private/lib/config";
+
+let dnsServer: AuthoritativeDNSServer | undefined;
+
+export async function startDnsServer() {
+    const dnsConfig = privateConfig.getRawPrivateConfig().dns;
+    if (!dnsConfig || !dnsConfig.enabled) {
+        return;
+    }
+
+    const cacheOptions = {
+        stdTTL: 300, // 5 minutes default TTL
+        checkperiod: 60, // Check for expired keys every 60 seconds
+        useClones: false // Better performance
+    };
+
+    // Create DNS server
+    dnsServer = new AuthoritativeDNSServer(dnsConfig.listen_port, cacheOptions);
+
+    await dnsServer.start();
+}
+
+export async function stopDnsServer() {
+    if (!dnsServer) {
+        return;
+    }
+
+    await dnsServer.stop();
+    dnsServer = undefined;
+}

server/private/lib/cache.ts

@@ -11,17 +11,11 @@
  * This file is not licensed under the AGPLv3.
  */
 
-import NodeCache from "node-cache";
 import logger from "@server/logger";
-import { redisManager, regionalRedisManager } from "@server/private/lib/redis";
+import { createLocalCache } from "@server/lib/createLocalCache";
+import { redisManager, regionalRedisManager } from "#private/lib/redis";
 
-// Create local cache with maxKeys limit to prevent memory leaks
+export const localCache = createLocalCache();
-// With ~10k requests/day and 5min TTL, 10k keys should be more than sufficient
-export const localCache = new NodeCache({
-    stdTTL: 3600,
-    checkperiod: 120,
-    maxKeys: 10000
-});
 
 // Log cache statistics periodically for monitoring
 // setInterval(() => {
@@ -97,11 +91,11 @@ class AdaptiveCache {
                 const value = await redisManager.get(key);
 
                 if (value !== null) {
-                    logger.debug(`Cache hit in Redis: ${key}`);
+                    // logger.debug(`Cache hit in Redis: ${key}`);
                     return JSON.parse(value) as T;
                 }
 
-                logger.debug(`Cache miss in Redis: ${key}`);
+                // logger.debug(`Cache miss in Redis: ${key}`);
                 return undefined;
             } catch (error) {
                 logger.error(`Redis get error for key ${key}:`, error);
@@ -134,7 +128,7 @@ class AdaptiveCache {
                     const success = await redisManager.del(k);
                     if (success) {
                         deletedCount++;
-                        logger.debug(`Deleted key from Redis: ${k}`);
+                        // logger.debug(`Deleted key from Redis: ${k}`);
                     }
                 }
 
@@ -161,7 +155,7 @@ class AdaptiveCache {
             const success = localCache.del(k);
             if (success > 0) {
                 deletedCount++;
-                logger.debug(`Deleted key from local cache: ${k}`);
+                // logger.debug(`Deleted key from local cache: ${k}`);
             }
         }
 
@@ -229,7 +223,7 @@ class AdaptiveCache {
         }
 
         localCache.flushAll();
-        logger.debug("Flushed local cache");
+        // logger.debug("Flushed local cache");
     }
 
     /**
@@ -301,15 +295,11 @@ export default cache;
 
 /**
  * Regional adaptive cache backed by the in-cluster Redis instance.
- * Falls back to a local NodeCache when the regional Redis is unavailable.
+ * Falls back to a local LRU cache when the regional Redis is unavailable.
  * Use this for data that is regional in nature (e.g. status history) so
  * reads are served from the same cluster the user is hitting.
  */
-const regionalLocalCache = new NodeCache({
+const regionalLocalCache = createLocalCache();
-    stdTTL: 3600,
-    checkperiod: 120,
-    maxKeys: 10000
-});
 
 class RegionalAdaptiveCache {
     private useRedis(): boolean {
@@ -332,7 +322,7 @@ class RegionalAdaptiveCache {
                     redisTtl
                 );
                 if (success) {
-                    logger.debug(`[regional] Set key in Redis: ${key}`);
+                    // logger.debug(`[regional] Set key in Redis: ${key}`);
                     return true;
                 }
             } catch (error) {
@@ -353,10 +343,10 @@ class RegionalAdaptiveCache {
             try {
                 const value = await regionalRedisManager.get(key);
                 if (value !== null) {
-                    logger.debug(`[regional] Cache hit in Redis: ${key}`);
+                    // logger.debug(`[regional] Cache hit in Redis: ${key}`);
                     return JSON.parse(value) as T;
                 }
-                logger.debug(`[regional] Cache miss in Redis: ${key}`);
+                // logger.debug(`[regional] Cache miss in Redis: ${key}`);
                 return undefined;
             } catch (error) {
                 logger.error(
@@ -385,7 +375,7 @@ class RegionalAdaptiveCache {
                     const success = await regionalRedisManager.del(k);
                     if (success) {
                         deletedCount++;
-                        logger.debug(`[regional] Deleted key from Redis: ${k}`);
+                        // logger.debug(`[regional] Deleted key from Redis: ${k}`);
                     }
                 }
                 if (deletedCount === keys.length) return deletedCount;
@@ -400,7 +390,7 @@ class RegionalAdaptiveCache {
             const count = regionalLocalCache.del(k);
             if (count > 0) {
                 deletedCount++;
-                logger.debug(`[regional] Deleted key from local cache: ${k}`);
+                // logger.debug(`[regional] Deleted key from local cache: ${k}`);
             }
         }
         return deletedCount;

server/private/lib/certificates/acme-client.ts

@@ -0,0 +1,298 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import * as acme from "acme-client";
+import * as fs from "fs";
+import { eq } from "drizzle-orm/sql";
+import { privateConfig as config } from "#private/lib/config";
+import { DnsChallenge, db, dnsChallenge } from "@server/db";
+import { withRetry } from "@server/lib/retry";
+import logger from "@server/logger";
+import { acmeRateLimiter } from "./acmeRateLimiter";
+
+// acme-client's own retry/backoff logging (429 retries, 5xx retries, each
+// status-poll tick in waitForValidStatus) is a no-op by default - it only
+// activates via DEBUG=acme-client or this call, neither of which was wired
+// up. Without it, a cert silently retrying a Let's Encrypt rate limit for
+// several minutes is indistinguishable in our logs from one that's actually
+// hung, since our own logging only wraps the call, not what happens inside
+// it. Must run before any AcmeClient method is called.
+acme.setLogger((msg: string) => logger.info(`[acme-client] ${msg}`));
+
+// acme-client's axios retry wrapper treats any response-less request error
+// (timeout, connection reset, DNS blip reaching the ACME server) as
+// retryable, but once its internal retries are exhausted it falls through to
+// `validateStatus(response)` with `response` still undefined, throwing this
+// uninformative TypeError instead of the real network error.
+// https://github.com/publishlab/node-acme-client/blob/master/src/axios.js
+function isUnresponsiveAcmeError(error: unknown): boolean {
+    return (
+        error instanceof TypeError &&
+        error.message ===
+            "Cannot read properties of undefined (reading 'config')"
+    );
+}
+
+function normalizeAcmeError(error: unknown): Error {
+    if (isUnresponsiveAcmeError(error)) {
+        return new Error(
+            "ACME server did not respond after repeated attempts (network error reaching the ACME endpoint)",
+            { cause: error }
+        );
+    }
+    return error instanceof Error ? error : new Error(String(error));
+}
+
+export class AcmeClientManager {
+    private client: acme.Client | null = null;
+    private accountKey: string | null = null;
+
+    async initialize() {
+        try {
+            this.accountKey = await this.loadAccountKey();
+
+            this.client = new acme.Client({
+                directoryUrl: config.getRawConfig().acme!.acme_directory_url,
+                accountKey: this.accountKey
+            });
+
+            // Try to create account or get existing one
+            await this.client.createAccount({
+                termsOfServiceAgreed: true,
+                contact: [`mailto:${config.getRawConfig().acme!.contact_email}`]
+            });
+
+            logger.info("ACME client initialized successfully");
+        } catch (error) {
+            logger.error("Failed to initialize ACME client:", error);
+            throw error;
+        }
+    }
+
+    private async loadAccountKey(): Promise<string> {
+        const keyPath = config.getRawConfig().acme!.acme_account_key_path;
+
+        if (fs.existsSync(keyPath)) {
+            logger.info("Loading existing account key");
+            return fs.readFileSync(keyPath, "utf8");
+        } else {
+            logger.info("Generating new account key");
+            const privateKey = await acme.crypto.createPrivateKey();
+            const privateKeyString = privateKey.toString();
+            fs.writeFileSync(keyPath, privateKeyString);
+            return privateKeyString;
+        }
+    }
+
+    getClient(): acme.Client {
+        if (!this.client) {
+            throw new Error("ACME client not initialized");
+        }
+        return this.client;
+    }
+
+    async createOrder(domain: string, wildcard: boolean = false): Promise<any> {
+        const client = this.getClient();
+
+        const identifiers = wildcard
+            ? [
+                  { type: "dns", value: domain },
+                  { type: "dns", value: `*.${domain}` }
+              ]
+            : [{ type: "dns", value: domain }];
+
+        await acmeRateLimiter.acquire();
+        const order = await client.createOrder({
+            identifiers
+        });
+
+        if (wildcard) {
+            logger.info(`Created wildcard order for domain: ${domain}`);
+        } else {
+            logger.info(`Created order for domain: ${domain}`);
+        }
+        return order;
+    }
+
+    async getAuthorizations(order: any): Promise<any[]> {
+        const client = this.getClient();
+        await acmeRateLimiter.acquire();
+        return client.getAuthorizations(order);
+    }
+
+    async handleDnsChallenge(
+        dnsChallenges: {
+            authz: any;
+            challenge: any;
+        }[]
+    ): Promise<void> {
+        const client = this.getClient();
+
+        let challengeDomains: DnsChallenge[] = [];
+
+        for (const { authz, challenge } of dnsChallenges) {
+            const keyAuthorization =
+                await client.getChallengeKeyAuthorization(challenge);
+
+            // Extract the domain from authorization
+            const domain = authz.identifier.value;
+
+            // Store challenge in database for DNS server to pick up
+            challengeDomains = await withRetry(
+                () =>
+                    db
+                        .insert(dnsChallenge)
+                        .values({
+                            domain: domain,
+                            token: challenge.token,
+                            keyAuthorization,
+                            createdAt: Math.floor(Date.now() / 1000),
+                            expiresAt: Math.floor(
+                                (Date.now() +
+                                    config.getRawConfig().acme!
+                                        .challenge_ttl_ms) /
+                                    1000
+                            )
+                        })
+                        .returning(),
+                { label: `insert dnsChallenge for domain ${domain}` }
+            );
+
+            logger.info(
+                `DNS challenge stored for domain: ${domain} as token ${challenge.token} and keyAuthorization`
+            );
+        }
+
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+
+        const failedDomains: string[] = [];
+
+        for (const { authz, challenge } of dnsChallenges) {
+            const domain = authz.identifier.value;
+            const challengeDomain = `_acme-challenge.${domain}`;
+
+            try {
+                // The ACME server occasionally has a transient network blip
+                // mid-sequence; retry the whole verify/complete/wait sequence
+                // rather than just the DNS challenge propagation wait, since
+                // these calls are safe to repeat against the ACME server.
+                await withRetry(
+                    async () => {
+                        // Verify challenge
+                        await acmeRateLimiter.acquire();
+                        await client.verifyChallenge(authz, challenge);
+
+                        // Complete challenge
+                        logger.info(
+                            `Completing challenge for domain: ${challengeDomain}`
+                        );
+                        await acmeRateLimiter.acquire();
+                        await client.completeChallenge(challenge);
+
+                        // Wait for validation
+                        logger.info(
+                            `Waiting for challenge to be validated for domain: ${challengeDomain}...`
+                        );
+                        await acmeRateLimiter.acquire();
+                        await client.waitForValidStatus(challenge);
+                    },
+                    {
+                        retries: 2,
+                        baseDelayMs: 5000,
+                        label: `ACME challenge completion for domain ${domain}`,
+                        // Only retry the known network-blip crash - a
+                        // genuine validation failure (e.g. challenge marked
+                        // "invalid" because the DNS record wasn't found) is
+                        // permanent and should fail immediately instead of
+                        // burning Let's Encrypt's per-hostname failed-
+                        // validation rate limit on retries that can't help.
+                        shouldRetry: isUnresponsiveAcmeError
+                    }
+                );
+
+                logger.info(`Challenge completed for domain: ${domain}`);
+            } catch (error) {
+                logger.error(
+                    `Failed to complete challenge for domain ${domain}:`,
+                    normalizeAcmeError(error)
+                );
+                failedDomains.push(domain);
+            }
+        }
+
+        for (const challengeDomain of challengeDomains) {
+            await this.removeDnsChallenge(challengeDomain.dnsChallengeId);
+            logger.info(
+                `Removed DNS challenge for domain: ${challengeDomain.domain}`
+            );
+        }
+
+        // A failed dns-01 challenge leaves the order stuck in "pending" -
+        // finalizing it would just fail with a confusing ACME error, so
+        // stop here and let the caller mark the certificate as failed.
+        if (failedDomains.length > 0) {
+            throw new Error(
+                `DNS-01 challenge validation failed for domain(s): ${failedDomains.join(", ")}`
+            );
+        }
+    }
+
+    async removeDnsChallenge(dnsChallengeId: number): Promise<void> {
+        try {
+            await withRetry(
+                () =>
+                    db
+                        .delete(dnsChallenge)
+                        .where(eq(dnsChallenge.dnsChallengeId, dnsChallengeId)),
+                { label: `delete dnsChallenge ${dnsChallengeId}` }
+            );
+        } catch (error) {
+            logger.error(
+                `Failed to clean up DNS challenge for id ${dnsChallengeId}:`,
+                error
+            );
+        }
+    }
+
+    async finalizeCertificate(
+        order: any,
+        domain: string,
+        wildcard: boolean = false
+    ): Promise<{ certificate: string; privateKey: string }> {
+        const client = this.getClient();
+
+        const altNames = wildcard ? [`*.${domain}`, domain] : [domain];
+
+        // Create CSR
+        const [privateKey, csr] = await acme.crypto.createCsr({
+            altNames
+        });
+
+        // Finalize order
+        await acmeRateLimiter.acquire();
+        const finalizedOrder = await client.finalizeOrder(order, csr);
+
+        // Get certificate
+        await acmeRateLimiter.acquire();
+        const certificate = await client.getCertificate(finalizedOrder);
+
+        logger.info(`Certificate obtained for domain: ${domain}`);
+
+        return {
+            certificate: certificate.toString(),
+            privateKey: privateKey.toString()
+        };
+    }
+}
+
+export const acmeClientManager = new AcmeClientManager();

server/private/lib/certificates/acmeRateLimiter.ts

@@ -0,0 +1,71 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { privateConfig as config } from "#private/lib/config";
+import logger from "@server/logger";
+import { redis } from "../redis";
+// Caps outgoing ACME API calls to a fixed budget per wall-clock second,
+// shared across all pops workers via Redis (mirrors the lockManager pattern
+// in @lib/lock) - a per-process limiter wouldn't be enough since multiple
+// workers issue certificates against the same Let's Encrypt account.
+const ACQUIRE_SCRIPT = `
+local key = KEYS[1]
+local limit = tonumber(ARGV[1])
+local current = redis.call('INCR', key)
+if current == 1 then
+    redis.call('PEXPIRE', key, 2000)
+end
+if current > limit then
+    return 0
+else
+    return 1
+end
+`;
+
+class AcmeRateLimiter {
+    async acquire(): Promise<void> {
+        const limit =
+            config.getRawConfig().acme?.acme_requests_per_second ?? 15;
+
+        for (;;) {
+            const bucket = Math.floor(Date.now() / 1000);
+            const key = `acme_rate_limit:${bucket}`;
+
+            let allowed: number;
+            try {
+                allowed = (await redis.eval(
+                    ACQUIRE_SCRIPT,
+                    1,
+                    key,
+                    limit.toString()
+                )) as number;
+            } catch (error) {
+                logger.error(
+                    "ACME rate limiter check failed, proceeding without throttling:",
+                    error
+                );
+                return;
+            }
+
+            if (allowed === 1) {
+                return;
+            }
+
+            // Budget for this second is spent - wait for the next window.
+            const waitMs = 1000 - (Date.now() % 1000) + 10;
+            await new Promise((resolve) => setTimeout(resolve, waitMs));
+        }
+    }
+}
+
+export const acmeRateLimiter = new AcmeRateLimiter();

server/private/lib/certificates/certificate-service.ts

@@ -0,0 +1,511 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { acmeClientManager } from "./acme-client";
+import { dnsValidator } from "./dns-validator";
+import { getTableColumns } from "drizzle-orm";
+import { eq, and, or, isNull, lt, asc } from "drizzle-orm/sql";
+import { config } from "@server/lib/config";
+import { db, certificates, domains, Certificate } from "@server/db";
+import { encrypt } from "@server/lib/crypto";
+import { withTimeout, withRetry } from "@server/lib/retry";
+import logger from "@server/logger";
+import { lockManager } from "../lock";
+import { pushCertUpdateToAffectedNewts } from "@server/lib/acmeCertSync";
+import crypto from "crypto";
+
+// Number of on-demand DNS validation attempts made right before a
+// certificate is (re)issued, to avoid burning Let's Encrypt rate limits on
+// domains whose DNS has drifted since they were last verified.
+const PRE_CERT_DNS_VALIDATION_ATTEMPTS = 3;
+
+// Hard ceiling on a single certificate's issuance/renewal flow. acme-client's
+// axios instance never sets a request timeout, so a stalled connection to
+// the ACME server hangs forever instead of erroring - and since
+// processPendingCertificates/processRenewalCandidates gate the *next* batch
+// on Promise.all(...) over the current one, one hung certificate would
+// otherwise stall every other domain permanently. Sized generously above the
+// legitimate worst case (acme-client's own bounded backoff is ~3.6min per
+// status-polling loop, and a wildcard cert's two identifiers plus order
+// finalization can chain a few of those) so this only fires on a genuine hang.
+const CERTIFICATE_ISSUANCE_TIMEOUT_MS = 20 * 60 * 1000;
+
+// "requested" is set the instant a cert starts processing and is never
+// queried anywhere else - processPendingCertificates only selects "pending"
+// and processRenewalCandidates only selects "valid". So if the *process*
+// dies mid-flight (OOM, node eviction, a rolling deploy) rather than just
+// hanging, the row is orphaned in "requested" permanently with nothing to
+// ever pick it back up, no matter how good the in-process timeouts are.
+// Threshold is set comfortably above CERTIFICATE_ISSUANCE_TIMEOUT_MS plus the
+// scheduler's own outer backstop so this never reclaims a cert that's still
+// genuinely being worked on.
+const STUCK_CERTIFICATE_THRESHOLD_MS = 40 * 60 * 1000;
+
+export class CertificateService {
+    // Runs at the top of every processPendingCertificates tick so an
+    // interrupted worker's leftovers always get put back in the queue
+    // instead of sitting invisible to every query forever.
+    private async reclaimStuckCertificates(): Promise<void> {
+        const staleBefore =
+            Math.floor(Date.now() / 1000) -
+            Math.floor(STUCK_CERTIFICATE_THRESHOLD_MS / 1000);
+
+        const reclaimed = await db
+            .update(certificates)
+            .set({
+                status: "pending",
+                errorMessage:
+                    'Reclaimed after being stuck in "requested" state - the worker processing it likely restarted or crashed',
+                updatedAt: Math.floor(Date.now() / 1000)
+            })
+            .where(
+                and(
+                    eq(certificates.status, "requested"),
+                    lt(certificates.updatedAt, staleBefore)
+                )
+            )
+            .returning({ domain: certificates.domain });
+
+        if (reclaimed.length > 0) {
+            logger.warn(
+                `Reclaimed ${reclaimed.length} certificate(s) stuck in "requested" state: ${reclaimed
+                    .map((c) => c.domain)
+                    .join(", ")}`
+            );
+        }
+    }
+
+    async processPendingCertificates(): Promise<void> {
+        logger.debug("Checking for pending certificates...");
+
+        await this.reclaimStuckCertificates();
+
+        const pendingCerts = await db
+            .select(getTableColumns(certificates))
+            .from(certificates)
+            .leftJoin(domains, eq(certificates.domainId, domains.domainId))
+            .where(
+                and(
+                    eq(certificates.status, "pending"),
+                    or(
+                        // Certs with no linked domain row (e.g. legacy certs
+                        // imported from acme.json) aren't gated on domain
+                        // verification since there's nothing to check.
+                        isNull(certificates.domainId),
+                        and(
+                            eq(domains.verified, true),
+                            eq(domains.failed, false)
+                        )
+                    )
+                )
+            )
+            .limit(10);
+
+        if (pendingCerts.length === 0) {
+            logger.debug("No pending certificates found");
+            return;
+        }
+
+        logger.info(`Found ${pendingCerts.length} pending certificates`);
+
+        // Process the batch concurrently so one domain stuck retrying a slow
+        // DNS-01 challenge (the ACME client's waitForValidStatus can spend
+        // minutes on a bad domain) doesn't stall the rest of the batch.
+        // processSingleCertificate catches its own errors and each cert uses
+        // an independent per-domain lock, so this is safe to parallelize.
+        await Promise.all(
+            pendingCerts.map((cert) => this.processSingleCertificate(cert))
+        );
+    }
+
+    async processRenewalCandidates(): Promise<void> {
+        logger.debug("Checking for certificates needing renewal...");
+
+        const now = Math.floor(Date.now() / 1000);
+
+        const renewalCandidates = await db
+            .select(getTableColumns(certificates))
+            .from(certificates)
+            .leftJoin(domains, eq(certificates.domainId, domains.domainId))
+            .where(
+                and(
+                    eq(certificates.status, "valid"),
+                    lt(certificates.expiresAt, now + 15 * 24 * 60 * 60), // 15 days from now
+                    or(
+                        // Certs with no linked domain row (e.g. legacy certs
+                        // imported from acme.json) aren't gated on domain
+                        // verification since there's nothing to check.
+                        isNull(certificates.domainId),
+                        and(
+                            eq(domains.verified, true),
+                            eq(domains.failed, false)
+                        )
+                    )
+                )
+            )
+            // Most urgent first, so already-expired certs aren't starved
+            // behind the limit by certs that still have weeks of runway.
+            .orderBy(asc(certificates.expiresAt))
+            .limit(50);
+
+        if (renewalCandidates.length === 0) {
+            logger.debug("No certificates need renewal");
+            return;
+        }
+
+        logger.info(
+            `Found ${renewalCandidates.length} certificates needing renewal`
+        );
+
+        for (const cert of renewalCandidates) {
+            if (cert.expiresAt !== null && cert.expiresAt < now) {
+                logger.warn(
+                    `Certificate for ${cert.domain} is marked "valid" but already expired at ${new Date(cert.expiresAt * 1000).toISOString()} (bad state) - renewing immediately`
+                );
+            }
+        }
+
+        // Process the batch concurrently - see processPendingCertificates for why.
+        await Promise.all(
+            renewalCandidates.map((cert) => this.renewCertificate(cert))
+        );
+    }
+
+    private async processSingleCertificate(cert: Certificate): Promise<void> {
+        const lockKey = `cert:${cert.domain}`;
+
+        const lockToken = await lockManager.acquireLock(lockKey);
+        if (!lockToken) {
+            logger.debug(
+                `Could not acquire lock for certificate: ${cert.domain}`
+            );
+            return;
+        }
+
+        try {
+            logger.info(`Processing certificate for domain: ${cert.domain}`);
+
+            // Update status to processing
+            await db
+                .update(certificates)
+                .set({
+                    status: "requested",
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+            //
+
+            await withTimeout(
+                this.obtainCertificate(cert),
+                CERTIFICATE_ISSUANCE_TIMEOUT_MS,
+                `certificate issuance for ${cert.domain}`
+            );
+        } catch (error) {
+            logger.error(
+                `Failed to process certificate for ${cert.domain}:`,
+                error
+            );
+
+            await db
+                .update(certificates)
+                .set({
+                    status: "failed",
+                    errorMessage:
+                        error instanceof Error
+                            ? error.message
+                            : "Unknown error",
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+        } finally {
+            await lockManager.releaseLock(lockKey, lockToken);
+        }
+    }
+
+    private async renewCertificate(cert: Certificate): Promise<void> {
+        const lockKey = `cert:${cert.domain}`;
+
+        const lockToken = await lockManager.acquireLock(lockKey);
+        if (!lockToken) {
+            logger.debug(
+                `Could not acquire lock for certificate renewal: ${cert.domain}`
+            );
+            return;
+        }
+
+        try {
+            logger.info(`Renewing certificate for domain: ${cert.domain}`);
+
+            // Update last renewal attempt
+            await db
+                .update(certificates)
+                .set({
+                    lastRenewalAttempt: Math.floor(Date.now() / 1000),
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+
+            await withTimeout(
+                this.obtainCertificate(cert),
+                CERTIFICATE_ISSUANCE_TIMEOUT_MS,
+                `certificate renewal for ${cert.domain}`
+            );
+        } catch (error) {
+            logger.error(
+                `Failed to renew certificate for ${cert.domain}:`,
+                error
+            );
+
+            await db
+                .update(certificates)
+                .set({
+                    status: "failed",
+                    errorMessage:
+                        error instanceof Error
+                            ? error.message
+                            : "Unknown error",
+                    lastRenewalAttempt: Math.floor(Date.now() / 1000),
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+        } finally {
+            await lockManager.releaseLock(lockKey, lockToken);
+        }
+    }
+
+    // Re-checks the domain's DNS records right before we spend a Let's
+    // Encrypt order on it, so drift that happened after the domain was
+    // originally verified doesn't burn ACME rate limits. Certs with no
+    // linked domain row (e.g. legacy/manually-managed certs) skip this and
+    // proceed as before, since there are no tracked DNS records to check.
+    private async verifyDomainBeforeIssuance(cert: Certificate): Promise<void> {
+        if (!cert.domainId) {
+            return;
+        }
+
+        const [domain] = await db
+            .select()
+            .from(domains)
+            .where(eq(domains.domainId, cert.domainId))
+            .limit(1);
+
+        if (!domain) {
+            return;
+        }
+
+        for (
+            let attempt = 1;
+            attempt <= PRE_CERT_DNS_VALIDATION_ATTEMPTS;
+            attempt++
+        ) {
+            // Offset `tries` so each attempt round-robins to a different
+            // privateConfigured DNS resolver instead of re-querying the same one.
+            const probe = { ...domain, tries: domain.tries + attempt - 1 };
+            if (
+                await dnsValidator.validateDomain(probe, {
+                    forceRecheck: true
+                })
+            ) {
+                await db
+                    .update(domains)
+                    .set({ verified: true, failed: false, errorMessage: null })
+                    .where(eq(domains.domainId, domain.domainId));
+                return;
+            }
+
+            logger.warn(
+                `Pre-certificate DNS check ${attempt}/${PRE_CERT_DNS_VALIDATION_ATTEMPTS} failed for domain ${domain.baseDomain} (cert: ${cert.domain})`
+            );
+        }
+
+        const errorMessage = `Domain failed DNS validation ${PRE_CERT_DNS_VALIDATION_ATTEMPTS} times before certificate issuance`;
+        await db
+            .update(domains)
+            .set({ verified: false, failed: true, errorMessage })
+            .where(eq(domains.domainId, domain.domainId));
+
+        throw new Error(errorMessage);
+    }
+
+    private async obtainCertificate(cert: Certificate): Promise<void> {
+        await this.verifyDomainBeforeIssuance(cert);
+
+        // Create order
+        const order = await acmeClientManager.createOrder(
+            cert.domain,
+            cert.wildcard || false
+        );
+
+        // Update with order ID
+        await withRetry(
+            () =>
+                db
+                    .update(certificates)
+                    .set({
+                        orderId: order.url,
+                        updatedAt: Math.floor(Date.now() / 1000)
+                    })
+                    .where(eq(certificates.certId, cert.certId)),
+            { label: `update orderId for certificate ${cert.domain}` }
+        );
+
+        // Get authorizations
+        const authorizations = await acmeClientManager.getAuthorizations(order);
+
+        // Aggregate all DNS-01 challenges
+        const dnsChallenges = authorizations.map((authz: any) => {
+            const dnsChallenge = authz.challenges.find(
+                (c: any) => c.type === "dns-01"
+            );
+            if (!dnsChallenge) {
+                throw new Error(
+                    `No DNS-01 challenge found for ${authz.identifier.value}`
+                );
+            }
+            return {
+                authz,
+                challenge: dnsChallenge
+            };
+        });
+
+        // Send all DNS-01 challenges in one request to handleDnsChallenge
+        await acmeClientManager.handleDnsChallenge(dnsChallenges);
+
+        // Finalize certificate
+        const { certificate, privateKey } =
+            await acmeClientManager.finalizeCertificate(
+                order,
+                cert.domain,
+                cert.wildcard || false
+            );
+
+        const encryptionKey = config.getRawConfig().server.secret;
+        if (!encryptionKey) {
+            throw new Error("Encryption key not provided");
+        }
+
+        // Encrypt certificate and private key
+        const encryptedCert = encrypt(certificate, encryptionKey);
+        const encryptedKey = encrypt(privateKey, encryptionKey);
+
+        // Parse certificate to get expiration date
+        const expiresAt = this.extractExpirationDate(certificate);
+
+        // Update database record. This persists the certificate we just
+        // obtained from the ACME server, so it's retried aggressively -
+        // losing this write means re-issuing the cert from scratch.
+        await withRetry(
+            () =>
+                db
+                    .update(certificates)
+                    .set({
+                        status: "valid",
+                        expiresAt: Math.floor(expiresAt.getTime() / 1000),
+                        renewalCount: (cert.renewalCount || 0) + 1,
+                        errorMessage: null,
+                        updatedAt: Math.floor(Date.now() / 1000),
+                        certFile: encryptedCert,
+                        keyFile: encryptedKey
+                    })
+                    .where(eq(certificates.certId, cert.certId)),
+            {
+                retries: 5,
+                label: `persist issued certificate for ${cert.domain}`
+            }
+        );
+
+        logger.info(
+            `Certificate successfully obtained/renewed for domain: ${cert.domain}`
+        );
+
+        await pushCertUpdateToAffectedNewts(
+            cert.domain,
+            cert.domainId ?? null,
+            certificate,
+            privateKey
+        );
+    }
+
+    private extractExpirationDate(certificate: string): Date {
+        try {
+            // Extract the certificate block
+            const pem = certificate
+                .replace(/-----BEGIN CERTIFICATE-----/g, "")
+                .replace(/-----END CERTIFICATE-----/g, "")
+                .replace(/\s+/g, "");
+            const der = Buffer.from(pem, "base64");
+
+            // Use Node.js crypto to parse the certificate
+            const x509 = new crypto.X509Certificate(der);
+            return new Date(x509.validTo);
+        } catch (error) {
+            logger.warn(
+                "Failed to parse certificate expiration date, using default",
+                error
+            );
+            // Default to 90 days from now (Let's Encrypt default)
+            return new Date(Date.now() + 90 * 24 * 60 * 60 * 1000);
+        }
+    }
+
+    async addCertificateRequest(domain: string): Promise<void> {
+        try {
+            await db.insert(certificates).values({
+                domain,
+                status: "pending",
+                createdAt: Math.floor(Date.now() / 1000),
+                updatedAt: Math.floor(Date.now() / 1000)
+            });
+            logger.info(`Certificate request added for domain: ${domain}`);
+        } catch (error) {
+            if (error instanceof Error && error.message.includes("unique")) {
+                logger.warn(
+                    `Certificate request already exists for domain: ${domain}`
+                );
+            } else {
+                throw error;
+            }
+        }
+    }
+
+    async getCertificateStatus(domain: string) {
+        const cert = await db
+            .select()
+            .from(certificates)
+            .where(eq(certificates.domain, domain))
+            .limit(1);
+
+        return cert[0] || null;
+    }
+
+    async cleanupExpiredChallenges(): Promise<void> {
+        try {
+            const result = await db
+                .delete(certificates)
+                .where(
+                    lt(certificates.expiresAt, Math.floor(Date.now() / 1000))
+                )
+                .returning();
+
+            if (result.length > 0) {
+                logger.info(
+                    `Cleaned up ${result.length} expired DNS challenges`
+                );
+            }
+        } catch (error) {
+            logger.error("Failed to cleanup expired challenges:", error);
+        }
+    }
+}
+
+export const certificateService = new CertificateService();

server/private/lib/certificates/dns-validator.ts

@@ -0,0 +1,334 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { eq, and, lt } from "drizzle-orm";
+import * as dns from "dns/promises";
+import { privateConfig as config } from "#private/lib/config";
+import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db";
+import logger from "@server/logger";
+import { lockManager } from "../lock";
+
+export const DNS_VALIDATOR_MAX_TRIES = 300;
+
+export class DNSValidator {
+    private static readonly MAX_TRIES = DNS_VALIDATOR_MAX_TRIES;
+
+    constructor() {}
+
+    async validateAll(): Promise<void> {
+        // Get all domains that are not yet verified and haven't exceeded max tries
+        const unverifiedDomains: Domain[] = await db
+            .select()
+            .from(domains)
+            .where(
+                and(
+                    eq(domains.verified, false),
+                    lt(domains.tries, DNSValidator.MAX_TRIES)
+                )
+            );
+
+        if (unverifiedDomains.length === 0) {
+            logger.debug("No unverified domains found for DNS validation");
+            return;
+        }
+
+        logger.info(`Validating ${unverifiedDomains.length} DNS records`);
+
+        for (const domain of unverifiedDomains) {
+            const lockKey = `dns:${domain.baseDomain}`;
+            const lockToken = await lockManager.acquireLock(lockKey);
+            if (!lockToken) {
+                logger.debug(
+                    `Could not acquire lock for DNS validation: ${domain.baseDomain}`
+                );
+                continue;
+            }
+            try {
+                const isValid = await this.validateDomain(domain);
+                if (isValid) {
+                    await db
+                        .update(domains)
+                        .set({
+                            verified: true,
+                            failed: false,
+                            tries: 0,
+                            errorMessage: null
+                        })
+                        .where(eq(domains.domainId, domain.domainId));
+                    logger.info(
+                        `Domain ${domain.baseDomain} validated successfully`
+                    );
+                } else {
+                    const newTries = domain.tries + 1;
+                    const shouldMarkAsFailed =
+                        newTries >= DNSValidator.MAX_TRIES;
+
+                    await db
+                        .update(domains)
+                        .set({
+                            tries: newTries,
+                            failed: shouldMarkAsFailed
+                        })
+                        .where(eq(domains.domainId, domain.domainId));
+
+                    if (shouldMarkAsFailed) {
+                        logger.warn(
+                            `Domain ${domain.baseDomain} exceeded maximum tries (${DNSValidator.MAX_TRIES}), marking as failed`
+                        );
+                    } else {
+                        logger.debug(
+                            `Domain ${domain.baseDomain} did not validate (attempt ${newTries}/${DNSValidator.MAX_TRIES})`
+                        );
+                    }
+                }
+            } catch (err) {
+                logger.warn(
+                    `Error validating domain ${domain.baseDomain}:`,
+                    err
+                );
+                // Increment tries even on error
+                const newTries = domain.tries + 1;
+                const shouldMarkAsFailed = newTries >= DNSValidator.MAX_TRIES;
+
+                await db
+                    .update(domains)
+                    .set({
+                        tries: newTries,
+                        failed: shouldMarkAsFailed
+                    })
+                    .where(eq(domains.domainId, domain.domainId));
+            } finally {
+                await lockManager.releaseLock(lockKey, lockToken);
+            }
+        }
+    }
+
+    async validateDomain(
+        domain: Domain,
+        opts: { forceRecheck?: boolean } = {}
+    ): Promise<boolean> {
+        const { forceRecheck = false } = opts;
+        const resolver = new dns.Resolver();
+        const servers = config.getRawConfig().acme?.dns_resolvers;
+        if (!servers || servers.length === 0) {
+            throw new Error("No DNS resolvers configured");
+        }
+        const dnsServer = servers[domain.tries % servers.length]!;
+        resolver.setServers([dnsServer]);
+        logger.debug(
+            `Using DNS server ${dnsServer} for domain ${domain.baseDomain} (try ${domain.tries})`
+        );
+
+        // Get all DNS records for this domain
+        const records: DnsRecord[] = await db
+            .select()
+            .from(dnsRecords)
+            .where(eq(dnsRecords.domainId, domain.domainId));
+
+        if (records.length === 0) {
+            logger.warn(`No DNS records found for domain ${domain.baseDomain}`);
+            return false;
+        }
+
+        if (!forceRecheck && records.every((r) => r.verified)) {
+            logger.info(
+                `All DNS records already verified for domain ${domain.baseDomain}`
+            );
+            return true;
+        }
+
+        logger.info(
+            `Validating ${records.length} DNS records for domain ${domain.baseDomain}`
+        );
+
+        // Collect the full set of expected NS values for this domain so we can
+        // detect extra records that are present in DNS but not in our DB.
+        const expectedNsValues = new Set<string>(
+            records.filter((r) => r.recordType === "NS").map((r) => r.value)
+        );
+
+        // Cache resolved NS records across iterations — there will be 3 NS
+        // records in the DB and we don't need to hit the upstream server 3 times.
+        let previousNs: string[] | null = null;
+
+        for (const record of records) {
+            // Skip already verified records, unless a live recheck was requested
+            if (record.verified && !forceRecheck) {
+                continue;
+            }
+
+            let isValid = false;
+
+            try {
+                if (record.recordType === "NS") {
+                    let nsRecords: string[] | null = previousNs;
+                    if (!nsRecords) {
+                        nsRecords = await resolver.resolveNs(
+                            record.baseDomain || domain.baseDomain
+                        );
+                    }
+                    logger.info(
+                        `NS records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        nsRecords
+                    );
+
+                    // Check if this expected NS value is present in the live records.
+                    // A stale/legacy expected value (e.g. left over from a
+                    // nameserver rebrand) is also accepted as long as the live
+                    // records resolve to some other known-valid nameserver —
+                    // the specific literal hostname stored per-domain isn't
+                    // meaningful once it's a recognized alias.
+                    isValid = nsRecords.some((ns) => ns === record.value);
+
+                    previousNs = nsRecords;
+                } else if (record.recordType === "CNAME") {
+                    const cnameRecords = await resolver.resolveCname(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    logger.info(
+                        `CNAME records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        cnameRecords
+                    );
+
+                    // Check if the CNAME record matches the expected value
+                    isValid =
+                        cnameRecords.length === 1 &&
+                        cnameRecords[0] === record.value;
+                } else if (record.recordType === "TXT") {
+                    const txtRecords = await resolver.resolveTxt(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    logger.info(
+                        `TXT records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        txtRecords
+                    );
+
+                    // TXT records come as an array of arrays, flatten and check
+                    const flatTxtRecords = txtRecords.flat();
+                    isValid = flatTxtRecords.includes(record.value);
+                } else if (record.recordType === "A") {
+                    const aRecords = await resolver.resolve4(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    logger.info(
+                        `A records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        aRecords
+                    );
+
+                    // Check if the A record matches the expected value
+                    isValid = aRecords.includes(record.value);
+                } else {
+                    logger.warn(
+                        `Unsupported record type: ${record.recordType}`
+                    );
+                    continue;
+                }
+            } catch (error) {
+                isValid = false;
+                logger.debug(
+                    `Did not resolve ${record.recordType} record for ${
+                        record.baseDomain || domain.baseDomain
+                    }:`,
+                    error
+                );
+            }
+
+            // Update the individual record verification status. Runs for
+            // both a mismatched value and a failed/thrown DNS lookup, so a
+            // previously-verified record that stops resolving (e.g. NXDOMAIN
+            // after NS delegation is dropped) gets downgraded instead of
+            // leaving stale `verified: true` state behind.
+            if (isValid) {
+                await db
+                    .update(dnsRecords)
+                    .set({ verified: true })
+                    .where(eq(dnsRecords.id, record.id));
+                logger.info(
+                    `DNS record ${record.id} (${record.recordType}) for ${
+                        record.baseDomain || domain.baseDomain
+                    } verified successfully`
+                );
+            } else {
+                if (record.verified) {
+                    await db
+                        .update(dnsRecords)
+                        .set({ verified: false })
+                        .where(eq(dnsRecords.id, record.id));
+                }
+                logger.debug(
+                    `DNS record ${record.id} (${record.recordType}) for ${
+                        record.baseDomain || domain.baseDomain
+                    } does not match expected value: ${record.value}`
+                );
+            }
+        }
+
+        // --- Extra NS record check ---
+        // If we resolved NS records during this pass, verify that the live DNS
+        // has no nameservers beyond the ones we expect.  Individual records may
+        // already be marked verified above, but we must block full domain
+        // verification until the extra records are removed.
+        if (previousNs !== null && expectedNsValues.size > 0) {
+            const extraNsRecords = previousNs.filter(
+                (ns) => !expectedNsValues.has(ns)
+            );
+
+            if (extraNsRecords.length > 0) {
+                const errorMessage = `Extra NS records found that are not expected: ${extraNsRecords.join(", ")}. Remove these nameservers to complete domain verification.`;
+
+                await db
+                    .update(domains)
+                    .set({ errorMessage })
+                    .where(eq(domains.domainId, domain.domainId));
+
+                logger.warn(
+                    `Domain ${domain.baseDomain} has extra NS records that prevent verification: ${extraNsRecords.join(", ")}`
+                );
+
+                return false;
+            }
+
+            // No extras — clear any stale error that was previously written
+            await db
+                .update(domains)
+                .set({ errorMessage: null })
+                .where(eq(domains.domainId, domain.domainId));
+        }
+
+        // Check if all records are now verified
+        const updatedRecords: DnsRecord[] = await db
+            .select()
+            .from(dnsRecords)
+            .where(eq(dnsRecords.domainId, domain.domainId));
+
+        const allRecordsVerified = updatedRecords.every((r) => r.verified);
+
+        logger.info(
+            `Domain ${domain.baseDomain}: ${
+                updatedRecords.filter((r) => r.verified).length
+            }/${updatedRecords.length} records verified`
+        );
+
+        return allRecordsVerified;
+    }
+}
+
+export const dnsValidator = new DNSValidator();

server/private/lib/certificates/domain-reverifier.ts

@@ -0,0 +1,233 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { eq, and, or, isNull, lt } from "drizzle-orm";
+import * as dns from "dns/promises";
+import { DNS_VALIDATOR_MAX_TRIES } from "./dns-validator";
+import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db";
+import logger from "@server/logger";
+import { lockManager } from "../lock";
+import { privateConfig as config } from "#private/lib/config";
+
+// Module-level counter so successive domains in a batch round-robin across servers.
+let serverIndex = 0;
+
+export class DomainReverifier {
+    async reverifyAll(): Promise<void> {
+        const certConfig = config.getRawConfig().acme;
+        if (!certConfig) {
+            logger.debug(
+                "No certificate config — skipping domain reverification"
+            );
+            return;
+        }
+
+        const windowMs = certConfig.domain_reverification_window_ms;
+        const batchSize = certConfig.domain_reverification_batch_size;
+        const windowSecs = Math.floor(windowMs / 1000);
+        const cutoff = Math.floor(Date.now() / 1000) - windowSecs;
+
+        const domainsToCheck: Domain[] = await db
+            .select()
+            .from(domains)
+            .where(
+                and(
+                    eq(domains.verified, true),
+                    or(
+                        isNull(domains.lastCheckedAt),
+                        lt(domains.lastCheckedAt, cutoff)
+                    )
+                )
+            )
+            .limit(batchSize);
+
+        if (domainsToCheck.length === 0) {
+            logger.debug("No verified domains due for reverification");
+            return;
+        }
+
+        logger.info(`Reverifying ${domainsToCheck.length} domains`);
+
+        for (const domain of domainsToCheck) {
+            const lockKey = `dns-reverify:${domain.baseDomain}`;
+            const lockToken = await lockManager.acquireLock(lockKey);
+            if (!lockToken) {
+                logger.debug(
+                    `Could not acquire lock for domain reverification: ${domain.baseDomain}`
+                );
+                continue;
+            }
+
+            try {
+                await this.reverifyDomain(domain, certConfig.dns_resolvers);
+            } catch (err) {
+                logger.warn(
+                    `Unexpected error reverifying domain ${domain.baseDomain}:`,
+                    err
+                );
+                // Still stamp lastCheckedAt so we don't hammer a broken domain every run.
+                await db
+                    .update(domains)
+                    .set({ lastCheckedAt: Math.floor(Date.now() / 1000) })
+                    .where(eq(domains.domainId, domain.domainId));
+            } finally {
+                await lockManager.releaseLock(lockKey, lockToken);
+            }
+        }
+    }
+
+    private async reverifyDomain(
+        domain: Domain,
+        servers: string[]
+    ): Promise<void> {
+        if (!servers || servers.length === 0) {
+            throw new Error("No DNS resolvers configured");
+        }
+
+        // Round-robin across servers; advance the global counter so the next
+        // domain in the same batch gets a different server.
+        const dnsServer = servers[serverIndex % servers.length]!;
+        serverIndex++;
+
+        const resolver = new dns.Resolver();
+        resolver.setServers([dnsServer]);
+
+        logger.debug(
+            `Reverifying domain ${domain.baseDomain} using DNS server ${dnsServer}`
+        );
+
+        const records: DnsRecord[] = await db
+            .select()
+            .from(dnsRecords)
+            .where(eq(dnsRecords.domainId, domain.domainId));
+
+        if (records.length === 0) {
+            logger.warn(
+                `No DNS records found for domain ${domain.baseDomain} during reverification — marking failed`
+            );
+            await this.markFailed(
+                domain.domainId,
+                "No DNS records found during periodic reverification"
+            );
+            return;
+        }
+
+        const expectedNsValues = new Set<string>(
+            records.filter((r) => r.recordType === "NS").map((r) => r.value)
+        );
+
+        let allValid = true;
+        let errorMessage: string | null = null;
+        let resolvedNs: string[] | null = null;
+
+        for (const record of records) {
+            let isValid = false;
+
+            try {
+                if (record.recordType === "NS") {
+                    if (!resolvedNs) {
+                        resolvedNs = await resolver.resolveNs(
+                            record.baseDomain || domain.baseDomain
+                        );
+                    }
+                    isValid = resolvedNs.some((ns) => ns === record.value);
+                } else if (record.recordType === "CNAME") {
+                    const cnameRecords = await resolver.resolveCname(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    isValid =
+                        cnameRecords.length === 1 &&
+                        cnameRecords[0] === record.value;
+                } else if (record.recordType === "TXT") {
+                    const txtRecords = await resolver.resolveTxt(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    isValid = txtRecords.flat().includes(record.value);
+                } else if (record.recordType === "A") {
+                    const aRecords = await resolver.resolve4(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    isValid = aRecords.includes(record.value);
+                } else {
+                    logger.warn(
+                        `Unsupported record type ${record.recordType} during reverification of ${domain.baseDomain}`
+                    );
+                    continue;
+                }
+            } catch (err) {
+                logger.debug(
+                    `DNS lookup failed for ${record.recordType} record on ${record.baseDomain || domain.baseDomain}:`,
+                    err
+                );
+                isValid = false;
+            }
+
+            if (!isValid) {
+                allValid = false;
+                errorMessage = `${record.recordType} record for ${record.baseDomain || domain.baseDomain} no longer resolves to expected value "${record.value}"`;
+                break;
+            }
+        }
+
+        // Check for extra NS records beyond what we expect.
+        if (allValid && resolvedNs !== null && expectedNsValues.size > 0) {
+            const extraNs = resolvedNs.filter(
+                (ns) => !expectedNsValues.has(ns)
+            );
+            if (extraNs.length > 0) {
+                allValid = false;
+                errorMessage = `Extra NS records found: ${extraNs.join(", ")}. Remove these nameservers.`;
+            }
+        }
+
+        const now = Math.floor(Date.now() / 1000);
+
+        if (allValid) {
+            await db
+                .update(domains)
+                .set({ lastCheckedAt: now, errorMessage: null })
+                .where(eq(domains.domainId, domain.domainId));
+            logger.debug(
+                `Domain ${domain.baseDomain} passed periodic reverification`
+            );
+        } else {
+            await this.markFailed(domain.domainId, errorMessage);
+            logger.warn(
+                `Domain ${domain.baseDomain} failed periodic reverification: ${errorMessage}`
+            );
+        }
+    }
+
+    private async markFailed(
+        domainId: string,
+        errorMessage: string | null
+    ): Promise<void> {
+        await db
+            .update(domains)
+            .set({
+                verified: false,
+                failed: true,
+                // Three below MAX_TRIES: keeps the domain out of the DNS
+                // validator's immediate retry loop, while still leaving it
+                // eligible (tries < MAX_TRIES) for a few more validation
+                // passes instead of being excluded forever once tries hits
+                // MAX_TRIES.
+                tries: DNS_VALIDATOR_MAX_TRIES - 3,
+                lastCheckedAt: Math.floor(Date.now() / 1000),
+                errorMessage
+            })
+            .where(eq(domains.domainId, domainId));
+    }
+}
+
+export const domainReverifier = new DomainReverifier();

server/private/lib/certificates/index.ts

@@ -0,0 +1,45 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import logger from "@server/logger";
+import { privateConfig } from "#private/lib/config";
+import { acmeClientManager } from "./acme-client";
+import { jobScheduler } from "./scheduler";
+
+export async function startCertificateManager() {
+    const acmeConfig = privateConfig.getRawPrivateConfig().acme;
+    if (
+        acmeConfig &&
+        acmeConfig.cert_mode === "pangolin" &&
+        acmeConfig.enable_acme_client
+    ) {
+        logger.info("Starting certificate management server...");
+
+        // Initialize ACME client
+        await acmeClientManager.initialize();
+
+        // Start certificate issuance/renewal jobs
+        await jobScheduler.start();
+    }
+
+    if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+        // DNS record validation/reverification doesn't require certs, so it
+        // runs whenever Pangolin is acting as the authoritative DNS server,
+        // independent of the cert manager above.
+        await jobScheduler.startDnsJobs();
+    }
+}
+
+export async function stopCertificateManager() {
+    await jobScheduler.stop();
+}

server/private/lib/certificates/scheduler.ts

@@ -0,0 +1,197 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { withTimeout } from "@server/lib/retry";
+import logger from "@server/logger";
+import { certificateService } from "./certificate-service";
+import { privateConfig as config } from "#private/lib/config";
+import { dnsValidator } from "./dns-validator";
+import { domainReverifier } from "./domain-reverifier";
+import license from "#private/license/license";
+
+// Backstop for runExclusive: no single job's own internal timeouts (e.g.
+// certificate-service's per-cert issuance timeout) are relied on here. This
+// is the last line of defense - if *anything* inside a job hangs with no
+// error (a stalled Redis/DB call, a future code path that forgets to bound
+// itself, etc.), state.active must still reset so the next tick can run.
+// Without it, one hung run permanently skips every future tick for that job,
+// since runExclusive only clears state.active after the job promise settles.
+const RUN_EXCLUSIVE_TIMEOUT_MS = 30 * 60 * 1000;
+
+export class JobScheduler {
+    private certIntervals: NodeJS.Timeout[] = [];
+    private dnsIntervals: NodeJS.Timeout[] = [];
+    private certRunning = false;
+    private dnsRunning = false;
+
+    // Guards against a slow batch (e.g. 10 certs whose DNS challenges take a
+    // while) still being processed when the next interval tick fires -
+    // without this, overlapping ticks would each pull their own batch of up
+    // to 10 pending/renewal certs and process them concurrently instead of
+    // waiting for the prior batch to finish.
+    private runExclusive(
+        job: () => Promise<void>,
+        state: { active: boolean },
+        label: string
+    ): () => Promise<void> {
+        return async () => {
+            if (!(await license.isUnlocked())) {
+                logger.debug(
+                    `Skipping ${label} tick - license is not subscribed`
+                );
+                return;
+            }
+            if (state.active) {
+                logger.debug(
+                    `Skipping ${label} tick - previous run still in progress`
+                );
+                return;
+            }
+            state.active = true;
+            try {
+                await withTimeout(job(), RUN_EXCLUSIVE_TIMEOUT_MS, label);
+            } catch (error) {
+                logger.error(`Error in ${label}:`, error);
+            } finally {
+                state.active = false;
+            }
+        };
+    }
+
+    // Certificate issuance/renewal - requires an ACME client, so this is
+    // only started when Pangolin is actually managing certs.
+    async start(): Promise<void> {
+        if (this.certRunning) {
+            logger.warn("Certificate job scheduler is already running");
+            return;
+        }
+
+        this.certRunning = true;
+        logger.info("Starting certificate job scheduler");
+
+        const newCertState = { active: false };
+        const renewalState = { active: false };
+
+        const runNewCertCheck = this.runExclusive(
+            () => certificateService.processPendingCertificates(),
+            newCertState,
+            "processing pending certificates"
+        );
+        const runRenewalCheck = this.runExclusive(
+            () => certificateService.processRenewalCandidates(),
+            renewalState,
+            "processing renewal candidates"
+        );
+
+        // Schedule new certificate processing
+        const newCertInterval = setInterval(
+            runNewCertCheck,
+            config.getRawConfig().acme!.new_cert_check_interval_ms
+        );
+
+        // Schedule renewal processing (every 24 hours)
+        const renewalInterval = setInterval(
+            runRenewalCheck,
+            config.getRawConfig().acme!.renewal_check_interval_ms
+        );
+
+        this.certIntervals.push(newCertInterval, renewalInterval);
+
+        // Run initial checks
+        setTimeout(async () => {
+            try {
+                await runNewCertCheck();
+                // await runRenewalCheck();
+            } catch (error) {
+                logger.error("Error in initial certificate processing:", error);
+            }
+        }, 1000); // Wait 1 second after startup
+
+        logger.info("Certificate job scheduler started successfully");
+    }
+
+    // DNS record validation/reverification - doesn't touch certs at all, so
+    // this runs independently whenever Pangolin is acting as the
+    // authoritative DNS server, regardless of cert_mode.
+    async startDnsJobs(): Promise<void> {
+        if (this.dnsRunning) {
+            logger.warn("DNS validation job scheduler is already running");
+            return;
+        }
+
+        this.dnsRunning = true;
+        logger.info("Starting DNS validation job scheduler");
+
+        const dnsValidationState = { active: false };
+        const reverifyState = { active: false };
+
+        const runDnsValidation = this.runExclusive(
+            () => dnsValidator.validateAll(),
+            dnsValidationState,
+            "validating DNS records"
+        );
+        const runReverify = this.runExclusive(
+            () => domainReverifier.reverifyAll(),
+            reverifyState,
+            "reverifying domains"
+        );
+
+        // Schedule DNS validation
+        const dnsValidationInterval = setInterval(
+            runDnsValidation,
+            config.getRawConfig().acme?.dns_check_interval_ms ?? 60000
+        );
+
+        // Schedule periodic reverification of already-verified domains
+        const reverifyInterval = setInterval(
+            runReverify,
+            config.getRawConfig().acme?.domain_reverification_interval_ms ??
+                3600000
+        );
+
+        this.dnsIntervals.push(dnsValidationInterval, reverifyInterval);
+
+        // Run an initial validation pass shortly after startup
+        setTimeout(async () => {
+            try {
+                await runDnsValidation();
+            } catch (error) {
+                logger.error("Error in initial DNS validation:", error);
+            }
+        }, 1000);
+
+        logger.info("DNS validation job scheduler started successfully");
+    }
+
+    async stop(): Promise<void> {
+        if (this.certRunning) {
+            logger.info("Stopping certificate job scheduler");
+            this.certRunning = false;
+            this.certIntervals.forEach((interval) => clearInterval(interval));
+            this.certIntervals = [];
+        }
+
+        if (this.dnsRunning) {
+            logger.info("Stopping DNS validation job scheduler");
+            this.dnsRunning = false;
+            this.dnsIntervals.forEach((interval) => clearInterval(interval));
+            this.dnsIntervals = [];
+        }
+    }
+
+    isRunning(): boolean {
+        return this.certRunning || this.dnsRunning;
+    }
+}
+
+export const jobScheduler = new JobScheduler();

server/private/lib/config.ts

@@ -146,12 +146,20 @@ export class PrivateConfig {
             process.env.USE_PANGOLIN_DNS =
                 this.rawPrivateConfig.flags.use_pangolin_dns.toString();
         }
+
+        if (this.rawPrivateConfig.acme?.cert_mode) {
+            process.env.CERT_MODE = this.rawPrivateConfig.acme.cert_mode;
+        }
     }
 
     public getRawPrivateConfig() {
         return this.rawPrivateConfig;
     }
 
+    public getRawConfig() {
+        return this.getRawPrivateConfig();
+    }
+
     // `flags.enable_acme_cert_sync`, `flags.disable_private_http_placeholder`,
     // and `acme` used to live in the private config file. They now live in
     // the public config file. If an operator still has them set in the

server/private/lib/dns/generateDomains.ts

@@ -0,0 +1,47 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { build } from "@server/build";
+import privateConfig from "#private/lib/config";
+
+export function createCname(domainId: string, baseDomain: string) {
+    if (!privateConfig.getRawPrivateConfig().dns?.cname_extension) {
+        throw new Error("CNAME extension not configured");
+    }
+
+    let cnameRecords = [
+        {
+            value: `${domainId}.${privateConfig.getRawPrivateConfig().dns?.cname_extension}`,
+            baseDomain: baseDomain
+        },
+        {
+            value: `_acme-challenge.${domainId}.${privateConfig.getRawPrivateConfig().dns?.cname_extension}`,
+            baseDomain: `_acme-challenge.${baseDomain}`
+        }
+    ];
+
+    return cnameRecords;
+}
+
+export function createNs() {
+    if (!privateConfig.getRawPrivateConfig().dns?.nameserver_name) {
+        throw new Error("Nameservers not configured");
+    }
+
+    const nsRecords = [
+        privateConfig.getRawPrivateConfig().dns?.nameserver_name,
+        ...(privateConfig.getRawPrivateConfig().dns?.alternate_nameservers ||
+            [])
+    ] as string[];
+    return nsRecords;
+}

server/private/lib/dns/index.ts

@@ -0,0 +1,14 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+export * from "./server";

server/private/lib/exitNodes/exitNodeComms.ts

@@ -18,6 +18,7 @@ import { eq } from "drizzle-orm";
 import { sendToClient } from "#private/routers/ws";
 import privateConfig from "#private/lib/config";
 import config from "@server/lib/config";
+import { hasExitNodeCheckedIn } from "@server/lib/exitNodes";
 
 interface ExitNodeRequest {
     remoteType?: string;
@@ -138,13 +139,19 @@ export async function sendToExitNode(
 
             return response.data;
         } catch (error) {
-            if (axios.isAxiosError(error)) {
+            const message = axios.isAxiosError(error)
-                logger.error(
+                ? `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}`
-                    `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}`
+                : `Error making ${method} request for exit node at ${hostname}: ${error}`;
-                );
+
+            // The exit node (gerbil) may still be starting up and not yet
+            // reachable. Until it has checked in at least once, log this at a
+            // lower level since it's expected; once it has checked in, a
+            // connection failure is a real problem.
+            if (hasExitNodeCheckedIn(exitNode.exitNodeId)) {
+                logger.error(message);
             } else {
-                logger.error(
+                logger.warn(
-                    `Error making ${method} request for exit node at ${hostname}: ${error}`
+                    `${message} (exit node has not checked in yet since startup, this is expected briefly)`
                 );
             }
         }

server/private/lib/readConfigFile.ts

@@ -95,6 +95,70 @@ export const privateConfigSchema = z
                     .optional()
             })
             .optional(),
+        dns: z
+            .object({
+                enabled: z.boolean().optional().default(false),
+                listen_port: z.number().int().positive().optional().default(53),
+                nameserver_name: z.string(),
+                cname_extension: z.string(),
+                site_extension: z.string().optional(),
+                cname_alternate_extensions: z
+                    .array(z.string())
+                    .optional()
+                    .default([]),
+                alternate_nameservers: z
+                    .array(z.string())
+                    .optional()
+                    .default([]),
+                rate_limit: z
+                    .object({
+                        enabled: z.boolean().optional().default(true),
+                        window_ms: z
+                            .number()
+                            .int()
+                            .min(1000)
+                            .max(600000)
+                            .optional()
+                            .default(60000),
+                        max_requests: z
+                            .number()
+                            .int()
+                            .min(50)
+                            .max(100000)
+                            .optional()
+                            .default(1200),
+                        max_requests_per_query_type: z
+                            .number()
+                            .int()
+                            .min(10)
+                            .max(50000)
+                            .optional()
+                            .default(600)
+                    })
+                    .default({
+                        enabled: true,
+                        window_ms: 60000,
+                        max_requests: 1200,
+                        max_requests_per_query_type: 600
+                    }),
+                static_records: z
+                    .array(
+                        z.object({
+                            domain: z.string(),
+                            type: z.enum(["TXT", "CNAME", "A", "NS"]),
+                            value: z.string(),
+                            ttl: z
+                                .number()
+                                .int()
+                                .positive()
+                                .optional()
+                                .default(300)
+                        })
+                    )
+                    .optional()
+                    .default([])
+            })
+            .optional(),
         gerbil: z
             .object({
                 local_exit_node_reachable_at: z
@@ -125,15 +189,86 @@ export const privateConfigSchema = z
             })
             .optional()
             .prefault({}),
-        // @deprecated Moved to the public config file as `acme`
-        // (server/lib/readConfigFile.ts). Kept here only so existing private
-        // config files keep parsing; any value set here is migrated into the
-        // public config at startup by PrivateConfig (server/private/lib/config.ts).
         acme: z
             .object({
+                cert_mode: z
+                    .enum(["traefik", "pangolin"])
+                    .optional()
+                    .default("traefik"),
+                enable_acme_client: z.boolean().optional().default(false),
+                // @deprecated Moved to the public config file
+                // (server/lib/readConfigFile.ts). Kept here only so existing private
+                // config files keep parsing; any value set here is migrated into the
+                // public config at startup by PrivateConfig (server/private/lib/config.ts).
                 acme_json_path: z.string().optional(),
+                // @deprecated Moved to the public config file
+                // (server/lib/readConfigFile.ts). Kept here only so existing private
+                // config files keep parsing; any value set here is migrated into the
+                // public config at startup by PrivateConfig (server/private/lib/config.ts).
                 acme_http_endpoint: z.string().optional(),
-                sync_interval_ms: z.number().optional()
+                // @deprecated Moved to the public config file
+                // (server/lib/readConfigFile.ts). Kept here only so existing private
+                // config files keep parsing; any value set here is migrated into the
+                // public config at startup by PrivateConfig (server/private/lib/config.ts).
+                sync_interval_ms: z.number().optional(),
+                acme_directory_url: z
+                    .string()
+                    .url()
+                    .default("https://acme-v02.api.letsencrypt.org/directory"),
+                contact_email: z.string().email().optional(),
+                acme_account_key_path: z
+                    .string()
+                    .default("./config/account.key"),
+                challenge_ttl_ms: z.number().int().positive().default(300000),
+                renewal_check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(3600000),
+                new_cert_check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(5000),
+                // Kept safely under Let's Encrypt's ~20 req/s limit since this
+                // budget is shared across all pops workers and only covers the
+                // request-issuing calls we make directly (not every request
+                // acme-client makes internally, e.g. while polling for
+                // challenge/order status).
+                acme_requests_per_second: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(15),
+                dns_check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(60000),
+                domain_reverification_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(3600000), // 1 hour — how often to run the reverification pass
+                domain_reverification_window_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(259200000), // 72 hours — how old checkedAt must be before rechecking
+                domain_reverification_batch_size: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(20), // max domains to recheck per pass
+                dns_resolvers: z
+                    .array(z.string())
+                    .optional()
+                    .default([
+                        "8.8.8.8",
+                        "1.1.1.1",
+                        "9.9.9.9",
+                        "208.67.222.222"
+                    ])
             })
             .optional(),
         branding: z

server/private/lib/traefik/getTraefikConfig.ts

@@ -396,7 +396,7 @@ export async function getTraefikConfig(
         );
 
     let validCerts: CertificateResult[] = [];
-    if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+    if (privateConfig.getRawPrivateConfig().acme?.cert_mode == "pangolin") {
         // create a list of all domains to get certs for
         const domains = new Set<string>();
         for (const resource of resourcesMap.values()) {
@@ -522,7 +522,10 @@ export async function getTraefikConfig(
             );
 
             let tls = {};
-            if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+            if (
+                privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                "pangolin"
+            ) {
                 tls = buildWildcardTls({
                     fullDomain,
                     hasSubdomain: !!resource.subdomain,
@@ -789,7 +792,8 @@ export async function getTraefikConfig(
                 preferWildcardCert
             }) => {
                 if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
+                    privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                    "pangolin"
                 ) {
                     return buildWildcardTls({
                         fullDomain,
@@ -832,7 +836,8 @@ export async function getTraefikConfig(
             redirectHttpsMiddlewareName,
             resolveTls: (fullDomain) => {
                 if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
+                    privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                    "pangolin"
                 ) {
                     // siteResource aliases don't have a per-domain cert
                     // resolver stored, so always fall back to the global
@@ -924,7 +929,10 @@ export async function getTraefikConfig(
             const rule = buildHostRule(fullDomain, ir.wildcard);
 
             let tls: any = {};
-            if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+            if (
+                privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                "pangolin"
+            ) {
                 tls = buildWildcardTls({
                     fullDomain,
                     hasSubdomain: !!ir.subdomain,
@@ -1005,7 +1013,8 @@ export async function getTraefikConfig(
 
                 let tls: any = {};
                 if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
+                    privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                    "pangolin"
                 ) {
                     // siteResource aliases don't have a per-domain cert
                     // resolver stored, so always fall back to the global
@@ -1080,7 +1089,7 @@ export async function getTraefikConfig(
             .where(eq(exitNodes.exitNodeId, exitNodeId));
 
         let validCertsLoginPages: CertificateResult[] = [];
-        if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+        if (privateConfig.getRawPrivateConfig().acme?.cert_mode == "pangolin") {
             // create a list of all domains to get certs for
             const domains = new Set<string>();
             for (const lp of exitNodeLoginPages) {
@@ -1126,7 +1135,8 @@ export async function getTraefikConfig(
 
                 const tls = {};
                 if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
+                    privateConfig.getRawPrivateConfig().acme?.cert_mode !=
+                    "pangolin"
                 ) {
                     // TODO: we need to add the wildcard logic here too
                 } else {

server/private/license/license.ts

@@ -14,7 +14,7 @@
 import { db, HostMeta, sites, users } from "@server/db";
 import { hostMeta, licenseKey } from "@server/db";
 import logger from "@server/logger";
-import NodeCache from "node-cache";
+import { createLocalCache } from "@server/lib/createLocalCache";
 import { validateJWT } from "./licenseJwt";
 import { count, eq } from "drizzle-orm";
 import moment from "moment";
@@ -65,8 +65,8 @@ export class License {
     private validationServerUrl = `${this.serverBaseUrl}/api/v1/license/enterprise/validate`;
     private activationServerUrl = `${this.serverBaseUrl}/api/v1/license/enterprise/activate`;
 
-    private statusCache = new NodeCache();
+    private statusCache = createLocalCache();
-    private licenseKeyCache = new NodeCache();
+    private licenseKeyCache = createLocalCache();
 
     private statusKey = "status";
     private serverSecret!: string;
@@ -179,7 +179,7 @@ LQIDAQAB
                 status.isHostLicensed = false;
                 // Invalidate all and set new cache (empty)
                 this.licenseKeyCache.flushAll();
-                this.statusCache.set(this.statusKey, status);
+                this.statusCache.set(this.statusKey, status, 0);
                 return status;
             }
 
@@ -389,7 +389,7 @@ LQIDAQAB
             // Invalidate old cache and set new cache
             this.licenseKeyCache.flushAll();
             for (const [key, value] of newCache.entries()) {
-                this.licenseKeyCache.set<LicenseKeyCache>(key, value);
+                this.licenseKeyCache.set(key, value, 0);
             }
         } catch (error) {
             logger.error("Error checking license status:");
@@ -398,7 +398,7 @@ LQIDAQAB
             this.checkInProgress = false;
         }
 
-        this.statusCache.set(this.statusKey, status);
+        this.statusCache.set(this.statusKey, status, 0);
         return status;
     }
 

server/private/routers/alertRule/testAlertRule.ts

@@ -14,7 +14,7 @@
 import { getRandomItemInArray } from "@app/lib/getRandomItemInArray";
 import response from "@server/lib/response";
 import logger from "@server/logger";
-import { processTestAlerts } from "@server/private/lib/alerts/processTestAlerts";
+import { processTestAlerts } from "#private/lib/alerts/processTestAlerts";
 import { type AlertAction } from "@server/routers/alertRule/types";
 import HttpCode from "@server/types/HttpCode";
 import { NextFunction, Request, Response } from "express";

server/private/routers/alertRule/updateAlertRule.ts

@@ -33,8 +33,11 @@ import { OpenAPITags, registry } from "@server/openApi";
 import { and, eq } from "drizzle-orm";
 import { encrypt } from "@server/lib/crypto";
 import config from "@server/lib/config";
-import { HC_EVENT_TYPES, SITE_EVENT_TYPES, RESOURCE_EVENT_TYPES } from "./createAlertRule";
+import {
-import { invalidateAllRemoteExitNodeSessions } from "@server/private/auth/sessions/remoteExitNode";
+    HC_EVENT_TYPES,
+    SITE_EVENT_TYPES,
+    RESOURCE_EVENT_TYPES
+} from "./createAlertRule";
 
 const paramsSchema = z
     .object({
@@ -85,35 +88,57 @@ const bodySchema = z
         const isHcEvent = (HC_EVENT_TYPES as readonly string[]).includes(
             val.eventType
         );
-        const isResourceEvent = (RESOURCE_EVENT_TYPES as readonly string[]).includes(
+        const isResourceEvent = (
-            val.eventType
+            RESOURCE_EVENT_TYPES as readonly string[]
-        );
+        ).includes(val.eventType);
 
-        if (isSiteEvent && val.siteIds !== undefined && val.siteIds.length === 0 && !val.allSites) {
+        if (
+            isSiteEvent &&
+            val.siteIds !== undefined &&
+            val.siteIds.length === 0 &&
+            !val.allSites
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one siteId is required for site event types when allSites is false",
+                message:
+                    "At least one siteId is required for site event types when allSites is false",
                 path: ["siteIds"]
             });
         }
 
-        if (isHcEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length === 0 && !val.allHealthChecks) {
+        if (
+            isHcEvent &&
+            val.healthCheckIds !== undefined &&
+            val.healthCheckIds.length === 0 &&
+            !val.allHealthChecks
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one healthCheckId is required for health check event types when allHealthChecks is false",
+                message:
+                    "At least one healthCheckId is required for health check event types when allHealthChecks is false",
                 path: ["healthCheckIds"]
             });
         }
 
-        if (isResourceEvent && val.resourceIds !== undefined && val.resourceIds.length === 0 && !val.allResources) {
+        if (
+            isResourceEvent &&
+            val.resourceIds !== undefined &&
+            val.resourceIds.length === 0 &&
+            !val.allResources
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one resourceId is required for resource event types when allResources is false",
+                message:
+                    "At least one resourceId is required for resource event types when allResources is false",
                 path: ["resourceIds"]
             });
         }
 
-        if (isSiteEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) {
+        if (
+            isSiteEvent &&
+            val.healthCheckIds !== undefined &&
+            val.healthCheckIds.length > 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
                 message: "healthCheckIds must not be set for site event types",
@@ -129,7 +154,11 @@ const bodySchema = z
             });
         }
 
-        if (isResourceEvent && val.siteIds !== undefined && val.siteIds.length > 0) {
+        if (
+            isResourceEvent &&
+            val.siteIds !== undefined &&
+            val.siteIds.length > 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
                 message: "siteIds must not be set for resource event types",
@@ -137,10 +166,15 @@ const bodySchema = z
             });
         }
 
-        if (isResourceEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) {
+        if (
+            isResourceEvent &&
+            val.healthCheckIds !== undefined &&
+            val.healthCheckIds.length > 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "healthCheckIds must not be set for resource event types",
+                message:
+                    "healthCheckIds must not be set for resource event types",
                 path: ["healthCheckIds"]
             });
         }
@@ -153,7 +187,6 @@ const UpdateAlertRuleResponseDataSchema = z.object({
     alertRuleId: z.number()
 });
 
-
 registry.registerPath({
     method: "post",
     path: "/org/{orgId}/alert-rule/{alertRuleId}",
@@ -174,7 +207,9 @@ registry.registerPath({
             description: "Successful response",
             content: {
                 "application/json": {
-                    schema: createApiResponseSchema(UpdateAlertRuleResponseDataSchema)
+                    schema: createApiResponseSchema(
+                        UpdateAlertRuleResponseDataSchema
+                    )
                 }
             }
         }
@@ -250,9 +285,11 @@ export async function updateAlertRule(
         if (name !== undefined) updateData.name = name;
         if (eventType !== undefined) updateData.eventType = eventType;
         if (enabled !== undefined) updateData.enabled = enabled;
-        if (cooldownSeconds !== undefined) updateData.cooldownSeconds = cooldownSeconds;
+        if (cooldownSeconds !== undefined)
+            updateData.cooldownSeconds = cooldownSeconds;
         if (allSites !== undefined) updateData.allSites = allSites;
-        if (allHealthChecks !== undefined) updateData.allHealthChecks = allHealthChecks;
+        if (allHealthChecks !== undefined)
+            updateData.allHealthChecks = allHealthChecks;
         if (allResources !== undefined) updateData.allResources = allResources;
 
         await db
@@ -273,7 +310,11 @@ export async function updateAlertRule(
 
             // Only insert junction rows when allSites is not true
             const effectiveAllSites = allSites ?? false;
-            if (!effectiveAllSites && siteIds !== undefined && siteIds.length > 0) {
+            if (
+                !effectiveAllSites &&
+                siteIds !== undefined &&
+                siteIds.length > 0
+            ) {
                 await db.insert(alertSites).values(
                     siteIds.map((siteId) => ({
                         alertRuleId,
@@ -290,7 +331,11 @@ export async function updateAlertRule(
                 .where(eq(alertHealthChecks.alertRuleId, alertRuleId));
 
             const effectiveAllHealthChecks = allHealthChecks ?? false;
-            if (!effectiveAllHealthChecks && healthCheckIds !== undefined && healthCheckIds.length > 0) {
+            if (
+                !effectiveAllHealthChecks &&
+                healthCheckIds !== undefined &&
+                healthCheckIds.length > 0
+            ) {
                 await db.insert(alertHealthChecks).values(
                     healthCheckIds.map((healthCheckId) => ({
                         alertRuleId,
@@ -307,7 +352,11 @@ export async function updateAlertRule(
                 .where(eq(alertResources.alertRuleId, alertRuleId));
 
             const effectiveAllResources = allResources ?? false;
-            if (!effectiveAllResources && resourceIds !== undefined && resourceIds.length > 0) {
+            if (
+                !effectiveAllResources &&
+                resourceIds !== undefined &&
+                resourceIds.length > 0
+            ) {
                 await db.insert(alertResources).values(
                     resourceIds.map((resourceId) => ({
                         alertRuleId,
@@ -392,7 +441,10 @@ export async function updateAlertRule(
                     webhookActions.map((wa) => ({
                         alertRuleId,
                         webhookUrl: wa.webhookUrl,
-                        config: wa.config != null ? encrypt(wa.config, serverSecret) : null,
+                        config:
+                            wa.config != null
+                                ? encrypt(wa.config, serverSecret)
+                                : null,
                         enabled: wa.enabled
                     }))
                 );

server/private/routers/generatedLicense/clearInstanceName.ts

@@ -31,7 +31,9 @@ export async function clearInstanceName(
     next: NextFunction
 ): Promise<any> {
     try {
-        const parsedParams = clearInstanceNameParamsSchema.safeParse(req.params);
+        const parsedParams = clearInstanceNameParamsSchema.safeParse(
+            req.params
+        );
         if (!parsedParams.success) {
             return next(
                 createHttpError(
@@ -63,7 +65,8 @@ export async function clearInstanceName(
             return next(
                 createHttpError(
                     data.status || HttpCode.BAD_REQUEST,
-                    data.message || "Failed to clear instance name from Fossorial API"
+                    data.message ||
+                        "Failed to clear server ID from Fossorial API"
                 )
             );
         }
@@ -72,7 +75,7 @@ export async function clearInstanceName(
             data: null,
             success: true,
             error: false,
-            message: "Instance name cleared successfully",
+            message: "Server ID cleared successfully",
             status: HttpCode.OK
         });
     } catch (error) {
@@ -80,8 +83,8 @@ export async function clearInstanceName(
         return next(
             createHttpError(
                 HttpCode.INTERNAL_SERVER_ERROR,
-                "An error occurred while clearing the instance name."
+                "An error occurred while clearing the server ID."
             )
         );
     }
-}
+}

server/private/routers/gerbil/createExitNode.ts

@@ -12,8 +12,8 @@
  */
 
 import { db, ExitNode, exitNodes } from "@server/db";
-import { getUniqueExitNodeEndpointName } from "@server/db/names";
 import config from "@server/lib/config";
+import privateConfig from "#private/lib/config";
 import { getNextAvailableSubnet } from "@server/lib/exitNodes";
 import logger from "@server/logger";
 import { eq } from "drizzle-orm";
@@ -45,6 +45,8 @@ export async function createExitNode(
                 .values({
                     publicKey,
                     endpoint: config.getRawConfig().gerbil.base_endpoint,
+                    region:
+                        privateConfig.getRawPrivateConfig().app.region || null,
                     address,
                     listenPort,
                     online: true,

server/private/routers/hybrid.ts

@@ -2478,7 +2478,12 @@ hybridRouter.post(
                 destinations: destinations
             });
         } catch (error) {
-            logger.error(error);
+            if (!(
+                error instanceof Error &&
+                error.message === "Exit node not allowed"
+            )) {
+                logger.error(error);
+            }
             return next(
                 createHttpError(
                     HttpCode.INTERNAL_SERVER_ERROR,

server/private/routers/remoteExitNode/exitNodeEvents.ts

@@ -0,0 +1,23 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { EventEmitter } from "events";
+
+export interface ExitNodeOnlineEvent {
+    exitNodeId: number;
+    endpoint: string;
+}
+
+export const EXIT_NODE_ONLINE_EVENT = "exit-node-online";
+
+export const exitNodeEvents = new EventEmitter();

server/private/routers/remoteExitNode/exitNodeReconnectScheduler.ts

@@ -12,11 +12,27 @@
  */
 
 import axios from "axios";
-import { db, exitNodes, newts, sites } from "@server/db";
+import { db, newts, sites } from "@server/db";
 import { eq } from "drizzle-orm";
 import logger from "@server/logger";
 import redisManager from "#private/lib/redis";
-// import { sendToClient } from "#private/routers/ws";
+import { sendToClient } from "../ws";
+import {
+    exitNodeEvents,
+    EXIT_NODE_ONLINE_EVENT,
+    ExitNodeOnlineEvent
+} from "./exitNodeEvents";
+
+exitNodeEvents.on(
+    EXIT_NODE_ONLINE_EVENT,
+    ({ exitNodeId, endpoint }: ExitNodeOnlineEvent) => {
+        scheduleExitNodeReconnect(exitNodeId, endpoint).catch((error) => {
+            logger.error("Failed to schedule exit node reconnect", {
+                error
+            });
+        });
+    }
+);
 
 const INITIAL_DELAY_MS = 15 * 1000; // 15 seconds before first check
 const CHECK_INTERVAL_MS = 10 * 1000; // Check every 10 seconds
@@ -26,7 +42,7 @@ const REDIS_HASH_PREFIX = "exit-node-reconnect:";
 
 interface PendingReconnect {
     startTime: number;
-    reachableAt: string;
+    endpoint: string;
 }
 
 // In-memory tracking for this node
@@ -40,15 +56,15 @@ let schedulerInterval: NodeJS.Timeout | null = null;
  */
 export async function scheduleExitNodeReconnect(
     exitNodeId: number,
-    reachableAt: string
+    endpoint: string
 ): Promise<void> {
     logger.info(
-        `Scheduling newt reconnect for exit node ${exitNodeId} (reachableAt: ${reachableAt})`
+        `Scheduling newt reconnect for exit node ${exitNodeId} (endpoint: ${endpoint})`
     );
 
     const entry: PendingReconnect = {
         startTime: Date.now(),
-        reachableAt
+        endpoint
     };
 
     pendingReconnects.set(exitNodeId, entry);
@@ -63,8 +79,8 @@ export async function scheduleExitNodeReconnect(
         );
         await redisManager.hset(
             `${REDIS_HASH_PREFIX}${exitNodeId}`,
-            "reachableAt",
+            "endpoint",
-            reachableAt
+            endpoint
         );
     }
 }
@@ -101,14 +117,14 @@ async function processPendingReconnects(): Promise<void> {
                     `${REDIS_HASH_PREFIX}${id}`,
                     "startTime"
                 );
-                const reachableAt = await redisManager.hget(
+                const endpoint = await redisManager.hget(
                     `${REDIS_HASH_PREFIX}${id}`,
-                    "reachableAt"
+                    "endpoint"
                 );
-                if (startTimeStr && reachableAt) {
+                if (startTimeStr && endpoint) {
                     toProcess.set(id, {
                         startTime: parseInt(startTimeStr, 10),
-                        reachableAt
+                        endpoint
                     });
                 }
             }
@@ -135,7 +151,7 @@ async function processPendingReconnects(): Promise<void> {
         }
 
         // Check if the exit node HTTP endpoint is reachable
-        const pingUrl = `${entry.reachableAt}/ping`;
+        const pingUrl = `http://${entry.endpoint}/ping`;
         try {
             await axios.get(pingUrl, { timeout: 5000 });
         } catch {
@@ -150,47 +166,47 @@ async function processPendingReconnects(): Promise<void> {
             `Exit node ${exitNodeId} is reachable. Sending newt/wg/reconnect to connected newts.`
         );
 
-        // await sendReconnectToNewts(exitNodeId);
+        await sendReconnectToNewts(exitNodeId);
         await removePending(exitNodeId);
     }
 }
 
-// async function sendReconnectToNewts(exitNodeId: number): Promise<void> {
+async function sendReconnectToNewts(exitNodeId: number): Promise<void> {
-// try {
+    try {
-//         const connectedNewts = await db
+        const connectedNewts = await db
-//             .select({ newtId: newts.newtId })
+            .select({ newtId: newts.newtId })
-//             .from(newts)
+            .from(newts)
-//             .innerJoin(sites, eq(newts.siteId, sites.siteId))
+            .innerJoin(sites, eq(newts.siteId, sites.siteId))
-//             .where(eq(sites.exitNodeId, exitNodeId));
+            .where(eq(sites.exitNodeId, exitNodeId));
 
-//         if (connectedNewts.length === 0) {
+        if (connectedNewts.length === 0) {
-//             logger.debug(
+            logger.debug(
-//                 `No newts found for exit node ${exitNodeId}, nothing to reconnect`
+                `No newts found for exit node ${exitNodeId}, nothing to reconnect`
-//             );
+            );
-//             return;
+            return;
-//         }
+        }
 
-//         logger.info(
+        logger.info(
-//             `Sending newt/wg/reconnect to ${connectedNewts.length} newt(s) for exit node ${exitNodeId}`
+            `Sending newt/wg/reconnect to ${connectedNewts.length} newt(s) for exit node ${exitNodeId}`
-//         );
+        );
 
-//         const reconnectMessage = {
+        const reconnectMessage = {
-//             type: "newt/wg/reconnect",
+            type: "newt/wg/reconnect",
-//             data: {}
+            data: {}
-//         };
+        };
 
-//         await Promise.allSettled(
+        await Promise.allSettled(
-//             connectedNewts.map(({ newtId }) =>
+            connectedNewts.map(({ newtId }) =>
-//                 sendToClient(newtId, reconnectMessage)
+                sendToClient(newtId, reconnectMessage)
-//             )
+            )
-//         );
+        );
-//     } catch (error) {
+    } catch (error) {
-//         logger.error(
+        logger.error(
-//             `Failed to send reconnect messages for exit node ${exitNodeId}`,
+            `Failed to send reconnect messages for exit node ${exitNodeId}`,
-//             { error }
+            { error }
-//         );
+        );
-//     }
+    }
-// }
+}
 
 async function removePending(exitNodeId: number): Promise<void> {
     pendingReconnects.delete(exitNodeId);

server/private/routers/remoteExitNode/getRemoteExitNodeToken.ts

@@ -26,7 +26,7 @@ import {
     validateRemoteExitNodeSessionToken,
     EXPIRES
 } from "#private/auth/sessions/remoteExitNode";
-import { getOrCreateCachedToken } from "@server/private/lib/tokenCache";
+import { getOrCreateCachedToken } from "#private/lib/tokenCache";
 import { verifyPassword } from "@server/auth/password";
 import logger from "@server/logger";
 import config from "@server/lib/config";

server/private/routers/remoteExitNode/handleRemoteExitNodePingMessage.ts

@@ -16,7 +16,7 @@ import { MessageHandler } from "@server/routers/ws";
 import { RemoteExitNode } from "@server/db";
 import { eq } from "drizzle-orm";
 import logger from "@server/logger";
-import { scheduleExitNodeReconnect } from "./exitNodeReconnectScheduler";
+import { exitNodeEvents, EXIT_NODE_ONLINE_EVENT } from "./exitNodeEvents";
 
 /**
  * Handles ping messages from clients and responds with pong
@@ -40,7 +40,7 @@ export const handleRemoteExitNodePingMessage: MessageHandler = async (
     try {
         // Fetch the current state before updating so we can detect the offline→online transition
         const [currentExitNode] = await db
-            .select({ online: exitNodes.online, reachableAt: exitNodes.reachableAt })
+            .select({ online: exitNodes.online, endpoint: exitNodes.endpoint })
             .from(exitNodes)
             .where(eq(exitNodes.exitNodeId, remoteExitNode.exitNodeId))
             .limit(1);
@@ -55,12 +55,14 @@ export const handleRemoteExitNodePingMessage: MessageHandler = async (
             .where(eq(exitNodes.exitNodeId, remoteExitNode.exitNodeId));
 
         // If the exit node was offline and is now coming online, schedule newt reconnects
-        if (currentExitNode && !currentExitNode.online && currentExitNode.reachableAt) {
+        if (
-            scheduleExitNodeReconnect(
+            currentExitNode &&
-                remoteExitNode.exitNodeId,
+            !currentExitNode.online &&
-                currentExitNode.reachableAt
+            currentExitNode.endpoint
-            ).catch((error) => {
+        ) {
-                logger.error("Failed to schedule exit node reconnect", { error });
+            exitNodeEvents.emit(EXIT_NODE_ONLINE_EVENT, {
+                exitNodeId: remoteExitNode.exitNodeId,
+                endpoint: currentExitNode.endpoint
             });
         }
     } catch (error) {

server/routers/badger/verifySession.ts

@@ -691,7 +691,9 @@ export async function verifyResourceSession(
                 );
 
                 resourceSession = result?.resourceSession;
-                localCache.set(sessionCacheKey, resourceSession, 5);
+                if (resourceSession) {
+                    localCache.set(sessionCacheKey, resourceSession, 5);
+                }
             }
 
             if (resourceSession?.isRequestToken) {
@@ -1121,7 +1123,9 @@ async function allowAccessToken(
                 resource.resourceId
             );
             resourceSession = result?.resourceSession;
-            localCache.set(sessionCacheKey, resourceSession, 5);
+            if (resourceSession) {
+                localCache.set(sessionCacheKey, resourceSession, 5);
+            }
         }
 
         if (

server/routers/domain/createOrgDomain.ts

@@ -21,6 +21,7 @@ import { LimitId } from "@server/lib/billing";
 import { isSecondLevelDomain, isValidDomain } from "@server/lib/validators";
 import { build } from "@server/build";
 import config from "@server/lib/config";
+import { createNs, createCname } from "#dynamic/lib/dns/generateDomains";
 
 const paramsSchema = z.strictObject({
     orgId: z.string()
@@ -283,8 +284,7 @@ export async function createOrgDomain(
 
             // TODO: This needs to be cross region and not hardcoded
             if (type === "ns") {
-                nsRecords = config.getRawConfig().dns.nameservers as string[];
+                nsRecords = createNs();
-
                 // Save NS records to database
                 for (const nsValue of nsRecords) {
                     recordsToInsert.push({
@@ -296,16 +296,7 @@ export async function createOrgDomain(
                     });
                 }
             } else if (type === "cname") {
-                cnameRecords = [
+                cnameRecords = createCname(domainId, baseDomain);
-                    {
-                        value: `${domainId}.${config.getRawConfig().dns.cname_extension}`,
-                        baseDomain: baseDomain
-                    },
-                    {
-                        value: `_acme-challenge.${domainId}.${config.getRawConfig().dns.cname_extension}`,
-                        baseDomain: `_acme-challenge.${baseDomain}`
-                    }
-                ];
 
                 // Save CNAME records to database
                 for (const cnameRecord of cnameRecords) {

server/routers/external.ts

@@ -87,6 +87,12 @@ authenticated.get("/org/checkId", org.checkId);
 authenticated.put("/org", getUserOrgs, org.createOrg);
 
 authenticated.get("/orgs", verifyUserIsServerAdmin, org.listOrgs);
+authenticated.get("/admin/orgs", verifyUserIsServerAdmin, org.adminListOrgs);
+authenticated.delete(
+    "/admin/org/:orgId",
+    verifyUserIsServerAdmin,
+    org.adminDeleteOrg
+);
 authenticated.get("/user/:userId/orgs", verifyIsLoggedInUser, org.listUserOrgs);
 
 authenticated.get(
@@ -1378,6 +1384,12 @@ if (build !== "saas") {
         user.adminGeneratePasswordResetCode
     );
 
+    authenticated.post(
+        "/user/:userId/server-admin",
+        verifyUserIsServerAdmin,
+        user.adminSetServerAdmin
+    );
+
     authenticated.delete(
         "/user/:userId",
         verifyUserIsServerAdmin,

server/routers/gerbil/getConfig.ts

@@ -10,6 +10,7 @@ import config from "@server/lib/config";
 import { fromError } from "zod-validation-error";
 import { getAllowedIps } from "../target/helpers";
 import { createExitNode } from "#dynamic/routers/gerbil/createExitNode";
+import { markExitNodeCheckedIn } from "@server/lib/exitNodes";
 
 // Define Zod schema for request validation
 const getConfigSchema = z.object({
@@ -65,6 +66,8 @@ export async function getConfig(
             );
         }
 
+        markExitNodeCheckedIn(exitNode.exitNodeId);
+
         const configResponse = await generateGerbilConfig(exitNode);
 
         logger.debug("Sending config: ", configResponse);

server/routers/launcher/launcherResourceAccess.ts

@@ -522,6 +522,7 @@ async function fetchLabelsForResources(
 type SiteGroupRow = {
     siteId: number;
     name: string;
+    niceId: string;
     type: string;
     online: boolean;
     itemCount: number;
@@ -556,6 +557,7 @@ async function listSiteGroups(
             .select({
                 siteId: sites.siteId,
                 name: sites.name,
+                niceId: sites.niceId,
                 type: sites.type,
                 online: sites.online,
                 itemCount: countDistinct(resources.resourceId)
@@ -576,7 +578,13 @@ async function listSiteGroups(
 
         const publicRows = await publicQuery
             .where(and(...publicConditions))
-            .groupBy(sites.siteId, sites.name, sites.type, sites.online);
+            .groupBy(
+                sites.siteId,
+                sites.name,
+                sites.niceId,
+                sites.type,
+                sites.online
+            );
 
         for (const row of publicRows) {
             const existing = siteCountMap.get(row.siteId);
@@ -586,6 +594,7 @@ async function listSiteGroups(
                 siteCountMap.set(row.siteId, {
                     siteId: row.siteId,
                     name: row.name,
+                    niceId: row.niceId,
                     type: row.type,
                     online: row.online,
                     itemCount: Number(row.itemCount)
@@ -612,6 +621,7 @@ async function listSiteGroups(
             .select({
                 siteId: sites.siteId,
                 name: sites.name,
+                niceId: sites.niceId,
                 type: sites.type,
                 online: sites.online,
                 itemCount: countDistinct(siteResources.siteResourceId)
@@ -638,7 +648,13 @@ async function listSiteGroups(
 
         const siteRows = await siteResourceQuery
             .where(and(...siteConditions))
-            .groupBy(sites.siteId, sites.name, sites.type, sites.online);
+            .groupBy(
+                sites.siteId,
+                sites.name,
+                sites.niceId,
+                sites.type,
+                sites.online
+            );
 
         for (const row of siteRows) {
             const existing = siteCountMap.get(row.siteId);
@@ -648,6 +664,7 @@ async function listSiteGroups(
                 siteCountMap.set(row.siteId, {
                     siteId: row.siteId,
                     name: row.name,
+                    niceId: row.niceId,
                     type: row.type,
                     online: row.online,
                     itemCount: Number(row.itemCount)
@@ -1061,6 +1078,43 @@ export async function listLauncherGroupsForUser(
     };
 }
 
+function toLauncherSiteInfo(row: {
+    siteId: number | null;
+    siteName: string | null;
+    siteNiceId: string | null;
+    siteType: string | null;
+    siteOnline: boolean | null;
+}): LauncherSiteInfo | null {
+    if (
+        row.siteId == null ||
+        row.siteName == null ||
+        row.siteNiceId == null ||
+        row.siteType == null
+    ) {
+        return null;
+    }
+    return {
+        siteId: row.siteId,
+        name: row.siteName,
+        niceId: row.siteNiceId,
+        type: row.siteType,
+        online: row.siteOnline ?? undefined
+    };
+}
+
+function pickPrimarySite(
+    sites: LauncherSiteInfo[],
+    siteIdFilter?: number
+): LauncherSiteInfo | undefined {
+    if (sites.length === 0) {
+        return undefined;
+    }
+    if (siteIdFilter != null) {
+        return sites.find((site) => site.siteId === siteIdFilter) ?? sites[0];
+    }
+    return sites[0];
+}
+
 async function mapPublicResources(
     orgId: string,
     resourceIds: number[],
@@ -1084,6 +1138,7 @@ async function mapPublicResources(
             enabled: resources.enabled,
             siteId: sites.siteId,
             siteName: sites.name,
+            siteNiceId: sites.niceId,
             siteType: sites.type,
             siteOnline: sites.online,
             exitNodeEndpoint: exitNodes.endpoint
@@ -1097,56 +1152,65 @@ async function mapPublicResources(
                 inArray(resources.resourceId, resourceIds),
                 eq(resources.orgId, orgId),
                 eq(resources.enabled, true),
-                eq(resources.status, "approved"),
+                eq(resources.status, "approved")
-                siteIdFilter != null
-                    ? eq(sites.siteId, siteIdFilter)
-                    : undefined
             )
         );
 
-    const seen = new Set<string>();
+    const byKey = new Map<string, LauncherResource>();
-    const result: LauncherResource[] = [];
+    const siteIdsByKey = new Map<string, Set<number>>();
 
     for (const row of rows) {
         const key = `public:${row.resourceId}`;
-        if (seen.has(key)) {
+        let item = byKey.get(key);
+
+        if (!item) {
+            const access = formatPublicResourceAccess({
+                mode: row.mode,
+                fullDomain: row.fullDomain,
+                ssl: row.ssl,
+                proxyPort: row.proxyPort,
+                wildcard: row.wildcard,
+                exitNodeEndpoint: row.exitNodeEndpoint
+            });
+
+            item = {
+                launcherResourceKey: key,
+                resourceType: "public",
+                resourceId: row.resourceId,
+                niceId: row.niceId,
+                name: row.name,
+                ...access,
+                iconUrl: null,
+                enabled: row.enabled,
+                mode: row.mode,
+                labels: labelMaps.byResourceId.get(row.resourceId) ?? [],
+                sites: []
+            };
+            byKey.set(key, item);
+            siteIdsByKey.set(key, new Set());
+        }
+
+        const site = toLauncherSiteInfo(row);
+        if (!site) {
             continue;
         }
-        seen.add(key);
 
-        const access = formatPublicResourceAccess({
+        const seenSiteIds = siteIdsByKey.get(key)!;
-            mode: row.mode,
+        if (seenSiteIds.has(site.siteId)) {
-            fullDomain: row.fullDomain,
+            continue;
-            ssl: row.ssl,
+        }
-            proxyPort: row.proxyPort,
+        seenSiteIds.add(site.siteId);
-            wildcard: row.wildcard,
+        item.sites.push(site);
-            exitNodeEndpoint: row.exitNodeEndpoint
-        });
-
-        result.push({
-            launcherResourceKey: key,
-            resourceType: "public",
-            resourceId: row.resourceId,
-            niceId: row.niceId,
-            name: row.name,
-            ...access,
-            iconUrl: null,
-            enabled: row.enabled,
-            mode: row.mode,
-            labels: labelMaps.byResourceId.get(row.resourceId) ?? [],
-            site:
-                row.siteId != null
-                    ? {
-                          siteId: row.siteId,
-                          name: row.siteName!,
-                          type: row.siteType!,
-                          online: row.siteOnline ?? undefined
-                      }
-                    : undefined
-        });
     }
 
-    return result;
+    for (const item of byKey.values()) {
+        item.sites.sort((a, b) =>
+            a.name.localeCompare(b.name, undefined, { sensitivity: "base" })
+        );
+        item.site = pickPrimarySite(item.sites, siteIdFilter);
+    }
+
+    return Array.from(byKey.values());
 }
 
 async function mapSiteResources(
@@ -1175,6 +1239,7 @@ async function mapSiteResources(
             enabled: siteResources.enabled,
             siteId: sites.siteId,
             siteName: sites.name,
+            siteNiceId: sites.niceId,
             siteType: sites.type,
             siteOnline: sites.online
         })
@@ -1189,59 +1254,69 @@ async function mapSiteResources(
                 inArray(siteResources.siteResourceId, siteResourceIds),
                 eq(siteResources.orgId, orgId),
                 eq(siteResources.enabled, true),
-                eq(siteResources.status, "approved"),
+                eq(siteResources.status, "approved")
-                siteIdFilter != null
-                    ? eq(sites.siteId, siteIdFilter)
-                    : undefined
             )
         );
 
-    const seen = new Set<string>();
+    const byKey = new Map<string, LauncherResource>();
-    const result: LauncherResource[] = [];
+    const siteIdsByKey = new Map<string, Set<number>>();
 
     for (const row of rows) {
         const key = `site:${row.siteResourceId}`;
-        if (seen.has(key)) {
+        let item = byKey.get(key);
+
+        if (!item) {
+            const access = formatSiteResourceAccess({
+                mode: row.mode,
+                destination: row.destination,
+                destinationPort: row.destinationPort,
+                scheme: row.scheme,
+                ssl: row.ssl,
+                fullDomain: row.fullDomain,
+                alias: row.alias,
+                aliasAddress: row.aliasAddress
+            });
+
+            item = {
+                launcherResourceKey: key,
+                resourceType: "site",
+                resourceId: row.siteResourceId,
+                siteResourceId: row.siteResourceId,
+                niceId: row.niceId,
+                name: row.name,
+                ...access,
+                iconUrl: null,
+                enabled: row.enabled,
+                mode: row.mode,
+                labels:
+                    labelMaps.bySiteResourceId.get(row.siteResourceId) ?? [],
+                sites: []
+            };
+            byKey.set(key, item);
+            siteIdsByKey.set(key, new Set());
+        }
+
+        const site = toLauncherSiteInfo(row);
+        if (!site) {
             continue;
         }
-        seen.add(key);
 
-        const access = formatSiteResourceAccess({
+        const seenSiteIds = siteIdsByKey.get(key)!;
-            mode: row.mode,
+        if (seenSiteIds.has(site.siteId)) {
-            destination: row.destination,
+            continue;
-            destinationPort: row.destinationPort,
+        }
-            scheme: row.scheme,
+        seenSiteIds.add(site.siteId);
-            ssl: row.ssl,
+        item.sites.push(site);
-            fullDomain: row.fullDomain,
-            alias: row.alias,
-            aliasAddress: row.aliasAddress
-        });
-
-        result.push({
-            launcherResourceKey: key,
-            resourceType: "site",
-            resourceId: row.siteResourceId,
-            siteResourceId: row.siteResourceId,
-            niceId: row.niceId,
-            name: row.name,
-            ...access,
-            iconUrl: null,
-            enabled: row.enabled,
-            mode: row.mode,
-            labels: labelMaps.bySiteResourceId.get(row.siteResourceId) ?? [],
-            site:
-                row.siteId != null
-                    ? {
-                          siteId: row.siteId,
-                          name: row.siteName!,
-                          type: row.siteType!,
-                          online: row.siteOnline ?? undefined
-                      }
-                    : undefined
-        });
     }
 
-    return result;
+    for (const item of byKey.values()) {
+        item.sites.sort((a, b) =>
+            a.name.localeCompare(b.name, undefined, { sensitivity: "base" })
+        );
+        item.site = pickPrimarySite(item.sites, siteIdFilter);
+    }
+
+    return Array.from(byKey.values());
 }
 
 function filterResourcesBySite(
@@ -1252,13 +1327,17 @@ function filterResourcesBySite(
         return items.filter((item) => item.mode === "inference");
     }
     if (groupKey === LAUNCHER_NO_SITE_GROUP_KEY) {
-        return items.filter((item) => !item.site && item.mode !== "inference");
+        return items.filter(
+            (item) => item.sites.length === 0 && item.mode !== "inference"
+        );
     }
     const siteId = Number.parseInt(groupKey, 10);
     if (!Number.isFinite(siteId)) {
         return items;
     }
-    return items.filter((item) => item.site?.siteId === siteId);
+    return items.filter((item) =>
+        item.sites.some((site) => site.siteId === siteId)
+    );
 }
 
 function filterResourcesByLabel(
@@ -1499,6 +1578,7 @@ async function collectAccessibleSites(
             .select({
                 siteId: sites.siteId,
                 name: sites.name,
+                niceId: sites.niceId,
                 type: sites.type,
                 online: sites.online,
                 itemCount: countDistinct(resources.resourceId)
@@ -1507,7 +1587,13 @@ async function collectAccessibleSites(
             .innerJoin(resources, eq(targets.resourceId, resources.resourceId))
             .innerJoin(sites, eq(targets.siteId, sites.siteId))
             .where(and(...publicConditions))
-            .groupBy(sites.siteId, sites.name, sites.type, sites.online);
+            .groupBy(
+                sites.siteId,
+                sites.name,
+                sites.niceId,
+                sites.type,
+                sites.online
+            );
 
         for (const row of publicRows) {
             const existing = siteCountMap.get(row.siteId);
@@ -1517,6 +1603,7 @@ async function collectAccessibleSites(
                 siteCountMap.set(row.siteId, {
                     siteId: row.siteId,
                     name: row.name,
+                    niceId: row.niceId,
                     type: row.type,
                     online: row.online,
                     itemCount: Number(row.itemCount)
@@ -1540,6 +1627,7 @@ async function collectAccessibleSites(
             .select({
                 siteId: sites.siteId,
                 name: sites.name,
+                niceId: sites.niceId,
                 type: sites.type,
                 online: sites.online,
                 itemCount: countDistinct(siteResources.siteResourceId)
@@ -1551,7 +1639,13 @@ async function collectAccessibleSites(
             )
             .innerJoin(sites, eq(siteNetworks.siteId, sites.siteId))
             .where(and(...siteConditions))
-            .groupBy(sites.siteId, sites.name, sites.type, sites.online);
+            .groupBy(
+                sites.siteId,
+                sites.name,
+                sites.niceId,
+                sites.type,
+                sites.online
+            );
 
         for (const row of siteRows) {
             const existing = siteCountMap.get(row.siteId);
@@ -1561,6 +1655,7 @@ async function collectAccessibleSites(
                 siteCountMap.set(row.siteId, {
                     siteId: row.siteId,
                     name: row.name,
+                    niceId: row.niceId,
                     type: row.type,
                     online: row.online,
                     itemCount: Number(row.itemCount)
@@ -1675,6 +1770,7 @@ export async function listAccessibleLauncherSitesForUser(
         .map((row) => ({
             siteId: row.siteId,
             name: row.name,
+            niceId: row.niceId,
             type: row.type,
             online: row.online
         }))

server/routers/launcher/types.ts

@@ -32,6 +32,7 @@ export type LauncherLabel = {
 export type LauncherSiteInfo = {
     siteId: number;
     name: string;
+    niceId: string;
     type: string;
     online?: boolean;
 };
@@ -51,6 +52,7 @@ export type LauncherResource = {
     mode: string;
     labels: LauncherLabel[];
     site?: LauncherSiteInfo;
+    sites: LauncherSiteInfo[];
 };
 
 export type LauncherGroup = {
@@ -184,8 +186,7 @@ export function parseIdListParam(value: string | undefined): number[] {
 export const DEFAULT_LAUNCHER_VIEW_ID = "default" as const;
 
 export type LauncherViewSelection =
-    | { type: "default" }
+    { type: "default" } | { type: "saved"; viewId: number };
-    | { type: "saved"; viewId: number };
 
 export type LauncherScaleCapabilities = {
     allowSiteGrouping: boolean;

server/routers/newt/getNewtVersion.ts

@@ -13,31 +13,40 @@ import logger from "@server/logger";
 import { regionalCache as cache } from "#dynamic/lib/cache";
 import config from "@server/lib/config";
 
-// Stale-while-revalidate in-memory fallback for the releases API.
 type ReleaseInfo = {
     version: string;
     // binary filename -> sha256 hex (sourced from asset `digest` field in GitHub API)
     assetDigests: Record<string, string>;
 };
-let staleReleaseInfo: ReleaseInfo | null = null;
+
+// Cache key holding the last known good release info. It never expires, so
+// it keeps serving if GitHub is unreachable, even across restarts/nodes.
+const RELEASE_INFO_KEY = "cache:releaseInfo";
+// Short-lived marker controlling how often we re-check GitHub. While it's
+// missing (expired, or a previous attempt failed) every request retries.
+const RELEASE_INFO_FRESH_KEY = "cache:releaseInfoFresh";
+const RELEASE_INFO_REFRESH_SECONDS = 3600;
 
 /**
  * Fetches the latest stable newt release from GitHub and returns the version
  * tag together with a map of asset-name → sha256 hex digest.
- * Results are cached for one hour; stale data is returned on failure.
+ * The last successful result is cached indefinitely and re-checked hourly;
+ * on failure the last known good data keeps being served and every
+ * subsequent request retries GitHub until it succeeds again.
  */
-async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
+async function getLatestReleaseInfo(repo: string): Promise<ReleaseInfo | null> {
-    try {
+    const stored = await cache.get<ReleaseInfo>(`${RELEASE_INFO_KEY}:${repo}`);
-        const cached = await cache.get<ReleaseInfo>("cache:newtReleaseInfo");
+    const isFresh = await cache.has(`${RELEASE_INFO_FRESH_KEY}:${repo}`);
-        if (cached) {
+    if (stored && isFresh) {
-            return cached;
+        return stored;
-        }
+    }
 
+    try {
         const controller = new AbortController();
         const timeoutId = setTimeout(() => controller.abort(), 5000);
 
         const fetchResponse = await fetch(
-            "https://api.github.com/repos/fosrl/newt/releases",
+            `https://api.github.com/repos/fosrl/${repo}/releases`,
             { signal: controller.signal }
         );
 
@@ -47,13 +56,13 @@ async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
             logger.warn(
                 `Failed to fetch Newt releases from GitHub: ${fetchResponse.status} ${fetchResponse.statusText}`
             );
-            return staleReleaseInfo;
+            return stored ?? null;
         }
 
         let releases: any[] = await fetchResponse.json();
         if (!Array.isArray(releases) || releases.length === 0) {
-            logger.warn("No releases found for Newt repository");
+            logger.warn("No releases found for repository");
-            return staleReleaseInfo;
+            return stored ?? null;
         }
 
         const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
@@ -81,8 +90,8 @@ async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
         });
 
         if (releases.length === 0) {
-            logger.warn("No stable releases found for Newt repository");
+            logger.warn("No stable releases found for repository");
-            return staleReleaseInfo;
+            return stored ?? null;
         }
 
         const latest = releases[0];
@@ -106,8 +115,12 @@ async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
         }
 
         const info: ReleaseInfo = { version, assetDigests };
-        staleReleaseInfo = info;
+        await cache.set(RELEASE_INFO_KEY, info, 0);
-        await cache.set("cache:newtReleaseInfo", info, 3600);
+        await cache.set(
+            RELEASE_INFO_FRESH_KEY,
+            true,
+            RELEASE_INFO_REFRESH_SECONDS
+        );
         return info;
     } catch (error: any) {
         if (error.name === "AbortError") {
@@ -118,14 +131,15 @@ async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
                 error.message || error
             );
         }
-        return staleReleaseInfo;
+        return stored ?? null;
     }
 }
 
 const bodySchema = z.object({
     newtId: z.string(),
     secret: z.string(),
-    platform: z.string() // e.g. "linux_amd64", "darwin_arm64"
+    platform: z.string(), // e.g. "linux_amd64", "darwin_arm64"
+    agent: z.string().optional().default("newt")
 });
 
 export type GetNewtVersionBody = z.infer<typeof bodySchema>;
@@ -153,7 +167,7 @@ export async function getNewtVersion(
         );
     }
 
-    const { newtId, secret, platform } = parsedBody.data;
+    const { newtId, secret, platform, agent } = parsedBody.data;
 
     try {
         // Verify newt credentials
@@ -258,9 +272,13 @@ export async function getNewtVersion(
         }
 
         // Fetch latest release info (version + asset digests) in one API call.
-        const releaseInfo = await getLatestReleaseInfo();
+        const releaseInfoNewt = await getLatestReleaseInfo("newt");
+        let releaseInfoCli: ReleaseInfo | undefined | null;
+        if (agent == "cli") {
+            releaseInfoCli = await getLatestReleaseInfo("cli");
+        }
 
-        if (!releaseInfo) {
+        if (!releaseInfoNewt || (agent == "cli" && !releaseInfoCli)) {
             return next(
                 createHttpError(
                     HttpCode.INTERNAL_SERVER_ERROR,
@@ -269,18 +287,25 @@ export async function getNewtVersion(
             );
         }
 
-        const latestVersion = releaseInfo.version;
+        const latestVersion = releaseInfoNewt.version;
 
         // Binary name follows the get-newt.sh convention: newt_<platform>[.exe]
-        const binaryName = platform.includes("windows")
+        const binaryNameNewt = platform.includes("windows")
             ? `newt_${platform}.exe`
             : `newt_${platform}`;
 
-        const downloadUrl = `https://github.com/fosrl/newt/releases/download/${latestVersion}/${binaryName}`;
+        const binaryNameCli = platform.includes("windows")
+            ? `pangolin-cli_${platform}.exe`
+            : `pangolin-cli_${platform}`;
+
+        const downloadUrl = `https://github.com/fosrl/${agent}/releases/download/${agent == "cli" ? releaseInfoCli?.version : releaseInfoNewt.version}/${agent == "cli" ? binaryNameCli : binaryNameNewt}`;
 
         // Look up the SHA256 digest for this specific binary from the GitHub
         // release asset metadata (the `digest` field, format "sha256:<hex>").
-        const sha256 = releaseInfo.assetDigests[binaryName] ?? "";
+        const sha256 =
+            releaseInfoNewt.assetDigests[
+                agent == "cli" ? binaryNameCli : binaryNameNewt
+            ] ?? "";
 
         // Determine whether the newt that's asking is already up to date.
         // We store the current version on the newt row when it registers.
@@ -300,8 +325,8 @@ export async function getNewtVersion(
 
         return response<GetNewtVersionResponse>(res, {
             data: {
-                latestVersion,
+                latestVersion, // this will always be the newt version
-                currentIsLatest,
+                currentIsLatest, // this will always be based on the newt version
                 downloadUrl,
                 sha256
             },

server/routers/newt/handleNewtRegisterMessage.ts

@@ -37,6 +37,8 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => {
         publicKey,
         pingResults,
         newtVersion,
+        agent,
+        agentVersion,
         backwardsCompatible,
         chainId
     } = message.data;
@@ -169,22 +171,21 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => {
         logger.error(`Failed to add peer to exit node: ${error}`);
     }
 
-    if (newtVersion && newtVersion !== newt.version) {
+    if (
+        newtVersion !== newt.version ||
+        agent !== newt.agent ||
+        agentVersion !== newt.agentVersion
+    ) {
         // update the newt version in the database
         await db
             .update(newts)
             .set({
-                version: newtVersion as string
+                version: newtVersion as string,
-            })
+                agent: agent,
-            .where(eq(newts.newtId, newt.newtId));
+                agentVersion:
-    }
+                    !agentVersion && agent == "newt"
-
+                        ? newtVersion
-    if (newtVersion && newtVersion !== newt.version) {
+                        : agentVersion
-        // update the newt version in the database
-        await db
-            .update(newts)
-            .set({
-                version: newtVersion as string
             })
             .where(eq(newts.newtId, newt.newtId));
     }

server/routers/newt/handleRequestLogMessage.ts

@@ -1,16 +1,3 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
 import { db } from "@server/db";
 import { MessageHandler } from "@server/routers/ws";
 import { sites, Newt, orgs, clients, clientSitesAssociationsCache, users } from "@server/db";

server/routers/olm/handleOlmServerInitAddPeerHandshake.ts

@@ -85,7 +85,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
             );
 
         if (!resources || resources.length === 0) {
-            logger.error(
+            logger.warn(
                 `handleOlmServerInitAddPeerHandshake: Resource not found`
             );
             await sendCancel();
@@ -94,7 +94,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
 
         if (resources.length > 1) {
             // error but this should not happen because the nice id cant contain a dot and the alias has to have a dot and both have to be unique within the org so there should never be multiple matches
-            logger.error(
+            logger.warn(
                 `handleOlmServerInitAddPeerHandshake: Multiple resources found matching the criteria`
             );
             return;
@@ -119,7 +119,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
             );
 
         if (currentResourceAssociationCaches.length === 0) {
-            logger.error(
+            logger.warn(
                 `handleOlmServerInitAddPeerHandshake: Client ${client.clientId} does not have access to resource ${resource.siteResourceId}`
             );
             await sendCancel();
@@ -127,7 +127,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
         }
 
         if (!resource.networkId) {
-            logger.error(
+            logger.warn(
                 `handleOlmServerInitAddPeerHandshake: Resource ${resource.siteResourceId} has no network`
             );
             await sendCancel();
@@ -141,7 +141,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
             .where(eq(siteNetworks.networkId, resource.networkId));
 
         if (!siteRows || siteRows.length === 0) {
-            logger.error(
+            logger.warn(
                 `handleOlmServerInitAddPeerHandshake: No sites found for resource ${resource.siteResourceId}`
             );
             await sendCancel();
@@ -164,9 +164,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
     }
 
     if (sitesToProcess.length === 0) {
-        logger.error(
+        logger.warn(`handleOlmServerInitAddPeerHandshake: No sites to process`);
-            `handleOlmServerInitAddPeerHandshake: No sites to process`
-        );
         await sendCancel();
         return;
     }
@@ -193,7 +191,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
         }
 
         if (!site.exitNodeId) {
-            logger.error(
+            logger.warn(
                 `handleOlmServerInitAddPeerHandshake: Site ${site.siteId} has no exit node, skipping`
             );
             continue;
@@ -205,7 +203,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
             .where(eq(exitNodes.exitNodeId, site.exitNodeId));
 
         if (!exitNode) {
-            logger.error(
+            logger.warn(
                 `handleOlmServerInitAddPeerHandshake: Exit node not found for site ${site.siteId}, skipping`
             );
             continue;
@@ -229,7 +227,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
     }
 
     if (!handshakeInitiated) {
-        logger.error(
+        logger.warn(
             `handleOlmServerInitAddPeerHandshake: No accessible sites with valid exit nodes found, cancelling chain`
         );
         await sendCancel();

server/routers/org/adminDeleteOrg.ts

@@ -0,0 +1,98 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import { deleteOrgById, sendTerminationMessages } from "@server/lib/deleteOrg";
+import { db, orgs } from "@server/db";
+import { eq } from "drizzle-orm";
+
+const adminDeleteOrgSchema = z.strictObject({
+    orgId: z.string()
+});
+
+export type AdminDeleteOrgResponse = {};
+
+registry.registerPath({
+    method: "delete",
+    path: "/admin/org/{orgId}",
+    description: "Delete any organization in the system (server admin).",
+    tags: [OpenAPITags.Org],
+    request: {
+        params: adminDeleteOrgSchema
+    },
+    responses: {
+        200: {
+            description: "Successful response",
+            content: {
+                "application/json": {
+                    schema: z.object({
+                        data: z.record(z.string(), z.any()).nullable(),
+                        success: z.boolean(),
+                        error: z.boolean(),
+                        message: z.string(),
+                        status: z.number()
+                    })
+                }
+            }
+        }
+    }
+});
+
+export async function adminDeleteOrg(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = adminDeleteOrgSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+        const { orgId } = parsedParams.data;
+
+        const [org] = await db
+            .select()
+            .from(orgs)
+            .where(eq(orgs.orgId, orgId))
+            .limit(1);
+
+        if (!org) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    `Organization with ID ${orgId} not found`
+                )
+            );
+        }
+
+        const result = await deleteOrgById(orgId);
+        sendTerminationMessages(result);
+        return response(res, {
+            data: null,
+            success: true,
+            error: false,
+            message: "Organization deleted successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        if (createHttpError.isHttpError(error)) {
+            return next(error);
+        }
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "An error occurred..."
+            )
+        );
+    }
+}

server/routers/org/adminListOrgs.ts

@@ -0,0 +1,241 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { db, users } from "@server/db";
+import { orgs, resources, sites, userOrgs } from "@server/db";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import { and, asc, desc, eq, like, or, sql, type SQL } from "drizzle-orm";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import { createApiResponseSchema } from "@server/lib/openapi/createApiResponseSchema";
+import type { PaginatedResponse } from "@server/types/Pagination";
+
+const adminListOrgsSchema = z.strictObject({
+    pageSize: z.coerce
+        .number<string>()
+        .int()
+        .positive()
+        .optional()
+        .catch(20)
+        .default(20)
+        .openapi({
+            type: "integer",
+            default: 20,
+            description: "Number of items per page"
+        }),
+    page: z.coerce
+        .number<string>()
+        .int()
+        .positive()
+        .optional()
+        .catch(1)
+        .default(1)
+        .openapi({
+            type: "integer",
+            default: 1,
+            description: "Page number to retrieve"
+        }),
+    query: z.string().optional(),
+    sort_by: z
+        .enum(["name", "createdAt"])
+        .optional()
+        .catch(undefined)
+        .openapi({
+            type: "string",
+            enum: ["name", "createdAt"],
+            description: "Field to sort by"
+        }),
+    order: z
+        .enum(["asc", "desc"])
+        .optional()
+        .default("asc")
+        .catch("asc")
+        .openapi({
+            type: "string",
+            enum: ["asc", "desc"],
+            default: "asc",
+            description: "Sort order"
+        })
+});
+
+export type AdminOrgRow = {
+    orgId: string;
+    name: string;
+    subnet: string | null;
+    utilitySubnet: string | null;
+    createdAt: string | null;
+    userCount: number;
+    siteCount: number;
+    resourceCount: number;
+    owner: {
+        userId: string;
+        username: string;
+    } | null;
+};
+
+export type AdminListOrgsResponse = PaginatedResponse<{
+    orgs: AdminOrgRow[];
+}>;
+
+const AdminListOrgsResponseDataSchema = z.object({
+    orgs: z.array(
+        z.object({
+            orgId: z.string(),
+            name: z.string(),
+            subnet: z.string().nullable(),
+            createdAt: z.string().nullable(),
+            userCount: z.number(),
+            siteCount: z.number(),
+            resourceCount: z.number()
+        })
+    ),
+    pagination: z.object({
+        total: z.number(),
+        page: z.number(),
+        pageSize: z.number()
+    })
+});
+
+registry.registerPath({
+    method: "get",
+    path: "/admin/orgs",
+    description:
+        "List all organizations in the system with usage counts (server admin).",
+    tags: [OpenAPITags.Org],
+    request: {
+        query: adminListOrgsSchema
+    },
+    responses: {
+        200: {
+            description: "Successful response",
+            content: {
+                "application/json": {
+                    schema: createApiResponseSchema(
+                        AdminListOrgsResponseDataSchema
+                    )
+                }
+            }
+        }
+    }
+});
+
+export async function adminListOrgs(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedQuery = adminListOrgsSchema.safeParse(req.query);
+        if (!parsedQuery.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedQuery.error)
+                )
+            );
+        }
+
+        const { pageSize, page, query, sort_by, order } = parsedQuery.data;
+
+        let conditions: (SQL<unknown> | undefined)[] = [];
+        if (query) {
+            const q = "%" + query.toLowerCase() + "%";
+            conditions.push(
+                or(
+                    like(sql`LOWER(${orgs.name})`, q),
+                    like(sql`LOWER(${orgs.orgId})`, q),
+                    like(sql`LOWER(${orgs.subnet})`, q)
+                )
+            );
+        }
+
+        const sortColumns = {
+            name: orgs.name,
+            createdAt: orgs.createdAt
+        } as const;
+
+        const orderBy = sort_by
+            ? order === "asc"
+                ? asc(sortColumns[sort_by])
+                : desc(sortColumns[sort_by])
+            : asc(orgs.name);
+
+        // Drizzle renders bare column references in the select list without their
+        // table prefix, which would make a correlated subquery compare a column to
+        // itself, so the outer `orgs` side is qualified explicitly.
+        const orgIdRef = sql`${sql.identifier("orgs")}.${sql.identifier("orgId")}`;
+
+        const [countRows, rows] = await Promise.all([
+            db
+                .select({ count: sql<number>`count(*)` })
+                .from(orgs)
+                .where(and(...conditions)),
+            db
+                .selectDistinct({
+                    orgId: orgs.orgId,
+                    name: orgs.name,
+                    subnet: orgs.subnet,
+                    utilitySubnet: orgs.utilitySubnet,
+                    createdAt: orgs.createdAt,
+                    userCount: sql<number>`(
+                        SELECT COUNT(*)
+                        FROM ${userOrgs}
+                        WHERE ${userOrgs.orgId} = ${orgIdRef}
+                    )`.as("userCount"),
+                    siteCount: sql<number>`(
+                        SELECT COUNT(*)
+                        FROM ${sites}
+                        WHERE ${sites.orgId} = ${orgIdRef}
+                    )`.as("siteCount"),
+                    resourceCount: sql<number>`(
+                        SELECT COUNT(*)
+                        FROM ${resources}
+                        WHERE ${resources.orgId} = ${orgIdRef}
+                    )`.as("resourceCount"),
+                    owner: {
+                        userId: users.userId,
+                        username: users.username
+                    }
+                })
+                .from(orgs)
+                .where(and(...conditions, eq(userOrgs.isOwner, true)))
+                .leftJoin(userOrgs, eq(userOrgs.orgId, orgs.orgId))
+                .leftJoin(users, eq(userOrgs.userId, users.userId))
+                .limit(pageSize)
+                .offset(pageSize * (page - 1))
+                .orderBy(orderBy)
+        ]);
+
+        const totalCount = Number(countRows[0]?.count ?? 0);
+
+        return response<AdminListOrgsResponse>(res, {
+            data: {
+                orgs: rows.map((row) => ({
+                    ...row,
+                    userCount: Number(row.userCount ?? 0),
+                    siteCount: Number(row.siteCount ?? 0),
+                    resourceCount: Number(row.resourceCount ?? 0)
+                })),
+                pagination: {
+                    total: totalCount,
+                    page,
+                    pageSize
+                }
+            },
+            success: true,
+            error: false,
+            message: "Organizations retrieved successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "An error occurred..."
+            )
+        );
+    }
+}

server/routers/org/index.ts

@@ -9,3 +9,5 @@ export * from "./listOrgs";
 export * from "./pickOrgDefaults";
 export * from "./checkOrgUserAccess";
 export * from "./resetOrgBandwidth";
+export * from "./adminListOrgs";
+export * from "./adminDeleteOrg";

server/routers/site/getSite.ts

@@ -48,6 +48,8 @@ type SiteQueryRow = NonNullable<Awaited<ReturnType<typeof query>>>;
 export type GetSiteResponse = SiteQueryRow["sites"] & {
     newtId: string | null;
     newtVersion: string | null;
+    agent: string | null;
+    agentVersion: string | null;
     countryCode: string | null;
 };
 
@@ -137,6 +139,8 @@ export async function getSite(
             ...site.sites,
             newtId: site.newt ? site.newt.newtId : null,
             newtVersion: site.newt?.version ?? null,
+            agent: site.newt?.agent ?? null,
+            agentVersion: site.newt?.agentVersion ?? null,
             countryCode: site.sites.endpoint
                 ? ((await getCountryCodeForIp(site.sites.endpoint)) ?? null)
                 : null

server/routers/site/listSites.ts

@@ -133,6 +133,8 @@ function querySitesBase() {
             online: sites.online,
             address: sites.address,
             newtVersion: newts.version,
+            agent: newts.agent,
+            agentVersion: newts.agentVersion,
             exitNodeId: sites.exitNodeId,
             exitNodeName: exitNodes.name,
             exitNodeEndpoint: exitNodes.endpoint,

server/routers/user/adminListUsers.ts

@@ -4,7 +4,7 @@ import { db, idp, users } from "@server/db";
 import response from "@server/lib/response";
 import HttpCode from "@server/types/HttpCode";
 import createHttpError from "http-errors";
-import { and, asc, desc, eq, like, or, sql } from "drizzle-orm";
+import { and, asc, desc, eq, like, or, sql, type SQL } from "drizzle-orm";
 import logger from "@server/logger";
 import { fromZodError } from "zod-validation-error";
 import { OpenAPITags, registry } from "@server/openApi";
@@ -88,6 +88,15 @@ const listUsersSchema = z.strictObject({
             type: "boolean",
             description:
                 "Filter by 2FA state matching: enabled if twoFactorEnabled or twoFactorSetupRequested"
+        }),
+    server_admin: z
+        .enum(["true", "false"])
+        .transform((v) => v === "true")
+        .optional()
+        .catch(undefined)
+        .openapi({
+            type: "boolean",
+            description: "Filter by server admin status"
         })
 });
 
@@ -177,7 +186,8 @@ export async function adminListUsers(
             sort_by,
             order,
             idp_id,
-            two_factor: twoFactorFilter
+            two_factor: twoFactorFilter,
+            server_admin: serverAdminFilter
         } = parsedQuery.data;
 
         if (typeof idp_id === "number") {
@@ -196,7 +206,7 @@ export async function adminListUsers(
             }
         }
 
-        const conditions = [eq(users.serverAdmin, false)];
+        const conditions: Array<SQL<unknown> | undefined> = [];
 
         if (query) {
             const q = "%" + query.toLowerCase() + "%";
@@ -233,6 +243,10 @@ export async function adminListUsers(
             }
         }
 
+        if (typeof serverAdminFilter === "boolean") {
+            conditions.push(eq(users.serverAdmin, serverAdminFilter));
+        }
+
         const whereClause = and(...conditions);
 
         const countQuery = db.$count(

server/routers/user/adminSetServerAdmin.ts

@@ -0,0 +1,151 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { db, users } from "@server/db";
+import { eq } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import { createApiResponseSchema } from "@server/lib/openapi/createApiResponseSchema";
+
+const setServerAdminParamsSchema = z.strictObject({
+    userId: z.string()
+});
+
+const setServerAdminBodySchema = z.strictObject({
+    serverAdmin: z.boolean()
+});
+
+export type AdminSetServerAdminResponse = {
+    userId: string;
+    serverAdmin: boolean;
+};
+
+const AdminSetServerAdminResponseDataSchema = z.object({
+    userId: z.string(),
+    serverAdmin: z.boolean()
+});
+
+registry.registerPath({
+    method: "post",
+    path: "/user/{userId}/server-admin",
+    description:
+        "Promote or demote a user's server admin status (server admin).",
+    tags: [OpenAPITags.User],
+    request: {
+        params: setServerAdminParamsSchema,
+        body: {
+            content: {
+                "application/json": {
+                    schema: setServerAdminBodySchema
+                }
+            }
+        }
+    },
+    responses: {
+        200: {
+            description: "Successful response",
+            content: {
+                "application/json": {
+                    schema: createApiResponseSchema(
+                        AdminSetServerAdminResponseDataSchema
+                    )
+                }
+            }
+        }
+    }
+});
+
+export async function adminSetServerAdmin(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = setServerAdminParamsSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const parsedBody = setServerAdminBodySchema.safeParse(req.body);
+        if (!parsedBody.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedBody.error).toString()
+                )
+            );
+        }
+
+        const { userId } = parsedParams.data;
+        const { serverAdmin } = parsedBody.data;
+
+        const [existingUser] = await db
+            .select({
+                userId: users.userId,
+                serverAdmin: users.serverAdmin,
+                type: users.type
+            })
+            .from(users)
+            .where(eq(users.userId, userId))
+            .limit(1);
+
+        if (!existingUser) {
+            return next(createHttpError(HttpCode.NOT_FOUND, "User not found"));
+        }
+
+        if (existingUser.type !== "internal") {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    "Server admin status can only be changed for internal users"
+                )
+            );
+        }
+
+        if (!serverAdmin && req.user?.userId === userId) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    "You cannot remove your own server admin status"
+                )
+            );
+        }
+
+        if (existingUser.serverAdmin !== serverAdmin) {
+            logger.info(
+                `${serverAdmin ? "Promoting" : "Demoting"} user ${userId} ${serverAdmin ? "to" : "from"} server admin (by ${req.user?.userId})`
+            );
+
+            await db
+                .update(users)
+                .set({ serverAdmin })
+                .where(eq(users.userId, userId));
+        }
+
+        return response<AdminSetServerAdminResponse>(res, {
+            data: {
+                userId: existingUser.userId,
+                serverAdmin
+            },
+            success: true,
+            error: false,
+            message: serverAdmin
+                ? "User promoted to server admin successfully"
+                : "User demoted from server admin successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
+        );
+    }
+}

server/routers/user/index.ts

@@ -11,6 +11,7 @@ export * from "./adminListUsers";
 export * from "./adminRemoveUser";
 export * from "./adminGetUser";
 export * from "./adminGeneratePasswordResetCode";
+export * from "./adminSetServerAdmin";
 export * from "./listInvitations";
 export * from "./removeInvitation";
 export * from "./createOrgUser";

server/routers/user/listUsers.ts

@@ -95,22 +95,34 @@ const listUsersSchema = z.strictObject({
                 'Filter by identity provider id, or "internal" for internal users'
         }),
     role_id: z
-        .preprocess((val) => {
+        .preprocess(
-            if (val === undefined || val === null || val === "") {
+            (val) => {
-                return undefined;
+                if (val === undefined || val === null || val === "") {
-            }
+                    return undefined;
-            const raw = Array.isArray(val) ? val : [val];
+                }
-            const nums = raw
+                const raw = Array.isArray(val) ? val : [val];
-                .map((v) =>
+                const includeOwner = raw.some((v) => v === "owner");
-                    typeof v === "string" ? parseInt(v, 10) : Number(v)
+                const nums = raw
-                )
+                    .map((v) =>
-                .filter((n) => Number.isInteger(n) && n > 0);
+                        typeof v === "string" ? parseInt(v, 10) : Number(v)
-            const unique = [...new Set(nums)];
+                    )
-            return unique.length ? unique : undefined;
+                    .filter((n) => Number.isInteger(n) && n > 0);
-        }, z.array(z.number().int().positive()).optional())
+                const unique = [...new Set(nums)];
+                if (!unique.length && !includeOwner) {
+                    return undefined;
+                }
+                return { roleIds: unique, includeOwner };
+            },
+            z
+                .object({
+                    roleIds: z.array(z.number().int().positive()),
+                    includeOwner: z.boolean()
+                })
+                .optional()
+        )
         .openapi({
             description:
-                "Filter users who have any of these role ids in the organization (repeat query param)"
+                'Filter users who have any of these role ids in the organization, or "owner" for organization owners (repeat query param)'
         })
 });
 
@@ -193,7 +205,8 @@ export async function listUsers(
         }
         const { page, pageSize, sort_by, order, query, idp_id, role_id } =
             parsedQuery.data;
-        const roleIds = role_id ?? [];
+        const roleIds = role_id?.roleIds ?? [];
+        const includeOwner = role_id?.includeOwner ?? false;
 
         const parsedParams = listUsersParamsSchema.safeParse(req.params);
         if (!parsedParams.success) {
@@ -267,21 +280,35 @@ export async function listUsers(
             conditions.push(eq(users.idpId, idp_id));
         }
 
-        if (roleIds.length > 0) {
+        if (roleIds.length > 0 || includeOwner) {
-            conditions.push(
+            const roleFilterParts = [];
-                exists(
+
-                    db
+            if (includeOwner) {
-                        .select()
+                roleFilterParts.push(eq(userOrgs.isOwner, true));
-                        .from(userOrgRoles)
+            }
-                        .where(
+
-                            and(
+            if (roleIds.length > 0) {
-                                eq(userOrgRoles.userId, users.userId),
+                roleFilterParts.push(
-                                eq(userOrgRoles.orgId, orgId),
+                    exists(
-                                inArray(userOrgRoles.roleId, roleIds)
+                        db
+                            .select()
+                            .from(userOrgRoles)
+                            .where(
+                                and(
+                                    eq(userOrgRoles.userId, users.userId),
+                                    eq(userOrgRoles.orgId, orgId),
+                                    inArray(userOrgRoles.roleId, roleIds)
+                                )
                             )
-                        )
+                    )
-                )
+                );
-            );
+            }
+
+            if (roleFilterParts.length === 1) {
+                conditions.push(roleFilterParts[0]);
+            } else if (roleFilterParts.length > 1) {
+                conditions.push(or(...roleFilterParts));
+            }
         }
 
         const countQuery = db.$count(

server/setup/migrationsPg.ts

@@ -66,6 +66,12 @@ const migrations = [
 
 await run();
 
+// The pg Pool is created with allowExitOnIdle: false (see poolConfig.ts) so
+// its sockets keep the event loop alive even when idle. Without an explicit
+// exit here, this one-shot script would hang until the pool's
+// idleTimeoutMillis elapses before the process could terminate.
+process.exit(0);
+
 async function run() {
     // run the migrations
     await runMigrations();

src/app/[orgId]/layout.tsx

@@ -104,7 +104,7 @@ export default async function OrgLayout(props: {
             subscriptionStatus = subRes.data.data;
         } catch (error) {
             // If subscription fetch fails, keep subscriptionStatus as null
-            console.error("Failed to fetch subscription status:", error);
+            // console.error("Failed to fetch subscription status:", error);
         }
     }
 

src/app/[orgId]/loading.tsx

@@ -1,9 +1,5 @@
-import { Loader2 } from "lucide-react";
+import OrgRouteLoading from "@app/components/OrgRouteLoading";
 
 export default function OrgPageLoading() {
-    return (
+    return <OrgRouteLoading />;
-        <div className="flex items-center justify-center py-16">
-            <Loader2 className="size-6 animate-spin text-muted-foreground" />
-        </div>
-    );
 }

src/app/[orgId]/settings/(private)/remote-exit-nodes/page.tsx

@@ -10,6 +10,7 @@ import { getTranslations } from "next-intl/server";
 import type { Metadata } from "next";
 import { build } from "@server/build";
 import { redirect } from "next/navigation";
+import { useEnvContext } from "@app/hooks/useEnvContext";
 
 export const metadata: Metadata = {
     title: "Remote Exit Nodes"
@@ -24,10 +25,6 @@ export const dynamic = "force-dynamic";
 export default async function RemoteExitNodesPage(
     props: RemoteExitNodesPageProps
 ) {
-    if (build != "saas") {
-        redirect("/");
-    }
-
     const params = await props.params;
     let remoteExitNodes: ListRemoteExitNodesResponse["remoteExitNodes"] = [];
     try {

src/app/[orgId]/settings/access/users/[userId]/general/page.tsx

@@ -38,7 +38,7 @@ import { useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import { z } from "zod";
 
-export default function AccessControlsPage() {
+export default function GeneralPage() {
     const { orgUser: user, updateOrgUser } = userOrgUserContext();
     const { user: sessionUser } = useUserContext();
     const { env } = useEnvContext();
@@ -57,7 +57,7 @@ export default function AccessControlsPage() {
             (build === "enterprise" && !isPaid) ||
             (build === "oss" && !isPaid));
 
-    const accessControlsFormSchema = z.object({
+    const generalFormSchema = z.object({
         username: z.string(),
         autoProvisioned: z.boolean(),
         roles: z
@@ -72,7 +72,7 @@ export default function AccessControlsPage() {
     });
 
     const form = useForm({
-        resolver: zodResolver(accessControlsFormSchema),
+        resolver: zodResolver(generalFormSchema),
         defaultValues: {
             username: user.username!,
             autoProvisioned: user.autoProvisioned || false,
@@ -155,7 +155,7 @@ export default function AccessControlsPage() {
         }
     }
 
-    async function handleAccessControlsSubmit(e: React.FormEvent) {
+    async function handleGeneralSubmit(e: React.FormEvent) {
         e.preventDefault();
 
         const isValid = await form.trigger();
@@ -196,11 +196,9 @@ export default function AccessControlsPage() {
 
             <SettingsSection>
                 <SettingsSectionHeader>
-                    <SettingsSectionTitle>
+                    <SettingsSectionTitle>{t("general")}</SettingsSectionTitle>
-                        {t("accessControls")}
-                    </SettingsSectionTitle>
                     <SettingsSectionDescription>
-                        {t("accessControlsDescription")}
+                        {t("userGeneralSettingsDescription")}
                     </SettingsSectionDescription>
                 </SettingsSectionHeader>
 
@@ -208,11 +206,9 @@ export default function AccessControlsPage() {
                     <SettingsSectionForm>
                         <Form {...form}>
                             <form
-                                onSubmit={(e) =>
+                                onSubmit={(e) => void handleGeneralSubmit(e)}
-                                    void handleAccessControlsSubmit(e)
-                                }
                                 className="space-y-4"
-                                id="access-controls-form"
+                                id="user-general-form"
                             >
                                 {user.type !== UserType.Internal &&
                                     user.idpType && (
@@ -281,9 +277,9 @@ export default function AccessControlsPage() {
                         type="submit"
                         loading={isSaving}
                         disabled={isSaving}
-                        form="access-controls-form"
+                        form="user-general-form"
                     >
-                        {t("accessControlsSubmit")}
+                        {t("saveSettings")}
                     </Button>
                 </SettingsSectionFooter>
             </SettingsSection>

src/app/[orgId]/settings/access/users/[userId]/layout.tsx

@@ -9,15 +9,16 @@ import { cache } from "react";
 import SettingsSectionTitle from "@app/components/SettingsSectionTitle";
 import { getTranslations } from "next-intl/server";
 import type { Metadata } from "next";
+import { getUserDisplayName } from "@app/lib/getUserDisplayName";
 
 export const metadata: Metadata = {
     title: "User"
 };
 
-interface UserLayoutProps {
+type UserLayoutProps = {
     children: React.ReactNode;
     params: Promise<{ userId: string; orgId: string }>;
-}
+};
 
 export default async function UserLayoutProps(props: UserLayoutProps) {
     const params = await props.params;
@@ -42,15 +43,23 @@ export default async function UserLayoutProps(props: UserLayoutProps) {
 
     const navItems = [
         {
-            title: t("accessControls"),
+            title: t("general"),
-            href: "/{orgId}/settings/access/users/{userId}/access-controls"
+            href: "/{orgId}/settings/access/users/{userId}/general"
         }
     ];
 
     return (
         <>
             <SettingsSectionTitle
-                title={`${user?.email}`}
+                title={
+                    user
+                        ? getUserDisplayName({
+                              email: user.email,
+                              name: user.name,
+                              username: user.username
+                          })
+                        : ""
+                }
                 description={t("userDescription2")}
             />
             <OrgUserProvider orgUser={user}>

src/app/[orgId]/settings/access/users/[userId]/page.tsx

@@ -9,5 +9,5 @@ export default async function UserPage(props: {
     params: Promise<{ orgId: string; userId: string }>;
 }) {
     const { orgId, userId } = await props.params;
-    redirect(`/${orgId}/settings/access/users/${userId}/access-controls`);
+    redirect(`/${orgId}/settings/access/users/${userId}/general`);
 }

src/app/[orgId]/settings/access/users/create/page.tsx

@@ -1,24 +1,20 @@
 "use client";
 
+import CopyTextBox from "@app/components/CopyTextBox";
 import {
-    SettingsContainer,
+    Credenza,
-    SettingsSection,
+    CredenzaBody,
-    SettingsSectionBody,
+    CredenzaContent,
-    SettingsSectionDescription,
+    CredenzaDescription,
-    SettingsSectionForm,
+    CredenzaFooter,
-    SettingsSectionHeader,
+    CredenzaHeader,
-    SettingsSectionTitle
+    CredenzaTitle
-} from "@app/components/Settings";
+} from "@app/components/Credenza";
-import { StrategyOption, StrategySelect } from "@app/components/StrategySelect";
+import { StrategySelect } from "@app/components/StrategySelect";
 import HeaderTitle from "@app/components/SettingsSectionTitle";
 import { Button } from "@app/components/ui/button";
 import { useParams, useRouter } from "next/navigation";
-import {
+import { useActionState, useRef, useState, startTransition } from "react";
-    useActionState,
-    useRef,
-    useState,
-    startTransition
-} from "react";
 import {
     Form,
     FormControl,
@@ -42,7 +38,6 @@ import { AxiosResponse } from "axios";
 import { useEffect } from "react";
 import { useForm } from "react-hook-form";
 import { z } from "zod";
-import CopyTextBox from "@app/components/CopyTextBox";
 import { useEnvContext } from "@app/hooks/useEnvContext";
 import { ListRolesResponse } from "@server/routers/role";
 import { formatAxiosError } from "@app/lib/api";
@@ -55,7 +50,15 @@ import IdpTypeIcon from "@app/components/IdpTypeIcon";
 import { usePaidStatus } from "@app/hooks/usePaidStatus";
 import { tierMatrix } from "@server/lib/billing/tierMatrix";
 import OrgRolesTagField from "@app/components/OrgRolesTagField";
-import CopyToClipboard from "@app/components/CopyToClipboard";
+import {
+    SettingsContainer,
+    SettingsSection,
+    SettingsSectionBody,
+    SettingsSectionDescription,
+    SettingsSectionForm,
+    SettingsSectionHeader,
+    SettingsSectionTitle
+} from "@app/components/Settings";
 
 type UserType = "internal" | "oidc";
 
@@ -96,6 +99,7 @@ export default function Page() {
         "internal"
     );
     const [inviteLink, setInviteLink] = useState<string | null>(null);
+    const [isInviteDialogOpen, setIsInviteDialogOpen] = useState(false);
 
     const [expiresInDays, setExpiresInDays] = useState(1);
     const [roles, setRoles] = useState<{ roleId: number; name: string }[]>([]);
@@ -246,9 +250,9 @@ export default function Page() {
                 build === "saas" || env.app.identityProviderMode === "org";
 
             const res = await api
-                .get<
+                .get<AxiosResponse<ListIdpsResponse>>(
-                    AxiosResponse<ListIdpsResponse>
+                    useOrgIdps ? `/org/${orgId}/idp` : "/idp"
-                >(useOrgIdps ? `/org/${orgId}/idp` : "/idp")
+                )
                 .catch((e) => {
                     console.error(e);
                     toast({
@@ -350,13 +354,13 @@ export default function Page() {
 
         if (res && res.status === 200) {
             setInviteLink(res.data.data.inviteLink);
+            setExpiresInDays(parseInt(values.validForHours) / 24);
+            setIsInviteDialogOpen(true);
             toast({
                 variant: "default",
                 title: t("userInvited"),
                 description: t("userInvitedDescription")
             });
-
-            setExpiresInDays(parseInt(values.validForHours) / 24);
         }
     }
 
@@ -460,6 +464,7 @@ export default function Page() {
             setSendEmail(env.email.emailEnabled);
             internalForm.reset();
             setInviteLink(null);
+            setIsInviteDialogOpen(false);
             setExpiresInDays(1);
         } else {
             googleAzureForm.reset();
@@ -486,7 +491,7 @@ export default function Page() {
 
             <div>
                 <SettingsContainer>
-                    {!inviteLink && userOptions.length > 1 ? (
+                    {userOptions.length > 1 ? (
                         <SettingsSection>
                             <SettingsSectionHeader>
                                 <SettingsSectionTitle>
@@ -508,177 +513,132 @@ export default function Page() {
                     ) : null}
 
                     {selectedOption === "internal" && dataLoaded && (
-                        <>
+                        <SettingsSection>
-                            {!inviteLink ? (
+                            <SettingsSectionHeader>
-                                <SettingsSection>
+                                <SettingsSectionTitle>
-                                    <SettingsSectionHeader>
+                                    {t("userSettings")}
-                                        <SettingsSectionTitle>
+                                </SettingsSectionTitle>
-                                            {t("userSettings")}
+                                <SettingsSectionDescription>
-                                        </SettingsSectionTitle>
+                                    {t("userSettingsDescription")}
-                                        <SettingsSectionDescription>
+                                </SettingsSectionDescription>
-                                            {t("userSettingsDescription")}
+                            </SettingsSectionHeader>
-                                        </SettingsSectionDescription>
+                            <SettingsSectionBody>
-                                    </SettingsSectionHeader>
+                                <SettingsSectionForm>
-                                    <SettingsSectionBody>
+                                    <Form {...internalForm}>
-                                        <SettingsSectionForm>
+                                        <form
-                                            <Form {...internalForm}>
+                                            onSubmit={(e) => {
-                                                <form
+                                                e.preventDefault();
-                                                    onSubmit={(e) => {
+                                                startTransition(() => {
-                                                        e.preventDefault();
+                                                    submitInternalAction();
-                                                        startTransition(() => {
+                                                });
-                                                            submitInternalAction();
+                                            }}
-                                                        });
+                                            className="space-y-4"
-                                                    }}
+                                            id="create-user-form"
-                                                    className="space-y-4"
+                                        >
-                                                    id="create-user-form"
+                                            <FormField
-                                                >
+                                                control={internalForm.control}
-                                                    <FormField
+                                                name="email"
-                                                        control={
+                                                render={({ field }) => (
-                                                            internalForm.control
+                                                    <FormItem>
-                                                        }
+                                                        <FormLabel>
-                                                        name="email"
+                                                            {t("email")}
-                                                        render={({ field }) => (
+                                                        </FormLabel>
-                                                            <FormItem>
+                                                        <FormControl>
-                                                                <FormLabel>
+                                                            <Input {...field} />
-                                                                    {t("email")}
+                                                        </FormControl>
-                                                                </FormLabel>
+                                                        <FormMessage />
-                                                                <FormControl>
+                                                    </FormItem>
-                                                                    <Input
+                                                )}
-                                                                        {...field}
-                                                                    />
-                                                                </FormControl>
-                                                                <FormMessage />
-                                                            </FormItem>
-                                                        )}
-                                                    />
-
-                                                    <FormField
-                                                        control={
-                                                            internalForm.control
-                                                        }
-                                                        name="validForHours"
-                                                        render={({ field }) => (
-                                                            <FormItem>
-                                                                <FormLabel>
-                                                                    {t(
-                                                                        "inviteValid"
-                                                                    )}
-                                                                </FormLabel>
-                                                                <Select
-                                                                    onValueChange={
-                                                                        field.onChange
-                                                                    }
-                                                                    defaultValue={
-                                                                        field.value
-                                                                    }
-                                                                >
-                                                                    <FormControl>
-                                                                        <SelectTrigger className="w-full">
-                                                                            <SelectValue
-                                                                                placeholder={t(
-                                                                                    "selectDuration"
-                                                                                )}
-                                                                            />
-                                                                        </SelectTrigger>
-                                                                    </FormControl>
-                                                                    <SelectContent>
-                                                                        {validFor.map(
-                                                                            (
-                                                                                option
-                                                                            ) => (
-                                                                                <SelectItem
-                                                                                    key={
-                                                                                        option.hours
-                                                                                    }
-                                                                                    value={option.hours.toString()}
-                                                                                >
-                                                                                    {
-                                                                                        option.name
-                                                                                    }
-                                                                                </SelectItem>
-                                                                            )
-                                                                        )}
-                                                                    </SelectContent>
-                                                                </Select>
-                                                                <FormMessage />
-                                                            </FormItem>
-                                                        )}
-                                                    />
-
-                                                    <OrgRolesTagField
-                                                        form={internalForm}
-                                                        name="roles"
-                                                        orgId={orgId as string}
-                                                        supportsMultipleRolesPerUser={
-                                                            supportsMultipleRolesPerUser
-                                                        }
-                                                        showMultiRolePaywallMessage={
-                                                            showMultiRolePaywallMessage
-                                                        }
-                                                        paywallMessage={
-                                                            invitePaywallMessage
-                                                        }
-                                                    />
-
-                                                    {env.email.emailEnabled && (
-                                                        <div className="flex items-center space-x-2">
-                                                            <Checkbox
-                                                                id="send-email"
-                                                                checked={
-                                                                    sendEmail
-                                                                }
-                                                                onCheckedChange={(
-                                                                    e
-                                                                ) =>
-                                                                    setSendEmail(
-                                                                        e as boolean
-                                                                    )
-                                                                }
-                                                            />
-                                                            <label
-                                                                htmlFor="send-email"
-                                                                className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
-                                                            >
-                                                                {t(
-                                                                    "inviteEmailSent"
-                                                                )}
-                                                            </label>
-                                                        </div>
-                                                    )}
-                                                </form>
-                                            </Form>
-                                        </SettingsSectionForm>
-                                    </SettingsSectionBody>
-                                </SettingsSection>
-                            ) : (
-                                <SettingsSection>
-                                    <SettingsSectionHeader>
-                                        <SettingsSectionTitle>
-                                            {t("userInvited")}
-                                        </SettingsSectionTitle>
-                                        <SettingsSectionDescription>
-                                            {sendEmail
-                                                ? t(
-                                                      "inviteEmailSentDescription"
-                                                  )
-                                                : t("inviteSentDescription")}
-                                        </SettingsSectionDescription>
-                                    </SettingsSectionHeader>
-                                    <SettingsSectionBody>
-                                        <div className="space-y-4">
-                                            <p>
-                                                {t("inviteExpiresIn", {
-                                                    days: expiresInDays
-                                                })}
-                                            </p>
-                                            <CopyToClipboard
-                                                text={inviteLink}
                                             />
-                                        </div>
+
-                                    </SettingsSectionBody>
+                                            {env.email.emailEnabled && (
-                                </SettingsSection>
+                                                <div className="flex items-center space-x-2">
-                            )}
+                                                    <Checkbox
-                        </>
+                                                        id="send-email"
+                                                        checked={sendEmail}
+                                                        onCheckedChange={(e) =>
+                                                            setSendEmail(
+                                                                e as boolean
+                                                            )
+                                                        }
+                                                    />
+                                                    <label
+                                                        htmlFor="send-email"
+                                                        className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
+                                                    >
+                                                        {t("inviteEmailSent")}
+                                                    </label>
+                                                </div>
+                                            )}
+
+                                            <FormField
+                                                control={internalForm.control}
+                                                name="validForHours"
+                                                render={({ field }) => (
+                                                    <FormItem>
+                                                        <FormLabel>
+                                                            {t("inviteValid")}
+                                                        </FormLabel>
+                                                        <Select
+                                                            onValueChange={
+                                                                field.onChange
+                                                            }
+                                                            defaultValue={
+                                                                field.value
+                                                            }
+                                                        >
+                                                            <FormControl>
+                                                                <SelectTrigger className="w-full">
+                                                                    <SelectValue
+                                                                        placeholder={t(
+                                                                            "selectDuration"
+                                                                        )}
+                                                                    />
+                                                                </SelectTrigger>
+                                                            </FormControl>
+                                                            <SelectContent>
+                                                                {validFor.map(
+                                                                    (
+                                                                        option
+                                                                    ) => (
+                                                                        <SelectItem
+                                                                            key={
+                                                                                option.hours
+                                                                            }
+                                                                            value={option.hours.toString()}
+                                                                        >
+                                                                            {
+                                                                                option.name
+                                                                            }
+                                                                        </SelectItem>
+                                                                    )
+                                                                )}
+                                                            </SelectContent>
+                                                        </Select>
+                                                        <FormMessage />
+                                                    </FormItem>
+                                                )}
+                                            />
+
+                                            <OrgRolesTagField
+                                                form={internalForm}
+                                                name="roles"
+                                                orgId={orgId as string}
+                                                supportsMultipleRolesPerUser={
+                                                    supportsMultipleRolesPerUser
+                                                }
+                                                showMultiRolePaywallMessage={
+                                                    showMultiRolePaywallMessage
+                                                }
+                                                paywallMessage={
+                                                    invitePaywallMessage
+                                                }
+                                            />
+                                        </form>
+                                    </Form>
+                                </SettingsSectionForm>
+                            </SettingsSectionBody>
+                        </SettingsSection>
                     )}
 
                     {selectedOption &&
@@ -898,24 +858,56 @@ export default function Page() {
                 <div className="flex justify-end space-x-2 mt-8">
                     {selectedOption && dataLoaded && (
                         <Button
-                            type={inviteLink ? "button" : "submit"}
+                            type="submit"
-                            form={inviteLink ? undefined : "create-user-form"}
+                            form="create-user-form"
                             loading={loading}
                             disabled={loading}
-                            onClick={
-                                inviteLink
-                                    ? () =>
-                                          router.push(
-                                              `/${orgId}/settings/access/users`
-                                          )
-                                    : undefined
-                            }
                         >
-                            {inviteLink ? t("done") : t("accessUserCreate")}
+                            {t("accessUserCreate")}
                         </Button>
                     )}
                 </div>
             </div>
+
+            <Credenza
+                open={isInviteDialogOpen}
+                onOpenChange={(open) => {
+                    setIsInviteDialogOpen(open);
+                    if (!open) {
+                        setInviteLink(null);
+                    }
+                }}
+            >
+                <CredenzaContent>
+                    <CredenzaHeader>
+                        <CredenzaTitle>{t("userInvited")}</CredenzaTitle>
+                        <CredenzaDescription>
+                            {sendEmail
+                                ? t("inviteEmailSentDescription")
+                                : t("inviteSentDescription")}
+                        </CredenzaDescription>
+                    </CredenzaHeader>
+                    <CredenzaBody>
+                        <div className="space-y-4">
+                            <p>
+                                {t("inviteExpiresIn", {
+                                    days: expiresInDays
+                                })}
+                            </p>
+                            {inviteLink && <CopyTextBox text={inviteLink} />}
+                        </div>
+                    </CredenzaBody>
+                    <CredenzaFooter>
+                        <Button
+                            onClick={() =>
+                                router.push(`/${orgId}/settings/access/users`)
+                            }
+                        >
+                            {t("done")}
+                        </Button>
+                    </CredenzaFooter>
+                </CredenzaContent>
+            </Credenza>
         </>
     );
 }

src/app/[orgId]/settings/access/users/page.tsx

@@ -78,12 +78,13 @@ export default async function UsersPage(props: UsersPageProps) {
         rolesRes && rolesRes.status === 200
             ? (rolesRes.data.data.roles ?? [])
             : [];
-    const roleFilterOptions = orgRoles.map(
+    const roleFilterOptions = [
-        (r: ListRolesResponse["roles"][number]) => ({
+        { value: "owner", label: t("accessRoleOwner") },
+        ...orgRoles.map((r: ListRolesResponse["roles"][number]) => ({
             value: String(r.roleId),
             label: r.name
-        })
+        }))
-    );
+    ];
 
     const invitationsRes = await internal
         .get(
@@ -126,14 +127,12 @@ export default async function UsersPage(props: UsersPageProps) {
             idpId: user.idpId,
             idpName: user.idpName || t("idpNameInternal"),
             status: t("userConfirmed"),
-            roleLabels: user.isOwner
+            roleLabels: (() => {
-                ? [t("accessRoleOwner")]
+                const names = (user.roles ?? [])
-                : (() => {
+                    .map((r) => r.roleName)
-                      const names = (user.roles ?? [])
+                    .filter((n): n is string => Boolean(n?.length));
-                          .map((r) => r.roleName)
+                return names.length ? names : [t("accessRoleMember")];
-                          .filter((n): n is string => Boolean(n?.length));
+            })(),
-                      return names.length ? names : [t("accessRoleMember")];
-                  })(),
             isOwner: user.isOwner || false
         };
     });

src/app/[orgId]/settings/api-keys/create/page.tsx

@@ -43,10 +43,18 @@ import {
 } from "@app/components/InfoSection";
 import CopyToClipboard from "@app/components/CopyToClipboard";
 import moment from "moment";
-import CopyCodeBox from "@server/emails/templates/components/CopyCodeBox";
 import CopyTextBox from "@app/components/CopyTextBox";
 import PermissionsSelectBox from "@app/components/PermissionsSelectBox";
 import { useTranslations } from "next-intl";
+import {
+    Credenza,
+    CredenzaBody,
+    CredenzaContent,
+    CredenzaDescription,
+    CredenzaFooter,
+    CredenzaHeader,
+    CredenzaTitle
+} from "@app/components/Credenza";
 
 export default function Page() {
     const { env } = useEnvContext();
@@ -58,6 +66,7 @@ export default function Page() {
     const [loadingPage, setLoadingPage] = useState(true);
     const [createLoading, setCreateLoading] = useState(false);
     const [apiKey, setApiKey] = useState<CreateOrgApiKeyResponse | null>(null);
+    const [isApiKeyDialogOpen, setIsApiKeyDialogOpen] = useState(false);
     const [selectedPermissions, setSelectedPermissions] = useState<
         Record<string, boolean>
     >({});
@@ -75,22 +84,6 @@ export default function Page() {
 
     type CreateFormValues = z.infer<typeof createFormSchema>;
 
-    const copiedFormSchema = z
-        .object({
-            copied: z.boolean()
-        })
-        .refine(
-            (data) => {
-                return data.copied;
-            },
-            {
-                message: t("apiKeysConfirmCopy2"),
-                path: ["copied"]
-            }
-        );
-
-    type CopiedFormValues = z.infer<typeof copiedFormSchema>;
-
     const form = useForm({
         resolver: zodResolver(createFormSchema),
         defaultValues: {
@@ -98,12 +91,9 @@ export default function Page() {
         }
     });
 
-    const copiedForm = useForm({
+    function goToApiKeysList() {
-        resolver: zodResolver(copiedFormSchema),
+        router.push(`/${orgId}/settings/api-keys`);
-        defaultValues: {
+    }
-            copied: true
-        }
-    });
 
     async function onSubmit(data: CreateFormValues) {
         setCreateLoading(true);
@@ -113,9 +103,10 @@ export default function Page() {
         };
 
         const res = await api
-            .put<
+            .put<AxiosResponse<CreateOrgApiKeyResponse>>(
-                AxiosResponse<CreateOrgApiKeyResponse>
+                `/org/${orgId}/api-key/`,
-            >(`/org/${orgId}/api-key/`, payload)
+                payload
+            )
             .catch((e) => {
                 toast({
                     variant: "destructive",
@@ -125,16 +116,10 @@ export default function Page() {
             });
 
         if (res && res.status === 201) {
-            const data = res.data.data;
+            const created = res.data.data;
-
-            console.log({
-                actionIds: Object.keys(selectedPermissions).filter(
-                    (key) => selectedPermissions[key]
-                )
-            });
 
             const actionsRes = await api
-                .post(`/org/${orgId}/api-key/${data.apiKeyId}/actions`, {
+                .post(`/org/${orgId}/api-key/${created.apiKeyId}/actions`, {
                     actionIds: Object.keys(selectedPermissions).filter(
                         (key) => selectedPermissions[key]
                     )
@@ -149,27 +134,14 @@ export default function Page() {
                 });
 
             if (actionsRes) {
-                setApiKey(data);
+                setApiKey(created);
+                setIsApiKeyDialogOpen(true);
             }
         }
 
         setCreateLoading(false);
     }
 
-    async function onCopiedSubmit(data: CopiedFormValues) {
-        if (!data.copied) {
-            return;
-        }
-
-        router.push(`/${orgId}/settings/api-keys`);
-    }
-
-    const formatLabel = (str: string) => {
-        return str
-            .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
-            .replace(/^./, (char) => char.toUpperCase());
-    };
-
     useEffect(() => {
         const load = async () => {
             setLoadingPage(false);
@@ -185,12 +157,7 @@ export default function Page() {
                     title={t("apiKeysCreate")}
                     description={t("apiKeysCreateDescription")}
                 />
-                <Button
+                <Button variant="outline" onClick={goToApiKeysList}>
-                    variant="outline"
-                    onClick={() => {
-                        router.push(`/${orgId}/settings/api-keys`);
-                    }}
-                >
                     {t("apiKeysSeeAll")}
                 </Button>
             </div>
@@ -198,206 +165,152 @@ export default function Page() {
             {!loadingPage && (
                 <div>
                     <SettingsContainer>
-                        {!apiKey && (
+                        <SettingsSection>
-                            <>
+                            <SettingsSectionHeader>
-                                <SettingsSection>
+                                <SettingsSectionTitle>
-                                    <SettingsSectionHeader>
+                                    {t("apiKeysTitle")}
-                                        <SettingsSectionTitle>
+                                </SettingsSectionTitle>
-                                            {t("apiKeysTitle")}
+                            </SettingsSectionHeader>
-                                        </SettingsSectionTitle>
+                            <SettingsSectionBody>
-                                    </SettingsSectionHeader>
+                                <SettingsSectionForm>
-                                    <SettingsSectionBody>
+                                    <Form {...form}>
-                                        <SettingsSectionForm>
+                                        <form
-                                            <Form {...form}>
+                                            onKeyDown={(e) => {
-                                                <form
+                                                if (e.key === "Enter") {
-                                                    onKeyDown={(e) => {
+                                                    e.preventDefault();
-                                                        if (e.key === "Enter") {
+                                                }
-                                                            e.preventDefault(); // block default enter refresh
+                                            }}
-                                                        }
+                                            className="space-y-4"
-                                                    }}
+                                            id="create-site-form"
-                                                    className="space-y-4"
+                                        >
-                                                    id="create-site-form"
+                                            <FormField
-                                                >
+                                                control={form.control}
-                                                    <FormField
+                                                name="name"
-                                                        control={form.control}
+                                                render={({ field }) => (
-                                                        name="name"
+                                                    <FormItem>
-                                                        render={({ field }) => (
+                                                        <FormLabel>
-                                                            <FormItem>
+                                                            {t("name")}
-                                                                <FormLabel>
+                                                        </FormLabel>
-                                                                    {t("name")}
+                                                        <FormControl>
-                                                                </FormLabel>
+                                                            <Input
-                                                                <FormControl>
+                                                                autoComplete="off"
-                                                                    <Input
+                                                                {...field}
-                                                                        autoComplete="off"
+                                                            />
-                                                                        {...field}
+                                                        </FormControl>
-                                                                    />
+                                                        <FormMessage />
-                                                                </FormControl>
+                                                    </FormItem>
-                                                                <FormMessage />
+                                                )}
-                                                            </FormItem>
+                                            />
-                                                        )}
+                                        </form>
-                                                    />
+                                    </Form>
-                                                </form>
+                                </SettingsSectionForm>
-                                            </Form>
+                            </SettingsSectionBody>
-                                        </SettingsSectionForm>
+                        </SettingsSection>
-                                    </SettingsSectionBody>
-                                </SettingsSection>
 
-                                <SettingsSection>
+                        <SettingsSection>
-                                    <SettingsSectionHeader>
+                            <SettingsSectionHeader>
-                                        <SettingsSectionTitle>
+                                <SettingsSectionTitle>
-                                            {t("apiKeysGeneralSettings")}
+                                    {t("apiKeysGeneralSettings")}
-                                        </SettingsSectionTitle>
+                                </SettingsSectionTitle>
-                                        <SettingsSectionDescription>
+                                <SettingsSectionDescription>
-                                            {t(
+                                    {t("apiKeysGeneralSettingsDescription")}
-                                                "apiKeysGeneralSettingsDescription"
+                                </SettingsSectionDescription>
-                                            )}
+                            </SettingsSectionHeader>
-                                        </SettingsSectionDescription>
+                            <SettingsSectionBody>
-                                    </SettingsSectionHeader>
+                                <PermissionsSelectBox
-                                    <SettingsSectionBody>
+                                    selectedPermissions={selectedPermissions}
-                                        <PermissionsSelectBox
+                                    onChange={setSelectedPermissions}
-                                            selectedPermissions={
+                                />
-                                                selectedPermissions
+                            </SettingsSectionBody>
-                                            }
+                        </SettingsSection>
-                                            onChange={setSelectedPermissions}
-                                        />
-                                    </SettingsSectionBody>
-                                </SettingsSection>
-                            </>
-                        )}
-
-                        {apiKey && (
-                            <SettingsSection>
-                                <SettingsSectionHeader>
-                                    <SettingsSectionTitle>
-                                        {t("apiKeysList")}
-                                    </SettingsSectionTitle>
-                                </SettingsSectionHeader>
-                                <SettingsSectionBody>
-                                    <InfoSections cols={2}>
-                                        <InfoSection>
-                                            <InfoSectionTitle>
-                                                {t("name")}
-                                            </InfoSectionTitle>
-                                            <InfoSectionContent>
-                                                <CopyToClipboard
-                                                    text={apiKey.name}
-                                                />
-                                            </InfoSectionContent>
-                                        </InfoSection>
-                                        <InfoSection>
-                                            <InfoSectionTitle>
-                                                {t("created")}
-                                            </InfoSectionTitle>
-                                            <InfoSectionContent>
-                                                {moment(
-                                                    apiKey.createdAt
-                                                ).format("lll")}
-                                            </InfoSectionContent>
-                                        </InfoSection>
-                                    </InfoSections>
-
-                                    <Alert variant="neutral">
-                                        <InfoIcon className="h-4 w-4" />
-                                        <AlertTitle className="font-semibold">
-                                            {t("apiKeysSave")}
-                                        </AlertTitle>
-                                        <AlertDescription>
-                                            {t("apiKeysSaveDescription")}
-                                        </AlertDescription>
-                                    </Alert>
-
-                                    {/* <h4 className="font-semibold"> */}
-                                    {/*     {t('apiKeysInfo')} */}
-                                    {/* </h4> */}
-
-                                    <CopyTextBox
-                                        text={`${apiKey.apiKeyId}.${apiKey.apiKey}`}
-                                    />
-
-                                    {/* <Form {...copiedForm}> */}
-                                    {/*     <form */}
-                                    {/*         className="space-y-4" */}
-                                    {/*         id="copied-form" */}
-                                    {/*     > */}
-                                    {/*         <FormField */}
-                                    {/*             control={copiedForm.control} */}
-                                    {/*             name="copied" */}
-                                    {/*             render={({ field }) => ( */}
-                                    {/*                 <FormItem> */}
-                                    {/*                     <div className="flex items-center space-x-2"> */}
-                                    {/*                         <Checkbox */}
-                                    {/*                             id="terms" */}
-                                    {/*                             defaultChecked={ */}
-                                    {/*                                 copiedForm.getValues( */}
-                                    {/*                                     "copied" */}
-                                    {/*                                 ) as boolean */}
-                                    {/*                             } */}
-                                    {/*                             onCheckedChange={( */}
-                                    {/*                                 e */}
-                                    {/*                             ) => { */}
-                                    {/*                                 copiedForm.setValue( */}
-                                    {/*                                     "copied", */}
-                                    {/*                                     e as boolean */}
-                                    {/*                                 ); */}
-                                    {/*                             }} */}
-                                    {/*                         /> */}
-                                    {/*                         <label */}
-                                    {/*                             htmlFor="terms" */}
-                                    {/*                             className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70" */}
-                                    {/*                         > */}
-                                    {/*                             {t('apiKeysConfirmCopy')} */}
-                                    {/*                         </label> */}
-                                    {/*                     </div> */}
-                                    {/*                     <FormMessage /> */}
-                                    {/*                 </FormItem> */}
-                                    {/*             )} */}
-                                    {/*         /> */}
-                                    {/*     </form> */}
-                                    {/* </Form> */}
-                                </SettingsSectionBody>
-                            </SettingsSection>
-                        )}
                     </SettingsContainer>
 
                     <div className="flex justify-end space-x-2 mt-8">
-                        {!apiKey && (
+                        <Button
-                            <Button
+                            type="button"
-                                type="button"
+                            variant="outline"
-                                variant="outline"
+                            disabled={createLoading || apiKey !== null}
-                                disabled={createLoading || apiKey !== null}
+                            onClick={goToApiKeysList}
-                                onClick={() => {
+                        >
-                                    router.push(`/${orgId}/settings/api-keys`);
+                            {t("cancel")}
-                                }}
+                        </Button>
-                            >
+                        <Button
-                                {t("cancel")}
+                            type="button"
-                            </Button>
+                            loading={createLoading}
-                        )}
+                            disabled={createLoading || apiKey !== null}
-                        {!apiKey && (
+                            onClick={() => {
-                            <Button
+                                form.handleSubmit(onSubmit)();
-                                type="button"
+                            }}
-                                loading={createLoading}
+                        >
-                                disabled={createLoading || apiKey !== null}
+                            {t("generate")}
-                                onClick={() => {
+                        </Button>
-                                    form.handleSubmit(onSubmit)();
-                                }}
-                            >
-                                {t("generate")}
-                            </Button>
-                        )}
-
-                        {apiKey && (
-                            <Button
-                                type="button"
-                                onClick={() => {
-                                    copiedForm.handleSubmit(onCopiedSubmit)();
-                                }}
-                            >
-                                {t("done")}
-                            </Button>
-                        )}
                     </div>
                 </div>
             )}
+
+            <Credenza
+                open={isApiKeyDialogOpen}
+                onOpenChange={(open) => {
+                    setIsApiKeyDialogOpen(open);
+                    if (!open && apiKey) {
+                        goToApiKeysList();
+                    }
+                }}
+            >
+                <CredenzaContent>
+                    <CredenzaHeader>
+                        <CredenzaTitle>{t("apiKeysList")}</CredenzaTitle>
+                        <CredenzaDescription>
+                            {t("apiKeysSaveDescription")}
+                        </CredenzaDescription>
+                    </CredenzaHeader>
+                    <CredenzaBody>
+                        {apiKey && (
+                            <div className="space-y-4">
+                                <InfoSections cols={2}>
+                                    <InfoSection>
+                                        <InfoSectionTitle>
+                                            {t("name")}
+                                        </InfoSectionTitle>
+                                        <InfoSectionContent>
+                                            <CopyToClipboard
+                                                text={apiKey.name}
+                                            />
+                                        </InfoSectionContent>
+                                    </InfoSection>
+                                    <InfoSection>
+                                        <InfoSectionTitle>
+                                            {t("created")}
+                                        </InfoSectionTitle>
+                                        <InfoSectionContent>
+                                            {moment(apiKey.createdAt).format(
+                                                "lll"
+                                            )}
+                                        </InfoSectionContent>
+                                    </InfoSection>
+                                </InfoSections>
+
+                                <Alert variant="neutral">
+                                    <InfoIcon className="h-4 w-4" />
+                                    <AlertTitle className="font-semibold">
+                                        {t("apiKeysSave")}
+                                    </AlertTitle>
+                                    <AlertDescription>
+                                        {t("apiKeysSaveDescription")}
+                                    </AlertDescription>
+                                </Alert>
+
+                                <CopyTextBox
+                                    text={`${apiKey.apiKeyId}.${apiKey.apiKey}`}
+                                />
+                            </div>
+                        )}
+                    </CredenzaBody>
+                    <CredenzaFooter>
+                        <Button onClick={goToApiKeysList}>{t("done")}</Button>
+                    </CredenzaFooter>
+                </CredenzaContent>
+            </Credenza>
         </>
     );
 }

src/app/[orgId]/settings/clients/user/[niceId]/general/page.tsx

@@ -19,7 +19,6 @@ import {
     InfoSections,
     InfoSectionTitle
 } from "@app/components/InfoSection";
-import { Badge } from "@app/components/ui/badge";
 import { Button } from "@app/components/ui/button";
 import ActionBanner from "@app/components/ActionBanner";
 import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert";
@@ -41,13 +40,6 @@ import { useParams } from "next/navigation";
 import { FaApple, FaWindows, FaLinux } from "react-icons/fa";
 import { SiAndroid } from "react-icons/si";
 import { tierMatrix } from "@server/lib/billing/tierMatrix";
-import {
-    productUpdatesQueries,
-    type LatestVersionResponse
-} from "@app/lib/queries";
-import { useQuery } from "@tanstack/react-query";
-import semver from "semver";
-import { InfoPopup } from "@app/components/ui/info-popup";
 
 function formatTimestamp(timestamp: number | null | undefined): string {
     if (!timestamp) return "-";
@@ -173,34 +165,6 @@ export default function GeneralPage() {
     }>(null);
     const [isCheckingCache, setIsCheckingCache] = useState(false);
     const [isRebuildingCache, setIsRebuildingCache] = useState(false);
-    const data = useQuery(productUpdatesQueries.latestVersion(true));
-    const latestPlatformVersions = data.data?.data;
-
-    const agentVersionMap: Record<string, string> = {
-        "Pangolin Windows": "windows",
-        "Pangolin Android": "android",
-        "Pangolin iOS": "ios",
-        "Pangolin iPadOS": "ios",
-        "Pangolin macOS": "mac",
-        "Pangolin CLI": "cli",
-        "Olm CLI": "olm"
-    };
-
-    let updateAvailable = false;
-    if (client.agent && client.olmVersion && latestPlatformVersions) {
-        const agent = agentVersionMap[
-            client.agent
-        ] as keyof LatestVersionResponse;
-
-        if (agent in latestPlatformVersions) {
-            const agentVersion = latestPlatformVersions[agent];
-
-            updateAvailable = Boolean(
-                semver.valid(client.olmVersion) &&
-                semver.lt(client.olmVersion, agentVersion.latestVersion)
-            );
-        }
-    }
 
     // get "imp" from local storage to determine if we should show the verify button (imp = "1" means show)
     const showVerifyButton =
@@ -467,7 +431,7 @@ export default function GeneralPage() {
             )}
 
             {/* Device Information Section */}
-            {(client.fingerprint || (client.agent && client.olmVersion)) && (
+            {client.fingerprint && (
                 <SettingsSection>
                     <SettingsSectionHeader>
                         <SettingsSectionTitle>
@@ -479,33 +443,6 @@ export default function GeneralPage() {
                     </SettingsSectionHeader>
 
                     <SettingsSectionBody>
-                        {client.agent && client.olmVersion && (
-                            <div className="mb-6">
-                                <InfoSection>
-                                    <InfoSectionTitle>
-                                        {t("agent")}
-                                    </InfoSectionTitle>
-                                    <InfoSectionContent>
-                                        <div className="flex items-center">
-                                            <Badge variant="secondary">
-                                                {client.agent +
-                                                    " v" +
-                                                    client.olmVersion}
-                                            </Badge>
-
-                                            {updateAvailable && (
-                                                <InfoPopup
-                                                    info={t(
-                                                        "updateAvailableInfo"
-                                                    )}
-                                                />
-                                            )}
-                                        </div>
-                                    </InfoSectionContent>
-                                </InfoSection>
-                            </div>
-                        )}
-
                         {client.fingerprint &&
                             (() => {
                                 const platform = client.fingerprint.platform;

src/app/[orgId]/settings/provisioning/pending/page.tsx

@@ -74,6 +74,8 @@ export default async function PendingSitesPage(props: PendingSitesPageProps) {
         type: site.type as any,
         online: site.online,
         newtVersion: site.newtVersion || undefined,
+        agent: site.agent || undefined,
+        agentVersion: site.agentVersion || undefined,
         newtUpdateAvailable: site.newtUpdateAvailable || false,
         exitNodeName: site.exitNodeName || undefined,
         exitNodeEndpoint: site.exitNodeEndpoint || undefined,

src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx

@@ -954,7 +954,6 @@ export const ProxyResourceTargetsForm = forwardRef<
                                 colSpan={columns.length}
                                 message={emptyMessage ?? t("targetNoOne")}
                                 action={addTargetButton}
-                                compact
                             />
                         )}
                     </TableBody>