Add some retry and database confict mitigation

server/lib/rebuildClientAssociations.ts

@@ -485,6 +485,7 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
         await trx
             .insert(clientSiteResourcesAssociationsCache)
             .values(clientSiteResourcesToInsert)
+            .onConflictDoNothing()
             .returning();
         logger.debug(
             `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteResourceId=${siteResource.siteResourceId} inserted clientSiteResource associations`
@@ -532,6 +533,7 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
     for (const site of sitesToProcess) {
         const siteId = site.siteId;
 
+        try {
             logger.debug(
                 `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] processing siteId=${siteId} for siteResourceId=${siteResource.siteResourceId}`
             );
@@ -561,11 +563,14 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
                               subnet: clients.subnet
                           })
                           .from(clients)
-                      .where(inArray(clients.clientId, existingClientSiteIds))
+                          .where(
+                              inArray(clients.clientId, existingClientSiteIds)
+                          )
                     : [];
 
             const otherResourceClientIds =
-            clientsFromOtherResourcesBySite.get(siteId) ?? new Set<number>();
+                clientsFromOtherResourcesBySite.get(siteId) ??
+                new Set<number>();
 
             logger.debug(
                 `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} otherResourceClientIds=[${[...otherResourceClientIds].join(", ")}] mergedAllClientIds=[${mergedAllClientIds.join(", ")}]`
@@ -599,6 +604,7 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
                 await trx
                     .insert(clientSitesAssociationsCache)
                     .values(clientSitesToInsert)
+                    .onConflictDoNothing()
                     .returning();
                 logger.debug(
                     `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} inserted clientSite associations`
@@ -647,6 +653,21 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
                 clientSitesToRemove,
                 trx
             );
+        } catch (err) {
+            // Don't let a failure on one site abort processing of every
+            // other site queued after it in this run. Since we're not
+            // re-throwing, the outer wrapper's retry/requeue logic never
+            // sees this failure, so explicitly queue this resource for a
+            // follow-up pass to reconcile whatever this site didn't get to.
+            logger.error(
+                `rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} failed while processing site for siteResourceId=${siteResource.siteResourceId}, continuing with remaining sites and queuing a follow-up pass:`,
+                err
+            );
+            await rebuildQueue.enqueue({
+                type: "site-resource",
+                id: siteResource.siteResourceId
+            });
+        }
     }
 
     // Handle subnet proxy target updates for the resource associations
@@ -939,7 +960,7 @@ export async function updateClientSiteDestinations(
 
     for (const site of sitesData) {
         if (!site.sites.subnet) {
-            logger.warn(`Site ${site.sites.siteId} has no subnet, skipping`);
+            logger.debug(`Site ${site.sites.siteId} has no subnet, skipping`);
             continue;
         }
 
@@ -1866,12 +1887,15 @@ async function rebuildClientAssociationsFromClientImpl(
 
     // Insert new associations
     if (resourcesToAdd.length > 0) {
-        await trx.insert(clientSiteResourcesAssociationsCache).values(
+        await trx
+            .insert(clientSiteResourcesAssociationsCache)
+            .values(
                 resourcesToAdd.map((siteResourceId) => ({
                     clientId: client.clientId,
                     siteResourceId
                 }))
-        );
+            )
+            .onConflictDoNothing();
     }
 
     // Remove old associations
@@ -1909,12 +1933,15 @@ async function rebuildClientAssociationsFromClientImpl(
 
     // Insert new site associations
     if (sitesToAdd.length > 0) {
-        await trx.insert(clientSitesAssociationsCache).values(
+        await trx
+            .insert(clientSitesAssociationsCache)
+            .values(
                 sitesToAdd.map((siteId) => ({
                     clientId: client.clientId,
                     siteId
                 }))
-        );
+            )
+            .onConflictDoNothing();
     }
 
     // Remove old site associations

server/routers/newt/buildConfiguration.ts

@@ -52,13 +52,13 @@ export async function buildClientConfigurationForNewtClient(
             clientsRes
                 .filter((client) => {
                     if (!client.clients.pubKey) {
-                        logger.warn(
+                        logger.debug(
                             `Client ${client.clients.clientId} has no public key, skipping`
                         );
                         return false;
                     }
                     if (!client.clients.subnet) {
-                        logger.warn(
+                        logger.debug(
                             `Client ${client.clients.clientId} has no subnet, skipping`
                         );
                         return false;

server/routers/olm/buildConfiguration.ts

@@ -161,7 +161,7 @@ export async function buildSiteConfigurationForOlmClient(
         }
 
         if (!site.subnet) {
-            logger.warn(`Site ${site.siteId} has no subnet, skipping`);
+            logger.debug(`Site ${site.siteId} has no subnet, skipping`);
             continue;
         }