Compare commits

...

3 Commits

Author SHA1 Message Date
Owen 733d3ece0e Quiet up error logs 2026-09-08 10:01:59 -04:00
Owen 59b228ce39 Quiet log message 2026-09-08 09:26:57 -04:00
Owen 080bcbaf97 Use endpoint instead of reachableAt for remote nodes 2026-09-07 11:55:45 -04:00
6 changed files with 72 additions and 63 deletions
+1 -1
View File
@@ -71,7 +71,7 @@ export async function withRetry<T>(
const jitter = Math.random() * baseDelay;
const delay = baseDelay + jitter;
logger.warn(
`Transient DB error in ${context}, retrying attempt ${attempt}/${maxRetries} after ${delay.toFixed(0)}ms`,
`Transient DB issue in ${context}, retrying attempt ${attempt}/${maxRetries} after ${delay.toFixed(0)}ms`,
{ code: error?.code ?? error?.cause?.code }
);
await new Promise((resolve) => setTimeout(resolve, delay));
+6 -1
View File
@@ -2478,7 +2478,12 @@ hybridRouter.post(
destinations: destinations
});
} catch (error) {
logger.error(error);
if (!(
error instanceof Error &&
error.message === "Exit node not allowed"
)) {
logger.error(error);
}
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
@@ -16,7 +16,7 @@ import { db, exitNodes, newts, sites } from "@server/db";
import { eq } from "drizzle-orm";
import logger from "@server/logger";
import redisManager from "#private/lib/redis";
// import { sendToClient } from "#private/routers/ws";
import { sendToClient } from "#private/routers/ws";
const INITIAL_DELAY_MS = 15 * 1000; // 15 seconds before first check
const CHECK_INTERVAL_MS = 10 * 1000; // Check every 10 seconds
@@ -26,7 +26,7 @@ const REDIS_HASH_PREFIX = "exit-node-reconnect:";
interface PendingReconnect {
startTime: number;
reachableAt: string;
endpoint: string;
}
// In-memory tracking for this node
@@ -40,15 +40,15 @@ let schedulerInterval: NodeJS.Timeout | null = null;
*/
export async function scheduleExitNodeReconnect(
exitNodeId: number,
reachableAt: string
endpoint: string
): Promise<void> {
logger.info(
`Scheduling newt reconnect for exit node ${exitNodeId} (reachableAt: ${reachableAt})`
`Scheduling newt reconnect for exit node ${exitNodeId} (endpoint: ${endpoint})`
);
const entry: PendingReconnect = {
startTime: Date.now(),
reachableAt
endpoint
};
pendingReconnects.set(exitNodeId, entry);
@@ -63,8 +63,8 @@ export async function scheduleExitNodeReconnect(
);
await redisManager.hset(
`${REDIS_HASH_PREFIX}${exitNodeId}`,
"reachableAt",
reachableAt
"endpoint",
endpoint
);
}
}
@@ -101,14 +101,14 @@ async function processPendingReconnects(): Promise<void> {
`${REDIS_HASH_PREFIX}${id}`,
"startTime"
);
const reachableAt = await redisManager.hget(
const endpoint = await redisManager.hget(
`${REDIS_HASH_PREFIX}${id}`,
"reachableAt"
"endpoint"
);
if (startTimeStr && reachableAt) {
if (startTimeStr && endpoint) {
toProcess.set(id, {
startTime: parseInt(startTimeStr, 10),
reachableAt
endpoint
});
}
}
@@ -135,7 +135,7 @@ async function processPendingReconnects(): Promise<void> {
}
// Check if the exit node HTTP endpoint is reachable
const pingUrl = `${entry.reachableAt}/ping`;
const pingUrl = `http://${entry.endpoint}/ping`;
try {
await axios.get(pingUrl, { timeout: 5000 });
} catch {
@@ -150,47 +150,47 @@ async function processPendingReconnects(): Promise<void> {
`Exit node ${exitNodeId} is reachable. Sending newt/wg/reconnect to connected newts.`
);
// await sendReconnectToNewts(exitNodeId);
await sendReconnectToNewts(exitNodeId);
await removePending(exitNodeId);
}
}
// async function sendReconnectToNewts(exitNodeId: number): Promise<void> {
// try {
// const connectedNewts = await db
// .select({ newtId: newts.newtId })
// .from(newts)
// .innerJoin(sites, eq(newts.siteId, sites.siteId))
// .where(eq(sites.exitNodeId, exitNodeId));
async function sendReconnectToNewts(exitNodeId: number): Promise<void> {
try {
const connectedNewts = await db
.select({ newtId: newts.newtId })
.from(newts)
.innerJoin(sites, eq(newts.siteId, sites.siteId))
.where(eq(sites.exitNodeId, exitNodeId));
// if (connectedNewts.length === 0) {
// logger.debug(
// `No newts found for exit node ${exitNodeId}, nothing to reconnect`
// );
// return;
// }
if (connectedNewts.length === 0) {
logger.debug(
`No newts found for exit node ${exitNodeId}, nothing to reconnect`
);
return;
}
// logger.info(
// `Sending newt/wg/reconnect to ${connectedNewts.length} newt(s) for exit node ${exitNodeId}`
// );
logger.info(
`Sending newt/wg/reconnect to ${connectedNewts.length} newt(s) for exit node ${exitNodeId}`
);
// const reconnectMessage = {
// type: "newt/wg/reconnect",
// data: {}
// };
const reconnectMessage = {
type: "newt/wg/reconnect",
data: {}
};
// await Promise.allSettled(
// connectedNewts.map(({ newtId }) =>
// sendToClient(newtId, reconnectMessage)
// )
// );
// } catch (error) {
// logger.error(
// `Failed to send reconnect messages for exit node ${exitNodeId}`,
// { error }
// );
// }
// }
await Promise.allSettled(
connectedNewts.map(({ newtId }) =>
sendToClient(newtId, reconnectMessage)
)
);
} catch (error) {
logger.error(
`Failed to send reconnect messages for exit node ${exitNodeId}`,
{ error }
);
}
}
async function removePending(exitNodeId: number): Promise<void> {
pendingReconnects.delete(exitNodeId);
@@ -40,7 +40,7 @@ export const handleRemoteExitNodePingMessage: MessageHandler = async (
try {
// Fetch the current state before updating so we can detect the offline→online transition
const [currentExitNode] = await db
.select({ online: exitNodes.online, reachableAt: exitNodes.reachableAt })
.select({ online: exitNodes.online, endpoint: exitNodes.endpoint })
.from(exitNodes)
.where(eq(exitNodes.exitNodeId, remoteExitNode.exitNodeId))
.limit(1);
@@ -55,12 +55,18 @@ export const handleRemoteExitNodePingMessage: MessageHandler = async (
.where(eq(exitNodes.exitNodeId, remoteExitNode.exitNodeId));
// If the exit node was offline and is now coming online, schedule newt reconnects
if (currentExitNode && !currentExitNode.online && currentExitNode.reachableAt) {
if (
currentExitNode &&
!currentExitNode.online &&
currentExitNode.endpoint
) {
scheduleExitNodeReconnect(
remoteExitNode.exitNodeId,
currentExitNode.reachableAt
currentExitNode.endpoint
).catch((error) => {
logger.error("Failed to schedule exit node reconnect", { error });
logger.error("Failed to schedule exit node reconnect", {
error
});
});
}
} catch (error) {
@@ -85,7 +85,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
);
if (!resources || resources.length === 0) {
logger.error(
logger.warn(
`handleOlmServerInitAddPeerHandshake: Resource not found`
);
await sendCancel();
@@ -94,7 +94,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
if (resources.length > 1) {
// error but this should not happen because the nice id cant contain a dot and the alias has to have a dot and both have to be unique within the org so there should never be multiple matches
logger.error(
logger.warn(
`handleOlmServerInitAddPeerHandshake: Multiple resources found matching the criteria`
);
return;
@@ -119,7 +119,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
);
if (currentResourceAssociationCaches.length === 0) {
logger.error(
logger.warn(
`handleOlmServerInitAddPeerHandshake: Client ${client.clientId} does not have access to resource ${resource.siteResourceId}`
);
await sendCancel();
@@ -127,7 +127,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
}
if (!resource.networkId) {
logger.error(
logger.warn(
`handleOlmServerInitAddPeerHandshake: Resource ${resource.siteResourceId} has no network`
);
await sendCancel();
@@ -141,7 +141,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
.where(eq(siteNetworks.networkId, resource.networkId));
if (!siteRows || siteRows.length === 0) {
logger.error(
logger.warn(
`handleOlmServerInitAddPeerHandshake: No sites found for resource ${resource.siteResourceId}`
);
await sendCancel();
@@ -164,9 +164,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
}
if (sitesToProcess.length === 0) {
logger.error(
`handleOlmServerInitAddPeerHandshake: No sites to process`
);
logger.warn(`handleOlmServerInitAddPeerHandshake: No sites to process`);
await sendCancel();
return;
}
@@ -193,7 +191,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
}
if (!site.exitNodeId) {
logger.error(
logger.warn(
`handleOlmServerInitAddPeerHandshake: Site ${site.siteId} has no exit node, skipping`
);
continue;
@@ -205,7 +203,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
.where(eq(exitNodes.exitNodeId, site.exitNodeId));
if (!exitNode) {
logger.error(
logger.warn(
`handleOlmServerInitAddPeerHandshake: Exit node not found for site ${site.siteId}, skipping`
);
continue;
@@ -229,7 +227,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
}
if (!handshakeInitiated) {
logger.error(
logger.warn(
`handleOlmServerInitAddPeerHandshake: No accessible sites with valid exit nodes found, cancelling chain`
);
await sendCancel();
+1 -1
View File
@@ -104,7 +104,7 @@ export default async function OrgLayout(props: {
subscriptionStatus = subRes.data.data;
} catch (error) {
// If subscription fetch fails, keep subscriptionStatus as null
console.error("Failed to fetch subscription status:", error);
// console.error("Failed to fetch subscription status:", error);
}
}