From 80dcdfe2512563b2cd42913fa923da49cd466986 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 16:08:13 -0400 Subject: [PATCH 1/5] change override to work for bother badger and ai gateway --- server/lib/readConfigFile.ts | 2 +- server/routers/traefik/traefikConfigProvider.ts | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/server/lib/readConfigFile.ts b/server/lib/readConfigFile.ts index 5774456a7..97dda31a2 100644 --- a/server/lib/readConfigFile.ts +++ b/server/lib/readConfigFile.ts @@ -79,7 +79,7 @@ export const configSchema = z .default(3001) .transform(stoi) .pipe(portSchema), - badger_override: z.string().optional(), + internal_api_override: z.string().optional(), next_port: portSchema .optional() .default(3002) diff --git a/server/routers/traefik/traefikConfigProvider.ts b/server/routers/traefik/traefikConfigProvider.ts index 898ac42ee..c2e88709b 100644 --- a/server/routers/traefik/traefikConfigProvider.ts +++ b/server/routers/traefik/traefikConfigProvider.ts @@ -20,9 +20,13 @@ export async function traefikConfigProvider( const maintenancePort = config.getRawConfig().server.next_port; const maintenanceHost = config.getRawConfig().server.internal_hostname; const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`; - const aiGatewayUrl = `http://${maintenanceHost}:${ - config.getRawConfig().server.internal_port - }/api/v1/ai-gateway`; + const aiGatewayUrl = + `${ + config.getRawConfig().server.internal_api_override + }/ai-gateway` || + `http://${maintenanceHost}:${ + config.getRawConfig().server.internal_port + }/api/v1/ai-gateway`; const traefikConfig = await getTraefikConfig( currentExitNodeId, @@ -41,7 +45,8 @@ export async function traefikConfigProvider( plugin: { [badgerMiddlewareName]: { apiBaseUrl: - config.getRawConfig().server.badger_override || + config.getRawConfig().server + .internal_api_override || new URL( "/api/v1", `http://${ From aad26b9ae40eefc819749f3aaf3980534f39b48c Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 16:52:48 -0400 Subject: [PATCH 2/5] move the ai gateway to its own server --- server/aiGatewayServer.ts | 39 +++++++++++++++++++ server/index.ts | 3 ++ server/lib/readConfigFile.ts | 9 ++++- server/lib/traefik/TraefikConfigManager.ts | 8 ++-- server/routers/internal.ts | 7 ---- .../routers/traefik/traefikConfigProvider.ts | 11 ++---- 6 files changed, 59 insertions(+), 18 deletions(-) create mode 100644 server/aiGatewayServer.ts diff --git a/server/aiGatewayServer.ts b/server/aiGatewayServer.ts new file mode 100644 index 000000000..db8b12324 --- /dev/null +++ b/server/aiGatewayServer.ts @@ -0,0 +1,39 @@ +import express from "express"; +import helmet from "helmet"; +import cors from "cors"; +import config from "@server/lib/config"; +import logger from "@server/logger"; +import { + errorHandlerMiddleware, + notFoundMiddleware +} from "@server/middlewares"; +import * as aiGateway from "@server/routers/aiGateway"; + +const aiGatewayPort = config.getRawConfig().server.ai_gateway_port; + +export function createAiGatewayServer() { + const aiGatewayServer = express(); + + const trustProxy = config.getRawConfig().server.trust_proxy; + if (trustProxy) { + aiGatewayServer.set("trust proxy", trustProxy); + } + + aiGatewayServer.use(helmet()); + aiGatewayServer.use(cors()); + aiGatewayServer.use(express.json()); + + aiGatewayServer.post("/chat/completions", aiGateway.chatCompletions); + + aiGatewayServer.use(notFoundMiddleware); + aiGatewayServer.use(errorHandlerMiddleware); + + aiGatewayServer.listen(aiGatewayPort, (err?: any) => { + if (err) throw err; + logger.info( + `AI gateway server is running on http://localhost:${aiGatewayPort}` + ); + }); + + return aiGatewayServer; +} diff --git a/server/index.ts b/server/index.ts index 88612b86a..3d9306d2b 100644 --- a/server/index.ts +++ b/server/index.ts @@ -5,6 +5,7 @@ import { runSetupFunctions } from "./setup"; import { createApiServer } from "./apiServer"; import { createNextServer } from "./nextServer"; import { createInternalServer } from "./internalServer"; +import { createAiGatewayServer } from "./aiGatewayServer"; import { createIntegrationApiServer } from "./integrationApiServer"; import { ApiKey, @@ -49,6 +50,7 @@ async function startServers() { // Start all servers const apiServer = createApiServer(); const internalServer = createInternalServer(); + const aiGatewayServer = createAiGatewayServer(); const nextServer = await createNextServer(); if (config.getRawConfig().traefik.file_mode) { @@ -67,6 +69,7 @@ async function startServers() { apiServer, nextServer, internalServer, + aiGatewayServer, integrationServer }; } diff --git a/server/lib/readConfigFile.ts b/server/lib/readConfigFile.ts index 97dda31a2..37418ed2f 100644 --- a/server/lib/readConfigFile.ts +++ b/server/lib/readConfigFile.ts @@ -79,7 +79,13 @@ export const configSchema = z .default(3001) .transform(stoi) .pipe(portSchema), - internal_api_override: z.string().optional(), + ai_gateway_port: portSchema + .optional() + .default(3005) + .transform(stoi) + .pipe(portSchema), + badger_override: z.string().optional(), + ai_gateway_override: z.string().optional(), next_port: portSchema .optional() .default(3002) @@ -139,6 +145,7 @@ export const configSchema = z integration_port: 3003, external_port: 3000, internal_port: 3001, + ai_gateway_port: 3005, next_port: 3002, internal_hostname: "pangolin", session_cookie_name: "p_session_token", diff --git a/server/lib/traefik/TraefikConfigManager.ts b/server/lib/traefik/TraefikConfigManager.ts index 73d46ee64..a52c5232a 100644 --- a/server/lib/traefik/TraefikConfigManager.ts +++ b/server/lib/traefik/TraefikConfigManager.ts @@ -516,9 +516,11 @@ export class TraefikConfigManager { const maintenanceHost = config.getRawConfig().server.internal_hostname; const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`; - const aiGatewayUrl = `http://${maintenanceHost}:${ - config.getRawConfig().server.internal_port - }/api/v1/ai-gateway`; + const aiGatewayUrl = + config.getRawConfig().server.ai_gateway_override || + `http://${maintenanceHost}:${ + config.getRawConfig().server.ai_gateway_port + }`; // logger.debug(`Fetching traefik config for exit node: ${currentExitNode}`); traefikConfig = await getTraefikConfig( diff --git a/server/routers/internal.ts b/server/routers/internal.ts index af12d4429..b06015a5f 100644 --- a/server/routers/internal.ts +++ b/server/routers/internal.ts @@ -3,7 +3,6 @@ import * as gerbil from "@server/routers/gerbil"; import * as traefik from "@server/routers/traefik"; import * as resource from "./resource"; import * as badger from "./badger"; -import * as aiGateway from "@server/routers/aiGateway"; import * as auth from "@server/routers/auth"; import * as supporterKey from "@server/routers/supporterKey"; import * as idp from "@server/routers/idp"; @@ -65,9 +64,3 @@ badgerRouter.post("/verify-session", badger.verifyResourceSession); badgerRouter.post("/exchange-session", badger.exchangeSession); -// AI inference gateway - minimal chat-completions proxy for inference-mode -// resources/siteResources -internalRouter.post( - "/ai-gateway/chat/completions", - aiGateway.chatCompletions -); diff --git a/server/routers/traefik/traefikConfigProvider.ts b/server/routers/traefik/traefikConfigProvider.ts index c2e88709b..e1847d2ed 100644 --- a/server/routers/traefik/traefikConfigProvider.ts +++ b/server/routers/traefik/traefikConfigProvider.ts @@ -21,12 +21,10 @@ export async function traefikConfigProvider( const maintenanceHost = config.getRawConfig().server.internal_hostname; const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`; const aiGatewayUrl = - `${ - config.getRawConfig().server.internal_api_override - }/ai-gateway` || + config.getRawConfig().server.ai_gateway_override || `http://${maintenanceHost}:${ - config.getRawConfig().server.internal_port - }/api/v1/ai-gateway`; + config.getRawConfig().server.ai_gateway_port + }`; const traefikConfig = await getTraefikConfig( currentExitNodeId, @@ -45,8 +43,7 @@ export async function traefikConfigProvider( plugin: { [badgerMiddlewareName]: { apiBaseUrl: - config.getRawConfig().server - .internal_api_override || + config.getRawConfig().server.badger_override || new URL( "/api/v1", `http://${ From 36c0edc62e63417c33b8aac5ba933fbd22dee0d7 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 16:55:39 -0400 Subject: [PATCH 3/5] Use p host to pass the host header --- .../private/lib/traefik/getTraefikConfig.ts | 47 +++++++++++++++++-- server/routers/aiGateway/chatCompletions.ts | 6 ++- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index c024607b0..2c46e08f6 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -1548,6 +1548,20 @@ export async function getTraefikConfig( } if (aiGatewayUrl) { + // The AI gateway may live on a different host than the inference + // resource itself (e.g. a remote exit node forwarding to the + // central dashboard over a tunnel). passHostHeader would forward + // the resource's own Host, which that external host won't + // recognize, so we pin the Host header to the gateway's own host + // and smuggle the original resource host through in "p-host" + // instead (same pattern as the maintenance-page routes above). + let aiGatewayHost: string | undefined; + try { + aiGatewayHost = new URL(aiGatewayUrl).host; + } catch { + aiGatewayHost = undefined; + } + // Public inference resources: same TLS/cert-resolver handling as // plain http-mode resources, but the service points at the AI // gateway instead of any real backend targets. @@ -1613,10 +1627,21 @@ export async function getTraefikConfig( } } + const irHeadersMiddlewareName = `${irKey}-headers-middleware`; + config_output.http.middlewares[irHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), + "p-host": fullDomain + } + } + }; + const additionalMiddlewares = config.getRawConfig().traefik.additional_middlewares || []; const routerMiddlewares = [ badgerMiddlewareName, + irHeadersMiddlewareName, ...additionalMiddlewares ]; @@ -1647,8 +1672,7 @@ export async function getTraefikConfig( config_output.http.services[serviceName] = { loadBalancer: { - servers: [{ url: `${aiGatewayUrl}/chat/completions` }], - passHostHeader: true + servers: [{ url: `${aiGatewayUrl}/chat/completions` }] } }; } @@ -1701,8 +1725,22 @@ export async function getTraefikConfig( } } + const srHeadersMiddlewareName = `${srKey}-headers-middleware`; + config_output.http.middlewares[srHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), + "p-host": alias + } + } + }; + const additionalMiddlewares = config.getRawConfig().traefik.additional_middlewares || []; + const routerMiddlewares = [ + srHeadersMiddlewareName, + ...additionalMiddlewares + ]; if (sr.ssl) { config_output.http.routers[routerName + "-redirect"] = { @@ -1722,7 +1760,7 @@ export async function getTraefikConfig( ? config.getRawConfig().traefik.https_entrypoint : config.getRawConfig().traefik.http_entrypoint ], - middlewares: additionalMiddlewares, + middlewares: routerMiddlewares, service: serviceName, rule, priority: 100, @@ -1731,8 +1769,7 @@ export async function getTraefikConfig( config_output.http.services[serviceName] = { loadBalancer: { - servers: [{ url: `${aiGatewayUrl}/chat/completions` }], - passHostHeader: true + servers: [{ url: `${aiGatewayUrl}/chat/completions` }] } }; } diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index f1ac231ff..995463c6a 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -416,7 +416,11 @@ export async function chatCompletions( res: Response ): Promise { try { - const host = (req.headers.host || "").split(":")[0]; + const host = ( + (req.headers["p-host"] as string | undefined) || + req.headers.host || + "" + ).split(":")[0]; if (!host) { return res .status(HttpCode.BAD_REQUEST) From 6526d7f225be5fd1d2ff06b444272e60f3da46dc Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 17:06:11 -0400 Subject: [PATCH 4/5] Remove completions endpoints --- server/lib/traefik/getTraefikConfig.ts | 2 +- server/private/lib/traefik/getTraefikConfig.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/lib/traefik/getTraefikConfig.ts b/server/lib/traefik/getTraefikConfig.ts index 574b8a374..ee972ea68 100644 --- a/server/lib/traefik/getTraefikConfig.ts +++ b/server/lib/traefik/getTraefikConfig.ts @@ -785,7 +785,7 @@ export async function getTraefikConfig( config_output.http.services[serviceName] = { loadBalancer: { - servers: [{ url: `${aiGatewayUrl}/chat/completions` }], + servers: [{ url: aiGatewayUrl }], passHostHeader: true } }; diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 2c46e08f6..79f31c75b 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -1672,7 +1672,7 @@ export async function getTraefikConfig( config_output.http.services[serviceName] = { loadBalancer: { - servers: [{ url: `${aiGatewayUrl}/chat/completions` }] + servers: [{ url: aiGatewayUrl }] } }; } @@ -1769,7 +1769,7 @@ export async function getTraefikConfig( config_output.http.services[serviceName] = { loadBalancer: { - servers: [{ url: `${aiGatewayUrl}/chat/completions` }] + servers: [{ url: aiGatewayUrl }] } }; } From 72d469b19cf945b68eeac80b2fd273d9c79368d1 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 17:17:51 -0400 Subject: [PATCH 5/5] Revert changes to traefik config to pull site resources --- .../private/lib/traefik/getTraefikConfig.ts | 48 ++++++------------- server/routers/aiGateway/chatCompletions.ts | 2 + 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 79f31c75b..67c037550 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -433,41 +433,21 @@ export async function getTraefikConfig( ) ); - let siteResourcesInference: { - siteResourceId: number; - alias: string | null; - ssl: boolean | null; - enabled: boolean | null; - }[] = []; - if (build == "enterprise") { - siteResourcesInference = await db - .selectDistinct({ - siteResourceId: siteResources.siteResourceId, - alias: siteResources.alias, - ssl: siteResources.ssl, - enabled: siteResources.enabled - }) - .from(siteResources) - .innerJoin( - siteResourceAiProviders, - eq( - siteResources.siteResourceId, - siteResourceAiProviders.siteResourceId - ) + const siteResourcesInference = await db + .selectDistinct({ + siteResourceId: siteResources.siteResourceId, + alias: siteResources.alias, + ssl: siteResources.ssl, + enabled: siteResources.enabled + }) + .from(siteResources) + .where( + and( + eq(siteResources.mode, "inference"), + eq(siteResources.enabled, true), + isNotNull(siteResources.alias) ) - .innerJoin( - aiProviders, - eq(siteResourceAiProviders.providerId, aiProviders.providerId) - ) - .where( - and( - eq(siteResources.mode, "inference"), - eq(siteResources.enabled, true), - eq(aiProviders.enabled, true), - isNotNull(siteResources.alias) - ) - ); - } + ); let validCerts: CertificateResult[] = []; if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index 995463c6a..41fd1f4e3 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -427,6 +427,8 @@ export async function chatCompletions( .json({ error: { message: "Missing Host header" } }); } + logger.info(`++++++++++++++AI gateway request for host: ${host}`); + const target = await resolveTarget(host); if (!target) { return res.status(HttpCode.NOT_FOUND).json({