From aad26b9ae40eefc819749f3aaf3980534f39b48c Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 16:52:48 -0400 Subject: [PATCH] move the ai gateway to its own server --- server/aiGatewayServer.ts | 39 +++++++++++++++++++ server/index.ts | 3 ++ server/lib/readConfigFile.ts | 9 ++++- server/lib/traefik/TraefikConfigManager.ts | 8 ++-- server/routers/internal.ts | 7 ---- .../routers/traefik/traefikConfigProvider.ts | 11 ++---- 6 files changed, 59 insertions(+), 18 deletions(-) create mode 100644 server/aiGatewayServer.ts diff --git a/server/aiGatewayServer.ts b/server/aiGatewayServer.ts new file mode 100644 index 000000000..db8b12324 --- /dev/null +++ b/server/aiGatewayServer.ts @@ -0,0 +1,39 @@ +import express from "express"; +import helmet from "helmet"; +import cors from "cors"; +import config from "@server/lib/config"; +import logger from "@server/logger"; +import { + errorHandlerMiddleware, + notFoundMiddleware +} from "@server/middlewares"; +import * as aiGateway from "@server/routers/aiGateway"; + +const aiGatewayPort = config.getRawConfig().server.ai_gateway_port; + +export function createAiGatewayServer() { + const aiGatewayServer = express(); + + const trustProxy = config.getRawConfig().server.trust_proxy; + if (trustProxy) { + aiGatewayServer.set("trust proxy", trustProxy); + } + + aiGatewayServer.use(helmet()); + aiGatewayServer.use(cors()); + aiGatewayServer.use(express.json()); + + aiGatewayServer.post("/chat/completions", aiGateway.chatCompletions); + + aiGatewayServer.use(notFoundMiddleware); + aiGatewayServer.use(errorHandlerMiddleware); + + aiGatewayServer.listen(aiGatewayPort, (err?: any) => { + if (err) throw err; + logger.info( + `AI gateway server is running on http://localhost:${aiGatewayPort}` + ); + }); + + return aiGatewayServer; +} diff --git a/server/index.ts b/server/index.ts index 88612b86a..3d9306d2b 100644 --- a/server/index.ts +++ b/server/index.ts @@ -5,6 +5,7 @@ import { runSetupFunctions } from "./setup"; import { createApiServer } from "./apiServer"; import { createNextServer } from "./nextServer"; import { createInternalServer } from "./internalServer"; +import { createAiGatewayServer } from "./aiGatewayServer"; import { createIntegrationApiServer } from "./integrationApiServer"; import { ApiKey, @@ -49,6 +50,7 @@ async function startServers() { // Start all servers const apiServer = createApiServer(); const internalServer = createInternalServer(); + const aiGatewayServer = createAiGatewayServer(); const nextServer = await createNextServer(); if (config.getRawConfig().traefik.file_mode) { @@ -67,6 +69,7 @@ async function startServers() { apiServer, nextServer, internalServer, + aiGatewayServer, integrationServer }; } diff --git a/server/lib/readConfigFile.ts b/server/lib/readConfigFile.ts index 97dda31a2..37418ed2f 100644 --- a/server/lib/readConfigFile.ts +++ b/server/lib/readConfigFile.ts @@ -79,7 +79,13 @@ export const configSchema = z .default(3001) .transform(stoi) .pipe(portSchema), - internal_api_override: z.string().optional(), + ai_gateway_port: portSchema + .optional() + .default(3005) + .transform(stoi) + .pipe(portSchema), + badger_override: z.string().optional(), + ai_gateway_override: z.string().optional(), next_port: portSchema .optional() .default(3002) @@ -139,6 +145,7 @@ export const configSchema = z integration_port: 3003, external_port: 3000, internal_port: 3001, + ai_gateway_port: 3005, next_port: 3002, internal_hostname: "pangolin", session_cookie_name: "p_session_token", diff --git a/server/lib/traefik/TraefikConfigManager.ts b/server/lib/traefik/TraefikConfigManager.ts index 73d46ee64..a52c5232a 100644 --- a/server/lib/traefik/TraefikConfigManager.ts +++ b/server/lib/traefik/TraefikConfigManager.ts @@ -516,9 +516,11 @@ export class TraefikConfigManager { const maintenanceHost = config.getRawConfig().server.internal_hostname; const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`; - const aiGatewayUrl = `http://${maintenanceHost}:${ - config.getRawConfig().server.internal_port - }/api/v1/ai-gateway`; + const aiGatewayUrl = + config.getRawConfig().server.ai_gateway_override || + `http://${maintenanceHost}:${ + config.getRawConfig().server.ai_gateway_port + }`; // logger.debug(`Fetching traefik config for exit node: ${currentExitNode}`); traefikConfig = await getTraefikConfig( diff --git a/server/routers/internal.ts b/server/routers/internal.ts index af12d4429..b06015a5f 100644 --- a/server/routers/internal.ts +++ b/server/routers/internal.ts @@ -3,7 +3,6 @@ import * as gerbil from "@server/routers/gerbil"; import * as traefik from "@server/routers/traefik"; import * as resource from "./resource"; import * as badger from "./badger"; -import * as aiGateway from "@server/routers/aiGateway"; import * as auth from "@server/routers/auth"; import * as supporterKey from "@server/routers/supporterKey"; import * as idp from "@server/routers/idp"; @@ -65,9 +64,3 @@ badgerRouter.post("/verify-session", badger.verifyResourceSession); badgerRouter.post("/exchange-session", badger.exchangeSession); -// AI inference gateway - minimal chat-completions proxy for inference-mode -// resources/siteResources -internalRouter.post( - "/ai-gateway/chat/completions", - aiGateway.chatCompletions -); diff --git a/server/routers/traefik/traefikConfigProvider.ts b/server/routers/traefik/traefikConfigProvider.ts index c2e88709b..e1847d2ed 100644 --- a/server/routers/traefik/traefikConfigProvider.ts +++ b/server/routers/traefik/traefikConfigProvider.ts @@ -21,12 +21,10 @@ export async function traefikConfigProvider( const maintenanceHost = config.getRawConfig().server.internal_hostname; const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`; const aiGatewayUrl = - `${ - config.getRawConfig().server.internal_api_override - }/ai-gateway` || + config.getRawConfig().server.ai_gateway_override || `http://${maintenanceHost}:${ - config.getRawConfig().server.internal_port - }/api/v1/ai-gateway`; + config.getRawConfig().server.ai_gateway_port + }`; const traefikConfig = await getTraefikConfig( currentExitNodeId, @@ -45,8 +43,7 @@ export async function traefikConfigProvider( plugin: { [badgerMiddlewareName]: { apiBaseUrl: - config.getRawConfig().server - .internal_api_override || + config.getRawConfig().server.badger_override || new URL( "/api/v1", `http://${