move the ai gateway to its own server

server/aiGatewayServer.ts

@@ -0,0 +1,39 @@
+import express from "express";
+import helmet from "helmet";
+import cors from "cors";
+import config from "@server/lib/config";
+import logger from "@server/logger";
+import {
+    errorHandlerMiddleware,
+    notFoundMiddleware
+} from "@server/middlewares";
+import * as aiGateway from "@server/routers/aiGateway";
+
+const aiGatewayPort = config.getRawConfig().server.ai_gateway_port;
+
+export function createAiGatewayServer() {
+    const aiGatewayServer = express();
+
+    const trustProxy = config.getRawConfig().server.trust_proxy;
+    if (trustProxy) {
+        aiGatewayServer.set("trust proxy", trustProxy);
+    }
+
+    aiGatewayServer.use(helmet());
+    aiGatewayServer.use(cors());
+    aiGatewayServer.use(express.json());
+
+    aiGatewayServer.post("/chat/completions", aiGateway.chatCompletions);
+
+    aiGatewayServer.use(notFoundMiddleware);
+    aiGatewayServer.use(errorHandlerMiddleware);
+
+    aiGatewayServer.listen(aiGatewayPort, (err?: any) => {
+        if (err) throw err;
+        logger.info(
+            `AI gateway server is running on http://localhost:${aiGatewayPort}`
+        );
+    });
+
+    return aiGatewayServer;
+}

server/index.ts

@@ -5,6 +5,7 @@ import { runSetupFunctions } from "./setup";
 import { createApiServer } from "./apiServer";
 import { createNextServer } from "./nextServer";
 import { createInternalServer } from "./internalServer";
+import { createAiGatewayServer } from "./aiGatewayServer";
 import { createIntegrationApiServer } from "./integrationApiServer";
 import {
     ApiKey,
@@ -49,6 +50,7 @@ async function startServers() {
     // Start all servers
     const apiServer = createApiServer();
     const internalServer = createInternalServer();
+    const aiGatewayServer = createAiGatewayServer();
 
     const nextServer = await createNextServer();
     if (config.getRawConfig().traefik.file_mode) {
@@ -67,6 +69,7 @@ async function startServers() {
         apiServer,
         nextServer,
         internalServer,
+        aiGatewayServer,
         integrationServer
     };
 }

server/lib/readConfigFile.ts

@@ -79,7 +79,13 @@ export const configSchema = z
                     .default(3001)
                     .transform(stoi)
                     .pipe(portSchema),
-                internal_api_override: z.string().optional(),
+                ai_gateway_port: portSchema
+                    .optional()
+                    .default(3005)
+                    .transform(stoi)
+                    .pipe(portSchema),
+                badger_override: z.string().optional(),
+                ai_gateway_override: z.string().optional(),
                 next_port: portSchema
                     .optional()
                     .default(3002)
@@ -139,6 +145,7 @@ export const configSchema = z
                 integration_port: 3003,
                 external_port: 3000,
                 internal_port: 3001,
+                ai_gateway_port: 3005,
                 next_port: 3002,
                 internal_hostname: "pangolin",
                 session_cookie_name: "p_session_token",

server/lib/traefik/TraefikConfigManager.ts

@@ -516,9 +516,11 @@ export class TraefikConfigManager {
             const maintenanceHost =
                 config.getRawConfig().server.internal_hostname;
             const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`;
-            const aiGatewayUrl = `http://${maintenanceHost}:${
+            const aiGatewayUrl =
-                config.getRawConfig().server.internal_port
+                config.getRawConfig().server.ai_gateway_override ||
-            }/api/v1/ai-gateway`;
+                `http://${maintenanceHost}:${
+                    config.getRawConfig().server.ai_gateway_port
+                }`;
 
             // logger.debug(`Fetching traefik config for exit node: ${currentExitNode}`);
             traefikConfig = await getTraefikConfig(

server/routers/internal.ts

@@ -3,7 +3,6 @@ import * as gerbil from "@server/routers/gerbil";
 import * as traefik from "@server/routers/traefik";
 import * as resource from "./resource";
 import * as badger from "./badger";
-import * as aiGateway from "@server/routers/aiGateway";
 import * as auth from "@server/routers/auth";
 import * as supporterKey from "@server/routers/supporterKey";
 import * as idp from "@server/routers/idp";
@@ -65,9 +64,3 @@ badgerRouter.post("/verify-session", badger.verifyResourceSession);
 
 badgerRouter.post("/exchange-session", badger.exchangeSession);
 
-// AI inference gateway - minimal chat-completions proxy for inference-mode
-// resources/siteResources
-internalRouter.post(
-    "/ai-gateway/chat/completions",
-    aiGateway.chatCompletions
-);

server/routers/traefik/traefikConfigProvider.ts

@@ -21,12 +21,10 @@ export async function traefikConfigProvider(
         const maintenanceHost = config.getRawConfig().server.internal_hostname;
         const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`;
         const aiGatewayUrl =
-            `${
+            config.getRawConfig().server.ai_gateway_override ||
-                config.getRawConfig().server.internal_api_override
-            }/ai-gateway` ||
             `http://${maintenanceHost}:${
-                config.getRawConfig().server.internal_port
+                config.getRawConfig().server.ai_gateway_port
-            }/api/v1/ai-gateway`;
+            }`;
 
         const traefikConfig = await getTraefikConfig(
             currentExitNodeId,
@@ -45,8 +43,7 @@ export async function traefikConfigProvider(
                 plugin: {
                     [badgerMiddlewareName]: {
                         apiBaseUrl:
-                            config.getRawConfig().server
+                            config.getRawConfig().server.badger_override ||
-                                .internal_api_override ||
                             new URL(
                                 "/api/v1",
                                 `http://${