From ba24e1c4f5ec5c8b404bd35746a986923fcc50f8 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 30 Jul 2026 11:57:48 -0400 Subject: [PATCH 001/296] Add exit node selection to the clients --- server/lib/calculateUserClientsForOrgs.ts | 14 --- .../lib/exitNodes/calculateExitNodeWeight.ts | 40 ++++++++ server/lib/exitNodes/exitNodes.ts | 11 ++- server/lib/exitNodes/index.ts | 1 + server/private/lib/exitNodes/exitNodes.ts | 11 ++- server/routers/client/createClient.ts | 15 --- server/routers/client/createUserClient.ts | 6 -- ...s => handleNewtExitNodesRequestMessage.ts} | 40 +++----- .../routers/newt/handleNewtRegisterMessage.ts | 11 +-- server/routers/newt/index.ts | 2 +- .../olm/handleOlmExitNodesRequestMessage.ts | 93 +++++++++++++++++++ .../routers/olm/handleOlmRegisterMessage.ts | 22 ++++- server/routers/ws/messageHandlers.ts | 4 +- 13 files changed, 190 insertions(+), 80 deletions(-) create mode 100644 server/lib/exitNodes/calculateExitNodeWeight.ts rename server/routers/newt/{handleNewtPingRequestMessage.ts => handleNewtExitNodesRequestMessage.ts} (64%) create mode 100644 server/routers/olm/handleOlmExitNodesRequestMessage.ts diff --git a/server/lib/calculateUserClientsForOrgs.ts b/server/lib/calculateUserClientsForOrgs.ts index 39585500a..5770029de 100644 --- a/server/lib/calculateUserClientsForOrgs.ts +++ b/server/lib/calculateUserClientsForOrgs.ts @@ -339,19 +339,6 @@ export async function calculateUserClientsForOrgs( continue; } - // Get exit nodes for this org - const exitNodesList = await getExitNodes(orgId); - - if (exitNodesList.length === 0) { - logger.warn( - `Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): no exit nodes found` - ); - continue; - } - - const randomExitNode = - exitNodesList[Math.floor(Math.random() * exitNodesList.length)]; - // Get next available subnet const { value: newSubnet, release: releaseSubnetLock } = await getNextAvailableClientSubnet(orgId, trx); @@ -370,7 +357,6 @@ export async function calculateUserClientsForOrgs( const newClientData: InferInsertModel = { userId, orgId: userOrg.orgId, - exitNodeId: randomExitNode.exitNodeId, name: olm.name || "User Client", subnet: updatedSubnet, olmId: olm.olmId, diff --git a/server/lib/exitNodes/calculateExitNodeWeight.ts b/server/lib/exitNodes/calculateExitNodeWeight.ts new file mode 100644 index 000000000..a740e7a89 --- /dev/null +++ b/server/lib/exitNodes/calculateExitNodeWeight.ts @@ -0,0 +1,40 @@ +import { db, sites, clients } from "@server/db"; +import { and, eq, count } from "drizzle-orm"; + +// (MAX_CONNECTIONS - current_connections) / MAX_CONNECTIONS) +// higher = more desirable +// like saying, this node has x% of its capacity left +export async function calculateExitNodeWeight( + exitNodeId: number, + maxConnections: number | null | undefined +): Promise { + if (maxConnections === null || maxConnections === undefined) { + return 1; + } + + const [[siteConnections], [clientConnections]] = await Promise.all([ + db + .select({ count: count() }) + .from(sites) + .where( + and(eq(sites.exitNodeId, exitNodeId), eq(sites.online, true)) + ), + db + .select({ count: count() }) + .from(clients) + .where( + and( + eq(clients.exitNodeId, exitNodeId), + eq(clients.online, true) + ) + ) + ]); + + const currentConnections = siteConnections.count + clientConnections.count; + + if (currentConnections >= maxConnections) { + return null; + } + + return (maxConnections - currentConnections) / maxConnections; +} diff --git a/server/lib/exitNodes/exitNodes.ts b/server/lib/exitNodes/exitNodes.ts index f405a1114..823fdde97 100644 --- a/server/lib/exitNodes/exitNodes.ts +++ b/server/lib/exitNodes/exitNodes.ts @@ -1,6 +1,5 @@ import { db, exitNodes, Transaction } from "@server/db"; import logger from "@server/logger"; -import { ExitNodePingResult } from "@server/routers/newt"; import { eq } from "drizzle-orm"; export async function verifyExitNodeOrgAccess( @@ -52,6 +51,16 @@ export async function listExitNodes( return allExitNodes; } +export type ExitNodePingResult = { + exitNodeId: number; + latencyMs: number; + weight: number; + error?: string; + exitNodeName: string; + endpoint: string; + wasPreviouslyConnected: boolean; +}; + export function selectBestExitNode( pingResults: ExitNodePingResult[] ): ExitNodePingResult | null { diff --git a/server/lib/exitNodes/index.ts b/server/lib/exitNodes/index.ts index d1477a683..459995074 100644 --- a/server/lib/exitNodes/index.ts +++ b/server/lib/exitNodes/index.ts @@ -2,3 +2,4 @@ export * from "./exitNodes"; export * from "./exitNodeComms"; export * from "./subnet"; export * from "./getCurrentExitNodeId"; +export * from "./calculateExitNodeWeight"; diff --git a/server/private/lib/exitNodes/exitNodes.ts b/server/private/lib/exitNodes/exitNodes.ts index 1f9517725..976f52cff 100644 --- a/server/private/lib/exitNodes/exitNodes.ts +++ b/server/private/lib/exitNodes/exitNodes.ts @@ -25,7 +25,6 @@ import { Transaction } from "@server/db"; import logger from "@server/logger"; -import { ExitNodePingResult } from "@server/routers/newt"; import { eq, and, or, ne, isNull, inArray } from "drizzle-orm"; import axios from "axios"; import config from "../config"; @@ -330,6 +329,16 @@ export async function listExitNodes( return exitNodesList; } +export type ExitNodePingResult = { + exitNodeId: number; + latencyMs: number; + weight: number; + error?: string; + exitNodeName: string; + endpoint: string; + wasPreviouslyConnected: boolean; +}; + /** * Selects the most suitable exit node from a list of ping results. * diff --git a/server/routers/client/createClient.ts b/server/routers/client/createClient.ts index 11c7c9ec2..d84a30301 100644 --- a/server/routers/client/createClient.ts +++ b/server/routers/client/createClient.ts @@ -255,20 +255,6 @@ export async function createClient( let newClient: Client | null = null; await db.transaction(async (trx) => { - // TODO: more intelligent way to pick the exit node - const exitNodesList = await listExitNodes(orgId); - const randomExitNode = - exitNodesList[Math.floor(Math.random() * exitNodesList.length)]; - - if (!randomExitNode) { - return next( - createHttpError( - HttpCode.NOT_FOUND, - `No exit nodes available. ${build == "saas" ? "Please contact support." : "You need to install gerbil to use the clients."}` - ) - ); - } - const [adminRole] = await trx .select() .from(roles) @@ -287,7 +273,6 @@ export async function createClient( .insert(clients) .values({ niceId, - exitNodeId: randomExitNode.exitNodeId, orgId, name, subnet: updatedSubnet, diff --git a/server/routers/client/createUserClient.ts b/server/routers/client/createUserClient.ts index 70027e4a5..bc1b25ebf 100644 --- a/server/routers/client/createUserClient.ts +++ b/server/routers/client/createUserClient.ts @@ -222,11 +222,6 @@ export async function createUserClient( let newClient: Client | null = null; await db.transaction(async (trx) => { - // TODO: more intelligent way to pick the exit node - const exitNodesList = await listExitNodes(orgId); - const randomExitNode = - exitNodesList[Math.floor(Math.random() * exitNodesList.length)]; - const [adminRole] = await trx .select() .from(roles) @@ -244,7 +239,6 @@ export async function createUserClient( [newClient] = await trx .insert(clients) .values({ - exitNodeId: randomExitNode.exitNodeId, orgId, niceId, name, diff --git a/server/routers/newt/handleNewtPingRequestMessage.ts b/server/routers/newt/handleNewtExitNodesRequestMessage.ts similarity index 64% rename from server/routers/newt/handleNewtPingRequestMessage.ts rename to server/routers/newt/handleNewtExitNodesRequestMessage.ts index f239dc4de..1adfd9f1c 100644 --- a/server/routers/newt/handleNewtPingRequestMessage.ts +++ b/server/routers/newt/handleNewtExitNodesRequestMessage.ts @@ -2,14 +2,17 @@ import { db, sites } from "@server/db"; import { MessageHandler } from "@server/routers/ws"; import { exitNodes, Newt } from "@server/db"; import logger from "@server/logger"; -import { ne, eq, or, and, count } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import { listExitNodes } from "#dynamic/lib/exitNodes"; +import { calculateExitNodeWeight } from "@server/lib/exitNodes"; -export const handleNewtPingRequestMessage: MessageHandler = async (context) => { +export const handleNewtExitNodesRequestMessage: MessageHandler = async ( + context +) => { const { message, client, sendToClient } = context; const newt = client as Newt; - logger.info("Handling ping request newt message!"); + logger.info("Handling exit nodes request newt message!"); if (!newt) { logger.warn("Newt not found"); @@ -54,32 +57,13 @@ export const handleNewtPingRequestMessage: MessageHandler = async (context) => { const exitNodesPayload = await Promise.all( exitNodesList.map(async (node) => { - // (MAX_CONNECTIONS - current_connections) / MAX_CONNECTIONS) - // higher = more desirable - // like saying, this node has x% of its capacity left + const weight = await calculateExitNodeWeight( + node.exitNodeId, + node.maxConnections + ); - let weight = 1; - const maxConnections = node.maxConnections; - if (maxConnections !== null && maxConnections !== undefined) { - const [currentConnections] = await db - .select({ - count: count() - }) - .from(sites) - .where( - and( - eq(sites.exitNodeId, node.exitNodeId), - eq(sites.online, true) - ) - ); - - if (currentConnections.count >= maxConnections) { - return null; - } - - weight = - (maxConnections - currentConnections.count) / - maxConnections; + if (weight === null) { + return null; } return { diff --git a/server/routers/newt/handleNewtRegisterMessage.ts b/server/routers/newt/handleNewtRegisterMessage.ts index 0dc8380c8..27956e845 100644 --- a/server/routers/newt/handleNewtRegisterMessage.ts +++ b/server/routers/newt/handleNewtRegisterMessage.ts @@ -7,6 +7,7 @@ import logger from "@server/logger"; import config from "@server/lib/config"; import { findNextAvailableCidr } from "@server/lib/ip"; import { + ExitNodePingResult, selectBestExitNode, verifyExitNodeOrgAccess } from "#dynamic/lib/exitNodes"; @@ -15,16 +16,6 @@ import { lockManager } from "#dynamic/lib/lock"; import { buildTargetConfigurationForNewtClient } from "./buildConfiguration"; import { canCompress } from "@server/lib/clientVersionChecks"; -export type ExitNodePingResult = { - exitNodeId: number; - latencyMs: number; - weight: number; - error?: string; - exitNodeName: string; - endpoint: string; - wasPreviouslyConnected: boolean; -}; - export const handleNewtRegisterMessage: MessageHandler = async (context) => { const { message, client, sendToClient } = context; const newt = client as Newt; diff --git a/server/routers/newt/index.ts b/server/routers/newt/index.ts index 1f6090358..5665a5420 100644 --- a/server/routers/newt/index.ts +++ b/server/routers/newt/index.ts @@ -5,7 +5,7 @@ export * from "./handleNewtRegisterMessage"; export * from "./handleReceiveBandwidthMessage"; export * from "./handleNewtGetConfigMessage"; export * from "./handleSocketMessages"; -export * from "./handleNewtPingRequestMessage"; +export * from "./handleNewtExitNodesRequestMessage"; export * from "./handleApplyBlueprintMessage"; export * from "./handleNewtPingMessage"; export * from "./handleNewtDisconnectingMessage"; diff --git a/server/routers/olm/handleOlmExitNodesRequestMessage.ts b/server/routers/olm/handleOlmExitNodesRequestMessage.ts new file mode 100644 index 000000000..bb8c7a48f --- /dev/null +++ b/server/routers/olm/handleOlmExitNodesRequestMessage.ts @@ -0,0 +1,93 @@ +import { db, clients } from "@server/db"; +import { MessageHandler } from "@server/routers/ws"; +import { exitNodes, Olm } from "@server/db"; +import logger from "@server/logger"; +import { eq } from "drizzle-orm"; +import { listExitNodes } from "#dynamic/lib/exitNodes"; +import { calculateExitNodeWeight } from "@server/lib/exitNodes"; + +export const handleOlmExitNodesRequestMessage: MessageHandler = async ( + context +) => { + const { message, client: olmClient, sendToClient } = context; + const olm = olmClient as Olm; + + logger.info("Handling exit nodes request olm message!"); + + if (!olm) { + logger.warn("olm not found"); + return; + } + + // Get the olm's orgId through the client relationship + if (!olm.clientId) { + logger.warn("olm clientId not found"); + return; + } + + const [client] = await db + .select({ orgId: clients.orgId }) + .from(clients) + .where(eq(clients.clientId, olm.clientId)) + .limit(1); + + if (!client || !client.orgId) { + logger.warn("client not found"); + return; + } + + const { noCloud, chainId } = message.data; + + const exitNodesList = await listExitNodes( + client.orgId, + true, + noCloud || false, + olm.clientId + ); // filter for only the online ones + + let lastExitNodeId = null; + if (olm.clientId) { + const [lastExitNode] = await db + .select() + .from(clients) + .where(eq(clients.clientId, olm.clientId)) + .limit(1); + lastExitNodeId = lastExitNode?.exitNodeId || null; + } + + const exitNodesPayload = await Promise.all( + exitNodesList.map(async (node) => { + const weight = await calculateExitNodeWeight( + node.exitNodeId, + node.maxConnections + ); + + if (weight === null) { + return null; + } + + return { + exitNodeId: node.exitNodeId, + exitNodeName: node.name, + endpoint: node.endpoint, + weight, + wasPreviouslyConnected: node.exitNodeId === lastExitNodeId + }; + }) + ); + + // filter out null values + const filteredExitNodes = exitNodesPayload.filter((node) => node !== null); + + return { + message: { + type: "olm/ping/exitNodes", + data: { + exitNodes: filteredExitNodes, + chainId: chainId + } + }, + broadcast: false, // Send to all clients + excludeSender: false // Include sender in broadcast + }; +}; diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index d386fe74e..6b0002b83 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -22,6 +22,7 @@ import { canCompress } from "@server/lib/clientVersionChecks"; import config from "@server/lib/config"; import cache from "#dynamic/lib/cache"; // not using regional here because we need this in the register message handler before we know where the client is import { waitForClientRebuildIdle } from "@server/lib/rebuildClientAssociations"; +import { ExitNodePingResult, selectBestExitNode } from "#dynamic/lib/exitNodes"; const HOLEPUNCH_STALE_CHAIN_THRESHOLD = 18; const HOLEPUNCH_STALE_CHAIN_TTL_SECONDS = 1800; @@ -49,6 +50,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { olmAgent, orgId, userToken, + pingResults, fingerprint, postures, chainId @@ -284,7 +286,22 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { return; } - if (client.pubKey !== publicKey || client.archived) { + let exitNodeId: number | undefined; + if (pingResults) { + const bestPingResult = selectBestExitNode( + pingResults as ExitNodePingResult[] + ); + if (!bestPingResult) { + logger.warn("No suitable exit node found based on ping results"); + } + exitNodeId = bestPingResult?.exitNodeId; + } + + if ( + client.pubKey !== publicKey || + client.archived || + client.exitNodeId !== exitNodeId + ) { logger.info( "[handleOlmRegisterMessage] Public key mismatch. Updating public key and clearing session info...", { orgId: client.orgId, clientId: client.clientId } @@ -294,7 +311,8 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { .update(clients) .set({ pubKey: publicKey, - archived: false + archived: false, + exitNodeId: exitNodeId // this can be undefined if no exit node was selected, which is fine just means we cant talk to the node or connect to it }) .where(eq(clients.clientId, client.clientId)); diff --git a/server/routers/ws/messageHandlers.ts b/server/routers/ws/messageHandlers.ts index 496002142..6b8a14127 100644 --- a/server/routers/ws/messageHandlers.ts +++ b/server/routers/ws/messageHandlers.ts @@ -5,7 +5,7 @@ import { handleNewtGetConfigMessage, handleDockerStatusMessage, handleDockerContainersMessage, - handleNewtPingRequestMessage, + handleNewtExitNodesRequestMessage, handleApplyBlueprintMessage, handleNewtPingMessage, startNewtOfflineChecker, @@ -45,7 +45,7 @@ export const messageHandlers: Record = { "newt/receive-bandwidth": handleReceiveBandwidthMessage, "newt/socket/status": handleDockerStatusMessage, "newt/socket/containers": handleDockerContainersMessage, - "newt/ping/request": handleNewtPingRequestMessage, + "newt/ping/request": handleNewtExitNodesRequestMessage, "newt/blueprint/apply": handleApplyBlueprintMessage, "newt/healthcheck/status": handleHealthcheckStatusMessage, "ws/round-trip/complete": handleRoundTripMessage From 33dd10c670715573dacaa474ffae206fdddf187a Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 30 Jul 2026 12:13:09 -0400 Subject: [PATCH 002/296] Rename subnet for clarity, pick subnet on client --- server/db/sqlite/schema/schema.ts | 3 +- .../exitNodes/getUniqueSubnetForExitNode.ts | 41 ++++++++++++++ server/lib/exitNodes/index.ts | 1 + server/lib/rebuildClientAssociations.ts | 6 +-- server/lib/traefik/getTraefikConfig.ts | 2 +- .../private/lib/traefik/getTraefikConfig.ts | 2 +- .../routers/re-key/reGenerateSiteSecret.ts | 2 +- server/routers/gerbil/getAllRelays.ts | 8 +-- server/routers/gerbil/getConfig.ts | 4 +- server/routers/gerbil/updateHolePunch.ts | 2 +- .../newt/handleNewtGetConfigMessage.ts | 11 ++-- .../routers/newt/handleNewtRegisterMessage.ts | 53 ++++--------------- server/routers/olm/buildConfiguration.ts | 2 +- .../routers/olm/handleOlmRegisterMessage.ts | 52 ++++++++++++++++-- server/routers/site/createSite.ts | 8 +-- server/routers/site/listSites.ts | 2 +- server/routers/site/pickSiteDefaults.ts | 7 +-- server/routers/target/createTarget.ts | 2 +- .../sites/[niceId]/credentials/page.tsx | 5 +- 19 files changed, 135 insertions(+), 78 deletions(-) create mode 100644 server/lib/exitNodes/getUniqueSubnetForExitNode.ts diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index d0db18555..1e2d822e7 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -107,7 +107,7 @@ export const sites = sqliteTable("sites", { }), name: text("name").notNull(), pubKey: text("pubKey"), - subnet: text("subnet"), + exitNodeSubnet: text("exitNodeSubnet"), megabytesIn: integer("bytesIn").default(0), megabytesOut: integer("bytesOut").default(0), lastBandwidthUpdate: text("lastBandwidthUpdate"), @@ -599,6 +599,7 @@ export const clients = sqliteTable("clients", { pubKey: text("pubKey"), olmId: text("olmId"), // to lock it to a specific olm optionally subnet: text("subnet").notNull(), + exitNodeSubnet: text("exitNodeSubnet"), // this is the subnet when connecting to an exit node megabytesIn: integer("bytesIn"), megabytesOut: integer("bytesOut"), lastBandwidthUpdate: text("lastBandwidthUpdate"), diff --git a/server/lib/exitNodes/getUniqueSubnetForExitNode.ts b/server/lib/exitNodes/getUniqueSubnetForExitNode.ts new file mode 100644 index 000000000..728049eb1 --- /dev/null +++ b/server/lib/exitNodes/getUniqueSubnetForExitNode.ts @@ -0,0 +1,41 @@ +import { db, ExitNode, Transaction, sites, clients } from "@server/db"; +import { eq } from "drizzle-orm"; +import config from "@server/lib/config"; +import { findNextAvailableCidr } from "@server/lib/ip"; +import { lockManager } from "#dynamic/lib/lock"; + +export async function getUniqueSubnetForExitNode( + exitNode: ExitNode, + trx: Transaction | typeof db = db +): Promise { + const lockKey = `subnet-allocation:${exitNode.exitNodeId}`; + + return await lockManager.withLock( + lockKey, + async () => { + const [sitesQuery, clientsQuery] = await Promise.all([ + trx + .select({ subnet: sites.exitNodeSubnet }) + .from(sites) + .where(eq(sites.exitNodeId, exitNode.exitNodeId)), + trx + .select({ subnet: clients.exitNodeSubnet }) + .from(clients) + .where(eq(clients.exitNodeId, exitNode.exitNodeId)) + ]); + + const blockSize = config.getRawConfig().gerbil.site_block_size; + const subnets = [...sitesQuery, ...clientsQuery] + .map((row) => row.subnet) + .filter( + (subnet): subnet is string => + !!subnet && + /^(\d{1,3}\.){3}\d{1,3}\/\d{1,2}$/.test(subnet) + ); + subnets.push(exitNode.address.replace(/\/\d+$/, `/${blockSize}`)); + + return findNextAvailableCidr(subnets, blockSize, exitNode.address); + }, + 5000 // 5 second lock TTL - subnet allocation should be quick + ); +} diff --git a/server/lib/exitNodes/index.ts b/server/lib/exitNodes/index.ts index 459995074..18474e27d 100644 --- a/server/lib/exitNodes/index.ts +++ b/server/lib/exitNodes/index.ts @@ -3,3 +3,4 @@ export * from "./exitNodeComms"; export * from "./subnet"; export * from "./getCurrentExitNodeId"; export * from "./calculateExitNodeWeight"; +export * from "./getUniqueSubnetForExitNode"; diff --git a/server/lib/rebuildClientAssociations.ts b/server/lib/rebuildClientAssociations.ts index eae579634..f8b83f5f7 100644 --- a/server/lib/rebuildClientAssociations.ts +++ b/server/lib/rebuildClientAssociations.ts @@ -966,7 +966,7 @@ export async function updateClientSiteDestinations( .where(eq(clientSitesAssociationsCache.clientId, client.clientId)); for (const site of sitesData) { - if (!site.sites.subnet) { + if (!site.sites.exitNodeSubnet) { logger.debug(`Site ${site.sites.siteId} has no subnet, skipping`); continue; } @@ -1002,7 +1002,7 @@ export async function updateClientSiteDestinations( sourcePort: parsedEndpoint.port, destinations: [ { - destinationIP: site.sites.subnet.split("/")[0], + destinationIP: site.sites.exitNodeSubnet.split("/")[0], destinationPort: site.sites.listenPort || 1 // this satisfies gerbil for now but should be reevaluated } ] @@ -1010,7 +1010,7 @@ export async function updateClientSiteDestinations( } else { // add to the existing destinations destinations.destinations.push({ - destinationIP: site.sites.subnet.split("/")[0], + destinationIP: site.sites.exitNodeSubnet.split("/")[0], destinationPort: site.sites.listenPort || 1 // this satisfies gerbil for now but should be reevaluated }); } diff --git a/server/lib/traefik/getTraefikConfig.ts b/server/lib/traefik/getTraefikConfig.ts index c63b5b718..3e84cd570 100644 --- a/server/lib/traefik/getTraefikConfig.ts +++ b/server/lib/traefik/getTraefikConfig.ts @@ -87,7 +87,7 @@ export async function getTraefikConfig( siteId: sites.siteId, siteType: sites.type, siteOnline: sites.online, - subnet: sites.subnet, + subnet: sites.exitNodeSubnet, exitNodeId: sites.exitNodeId, // Domain cert resolver fields domainCertResolver: domains.certResolver, diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index b1a34842a..4be4c6d2b 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -136,7 +136,7 @@ export async function getTraefikConfig( siteId: sites.siteId, siteType: sites.type, siteOnline: sites.online, - subnet: sites.subnet, + subnet: sites.exitNodeSubnet, exitNodeId: sites.exitNodeId, // Namespace domainNamespaceId: domainNamespaces.domainNamespaceId, diff --git a/server/private/routers/re-key/reGenerateSiteSecret.ts b/server/private/routers/re-key/reGenerateSiteSecret.ts index 6e69eb3f1..2bb29b202 100644 --- a/server/private/routers/re-key/reGenerateSiteSecret.ts +++ b/server/private/routers/re-key/reGenerateSiteSecret.ts @@ -178,7 +178,7 @@ export async function reGenerateSiteSecret( ); } - if (site.exitNodeId && site.subnet) { + if (site.exitNodeId && site.exitNodeSubnet) { await deletePeer(site.exitNodeId, site.pubKey!); // the old pubkey await addPeer(site.exitNodeId, { publicKey: pubKey, diff --git a/server/routers/gerbil/getAllRelays.ts b/server/routers/gerbil/getAllRelays.ts index 23d538e62..a9a6037e1 100644 --- a/server/routers/gerbil/getAllRelays.ts +++ b/server/routers/gerbil/getAllRelays.ts @@ -100,7 +100,7 @@ export async function generateRelayMappings(exitNode: ExitNode) { // Filter to sites with the required fields up front so the rest of the // function can safely treat endpoint/subnet/listenPort as defined. const validSites = sitesRes.filter( - (s) => s.endpoint && s.subnet && s.listenPort + (s) => s.endpoint && s.exitNodeSubnet && s.listenPort ); if (validSites.length === 0) { @@ -136,7 +136,7 @@ export async function generateRelayMappings(exitNode: ExitNode) { if ( peer.orgId == null || !peer.endpoint || - !peer.subnet || + !peer.exitNodeSubnet || !peer.listenPort ) { continue; @@ -183,7 +183,7 @@ export async function generateRelayMappings(exitNode: ExitNode) { // Process each site using the pre-fetched data. for (const site of validSites) { const siteDestination: PeerDestination = { - destinationIP: site.subnet!.split("/")[0], + destinationIP: site.exitNodeSubnet!.split("/")[0], destinationPort: site.listenPort! || 1 // this satisfies gerbil for now but should be reevaluated }; @@ -207,7 +207,7 @@ export async function generateRelayMappings(exitNode: ExitNode) { continue; } addDestination(site.endpoint!, { - destinationIP: peer.subnet!.split("/")[0], + destinationIP: peer.exitNodeSubnet!.split("/")[0], destinationPort: peer.listenPort! || 1 // this satisfies gerbil for now but should be reevaluated }); } diff --git a/server/routers/gerbil/getConfig.ts b/server/routers/gerbil/getConfig.ts index 1557edce9..77921c39f 100644 --- a/server/routers/gerbil/getConfig.ts +++ b/server/routers/gerbil/getConfig.ts @@ -89,7 +89,7 @@ export async function generateGerbilConfig(exitNode: ExitNode) { and( eq(sites.exitNodeId, exitNode.exitNodeId), isNotNull(sites.pubKey), - isNotNull(sites.subnet) + isNotNull(sites.exitNodeSubnet) ) ); @@ -103,7 +103,7 @@ export async function generateGerbilConfig(exitNode: ExitNode) { } else if (site.type === "newt") { return { publicKey: site.pubKey, - allowedIps: [site.subnet!] + allowedIps: [site.exitNodeSubnet!] }; } return { diff --git a/server/routers/gerbil/updateHolePunch.ts b/server/routers/gerbil/updateHolePunch.ts index ca40fd08b..54906e240 100644 --- a/server/routers/gerbil/updateHolePunch.ts +++ b/server/routers/gerbil/updateHolePunch.ts @@ -188,7 +188,7 @@ export async function updateAndGenerateEndpointDestinations( .select({ siteId: sites.siteId, newtId: newts.newtId, - subnet: sites.subnet, + subnet: sites.exitNodeSubnet, listenPort: sites.listenPort, publicKey: sites.publicKey, endpoint: clientSitesAssociationsCache.endpoint, diff --git a/server/routers/newt/handleNewtGetConfigMessage.ts b/server/routers/newt/handleNewtGetConfigMessage.ts index cb8b8d931..169c8a0f0 100644 --- a/server/routers/newt/handleNewtGetConfigMessage.ts +++ b/server/routers/newt/handleNewtGetConfigMessage.ts @@ -95,16 +95,16 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => { .limit(1); if ( exitNode.reachableAt && - existingSite.subnet && + existingSite.exitNodeSubnet && existingSite.listenPort ) { const payload = { oldDestination: { - destinationIP: existingSite.subnet?.split("/")[0], + destinationIP: existingSite.exitNodeSubnet?.split("/")[0], destinationPort: existingSite.listenPort || 1 // this satisfies gerbil for now but should be reevaluated }, newDestination: { - destinationIP: site.subnet?.split("/")[0], + destinationIP: site.exitNodeSubnet?.split("/")[0], destinationPort: site.listenPort || 1 // this satisfies gerbil for now but should be reevaluated } }; @@ -132,7 +132,10 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => { ({ targets: dedupedTargets, certs } = dedupeCertsForTargets(targets)); } - const targetsToSend = await convertTargetsIfNecessary(newt.newtId, dedupedTargets); // for backward compatibility with old newt versions that don't support the new target format + const targetsToSend = await convertTargetsIfNecessary( + newt.newtId, + dedupedTargets + ); // for backward compatibility with old newt versions that don't support the new target format return { message: { diff --git a/server/routers/newt/handleNewtRegisterMessage.ts b/server/routers/newt/handleNewtRegisterMessage.ts index 27956e845..b04df1c83 100644 --- a/server/routers/newt/handleNewtRegisterMessage.ts +++ b/server/routers/newt/handleNewtRegisterMessage.ts @@ -1,18 +1,17 @@ -import { db, ExitNode, newts, remoteExitNodes, Transaction } from "@server/db"; +import { db, newts, remoteExitNodes } from "@server/db"; import { MessageHandler } from "@server/routers/ws"; import { exitNodes, Newt, sites } from "@server/db"; import { eq } from "drizzle-orm"; import { addPeer, deletePeer } from "../gerbil/peers"; import logger from "@server/logger"; import config from "@server/lib/config"; -import { findNextAvailableCidr } from "@server/lib/ip"; import { ExitNodePingResult, selectBestExitNode, verifyExitNodeOrgAccess } from "#dynamic/lib/exitNodes"; +import { getUniqueSubnetForExitNode } from "@server/lib/exitNodes"; import { fetchContainers } from "./dockerSocket"; -import { lockManager } from "#dynamic/lib/lock"; import { buildTargetConfigurationForNewtClient } from "./buildConfiguration"; import { canCompress } from "@server/lib/clientVersionChecks"; @@ -85,9 +84,12 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => { fetchContainers(newt.newtId); } - let siteSubnet = oldSite.subnet; + let siteSubnet = oldSite.exitNodeSubnet; let exitNodeIdToQuery = oldSite.exitNodeId; - if (exitNodeId && (oldSite.exitNodeId !== exitNodeId || !oldSite.subnet)) { + if ( + exitNodeId && + (oldSite.exitNodeId !== exitNodeId || !oldSite.exitNodeSubnet) + ) { // This effectively moves the exit node to the new one exitNodeIdToQuery = exitNodeId; // Use the provided exitNodeId if it differs from the site's exitNodeId @@ -106,7 +108,7 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => { return; } - const newSubnet = await getUniqueSubnetForSite(exitNode); + const newSubnet = await getUniqueSubnetForExitNode(exitNode); if (!newSubnet) { logger.error( @@ -122,7 +124,7 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => { .set({ pubKey: publicKey, exitNodeId: exitNodeId, - subnet: newSubnet + exitNodeSubnet: newSubnet }) .where(eq(sites.siteId, siteId)) .returning(); @@ -241,40 +243,3 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => { excludeSender: false // Include sender in broadcast }; }; - -async function getUniqueSubnetForSite( - exitNode: ExitNode, - trx: Transaction | typeof db = db -): Promise { - const lockKey = `subnet-allocation:${exitNode.exitNodeId}`; - - return await lockManager.withLock( - lockKey, - async () => { - const sitesQuery = await trx - .select({ - subnet: sites.subnet - }) - .from(sites) - .where(eq(sites.exitNodeId, exitNode.exitNodeId)); - - const blockSize = config.getRawConfig().gerbil.site_block_size; - const subnets = sitesQuery - .map((site) => site.subnet) - .filter( - (subnet) => - subnet && - /^(\d{1,3}\.){3}\d{1,3}\/\d{1,2}$/.test(subnet) - ) - .filter((subnet) => subnet !== null); - subnets.push(exitNode.address.replace(/\/\d+$/, `/${blockSize}`)); - const newSubnet = findNextAvailableCidr( - subnets, - blockSize, - exitNode.address - ); - return newSubnet; - }, - 5000 // 5 second lock TTL - subnet allocation should be quick - ); -} diff --git a/server/routers/olm/buildConfiguration.ts b/server/routers/olm/buildConfiguration.ts index 37355eae5..ed1b0cfae 100644 --- a/server/routers/olm/buildConfiguration.ts +++ b/server/routers/olm/buildConfiguration.ts @@ -167,7 +167,7 @@ export async function buildSiteConfigurationForOlmClient( peerOps.push(deletePeer(site.siteId, client.pubKey!)); } - if (!site.subnet) { + if (!site.exitNodeSubnet) { logger.debug(`Site ${site.siteId} has no subnet, skipping`); continue; } diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 6b0002b83..c0c395fb5 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -22,7 +22,12 @@ import { canCompress } from "@server/lib/clientVersionChecks"; import config from "@server/lib/config"; import cache from "#dynamic/lib/cache"; // not using regional here because we need this in the register message handler before we know where the client is import { waitForClientRebuildIdle } from "@server/lib/rebuildClientAssociations"; -import { ExitNodePingResult, selectBestExitNode } from "#dynamic/lib/exitNodes"; +import { + ExitNodePingResult, + selectBestExitNode, + verifyExitNodeOrgAccess +} from "#dynamic/lib/exitNodes"; +import { getUniqueSubnetForExitNode } from "@server/lib/exitNodes"; const HOLEPUNCH_STALE_CHAIN_THRESHOLD = 18; const HOLEPUNCH_STALE_CHAIN_TTL_SECONDS = 1800; @@ -297,10 +302,50 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { exitNodeId = bestPingResult?.exitNodeId; } + let clientSubnet = client.exitNodeSubnet; + if ( + exitNodeId && + (client.exitNodeId !== exitNodeId || !client.exitNodeSubnet) + ) { + const { exitNode, hasAccess } = await verifyExitNodeOrgAccess( + exitNodeId, + client.orgId + ); + + if (!exitNode) { + logger.warn("[handleOlmRegisterMessage] Exit node not found", { + orgId: client.orgId, + clientId: client.clientId + }); + return; + } + + if (!hasAccess) { + logger.warn( + "[handleOlmRegisterMessage] Not authorized to use this exit node", + { orgId: client.orgId, clientId: client.clientId } + ); + return; + } + + const newSubnet = await getUniqueSubnetForExitNode(exitNode); + + if (!newSubnet) { + logger.error( + `[handleOlmRegisterMessage] No available subnets found for exit node id ${exitNodeId} and client id ${client.clientId}`, + { orgId: client.orgId, clientId: client.clientId } + ); + return; + } + + clientSubnet = newSubnet; + } + if ( client.pubKey !== publicKey || client.archived || - client.exitNodeId !== exitNodeId + client.exitNodeId !== exitNodeId || + client.exitNodeSubnet !== clientSubnet ) { logger.info( "[handleOlmRegisterMessage] Public key mismatch. Updating public key and clearing session info...", @@ -312,7 +357,8 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { .set({ pubKey: publicKey, archived: false, - exitNodeId: exitNodeId // this can be undefined if no exit node was selected, which is fine just means we cant talk to the node or connect to it + exitNodeId: exitNodeId, // this can be undefined if no exit node was selected, which is fine just means we cant talk to the node or connect to it + exitNodeSubnet: clientSubnet }) .where(eq(clients.clientId, client.clientId)); diff --git a/server/routers/site/createSite.ts b/server/routers/site/createSite.ts index b24608609..574751e93 100644 --- a/server/routers/site/createSite.ts +++ b/server/routers/site/createSite.ts @@ -311,13 +311,13 @@ export async function createSite( // lets also make sure there is no overlap with other sites on the exit node const sitesQuery = await db .select({ - subnet: sites.subnet + subnet: sites.exitNodeSubnet }) .from(sites) .where( and( eq(sites.exitNodeId, exitNodeId), - eq(sites.subnet, subnet) + eq(sites.exitNodeSubnet, subnet) ) ); @@ -427,7 +427,7 @@ export async function createSite( exitNodeId, name, niceId: updatedNiceId!, - subnet, + exitNodeSubnet: subnet, type, pubKey: pubKey || null, status: "approved" @@ -444,7 +444,7 @@ export async function createSite( type, dockerSocketEnabled: false, online: true, - subnet: "0.0.0.0/32", + exitNodeSubnet: "0.0.0.0/32", status: "approved" }) .returning(); diff --git a/server/routers/site/listSites.ts b/server/routers/site/listSites.ts index cdc3a5630..0d1a588a1 100644 --- a/server/routers/site/listSites.ts +++ b/server/routers/site/listSites.ts @@ -125,7 +125,7 @@ function querySitesBase() { niceId: sites.niceId, name: sites.name, pubKey: sites.pubKey, - subnet: sites.subnet, + subnet: sites.exitNodeSubnet, megabytesIn: sites.megabytesIn, megabytesOut: sites.megabytesOut, orgName: orgs.name, diff --git a/server/routers/site/pickSiteDefaults.ts b/server/routers/site/pickSiteDefaults.ts index ea04a4ada..c65749112 100644 --- a/server/routers/site/pickSiteDefaults.ts +++ b/server/routers/site/pickSiteDefaults.ts @@ -43,7 +43,6 @@ const PickSiteDefaultsResponseDataSchema = z.object({ clientAddress: z.string().optional() }); - registry.registerPath({ method: "get", path: "/org/{orgId}/pick-site-defaults", @@ -60,7 +59,9 @@ registry.registerPath({ description: "Successful response", content: { "application/json": { - schema: createApiResponseSchema(PickSiteDefaultsResponseDataSchema) + schema: createApiResponseSchema( + PickSiteDefaultsResponseDataSchema + ) } } } @@ -108,7 +109,7 @@ export async function pickSiteDefaults( // list all of the sites on that exit node const sitesQuery = await db .select({ - subnet: sites.subnet + subnet: sites.exitNodeSubnet }) .from(sites) .where(eq(sites.exitNodeId, randomExitNode.exitNodeId)); diff --git a/server/routers/target/createTarget.ts b/server/routers/target/createTarget.ts index cf2cd6b5f..84fa00cec 100644 --- a/server/routers/target/createTarget.ts +++ b/server/routers/target/createTarget.ts @@ -263,7 +263,7 @@ export async function createTarget( // make sure the target is within the site subnet if ( site.type == "wireguard" && - !isIpInCidr(targetData.ip, site.subnet!) + !isIpInCidr(targetData.ip, site.exitNodeSubnet!) ) { return next( createHttpError( diff --git a/src/app/[orgId]/settings/sites/[niceId]/credentials/page.tsx b/src/app/[orgId]/settings/sites/[niceId]/credentials/page.tsx index 41b8f62eb..62b166939 100644 --- a/src/app/[orgId]/settings/sites/[niceId]/credentials/page.tsx +++ b/src/app/[orgId]/settings/sites/[niceId]/credentials/page.tsx @@ -72,8 +72,7 @@ export default function CredentialsPage() { const { data: latestVersions } = useQuery( productUpdatesQueries.latestVersion(true) ); - const newtVersion = - latestVersions?.data?.newt?.latestVersion ?? "latest"; + const newtVersion = latestVersions?.data?.newt?.latestVersion ?? "latest"; // Fetch site defaults for wireguard sites to show in obfuscated config useEffect(() => { @@ -354,7 +353,7 @@ export default function CredentialsPage() { text={generateObfuscatedWireGuardConfig( { subnet: - site?.subnet || + site?.exitNodeSubnet || siteDefaults?.subnet || null, address: From fa8b9216350a1585de10ea2879516d46a1428958 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 30 Jul 2026 13:57:07 -0400 Subject: [PATCH 003/296] Fix postgres schema exit node subnet --- server/db/pg/schema/schema.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 17a58d94d..4e0719284 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -99,6 +99,7 @@ export const sites = pgTable( name: varchar("name").notNull(), pubKey: varchar("pubKey"), subnet: varchar("subnet"), + exitNodeSubnet: text("exitNodeSubnet"), // this is the subnet when connecting to an exit node megabytesIn: real("bytesIn").default(0), megabytesOut: real("bytesOut").default(0), lastBandwidthUpdate: varchar("lastBandwidthUpdate"), @@ -1181,7 +1182,7 @@ export const clients = pgTable( olmId: text("olmId"), // to lock it to a specific olm optionally name: varchar("name").notNull(), pubKey: varchar("pubKey"), - subnet: varchar("subnet").notNull(), + exitNodeSubnet: varchar("exitNodeSubnet").notNull(), megabytesIn: real("bytesIn"), megabytesOut: real("bytesOut"), lastBandwidthUpdate: varchar("lastBandwidthUpdate"), From 4dbb04bfb81934168b3214473fb6d801befc35cd Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 30 Jul 2026 13:57:45 -0400 Subject: [PATCH 004/296] Handle the ping exit node request backward mode for olm and install handler --- server/routers/olm/handleOlmRegisterMessage.ts | 8 ++++++++ server/routers/olm/index.ts | 1 + server/routers/ws/messageHandlers.ts | 4 +++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index c0c395fb5..f97c3328c 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -58,9 +58,17 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { pingResults, fingerprint, postures, + backwardsCompatible, chainId } = message.data; + if (backwardsCompatible) { + logger.debug( + "[handleOlmRegisterMessage] Backwards compatible mode detected - not sending connect message and waiting for ping response." + ); + return; + } + if (!olm.clientId) { logger.warn("[handleOlmRegisterMessage] Olm client ID not found"); sendOlmError(OlmErrorCodes.CLIENT_ID_NOT_FOUND, olm.olmId); diff --git a/server/routers/olm/index.ts b/server/routers/olm/index.ts index e11d4e48e..f02672b77 100644 --- a/server/routers/olm/index.ts +++ b/server/routers/olm/index.ts @@ -15,3 +15,4 @@ export * from "./handleOlmServerInitAddPeerHandshake"; export * from "./offlineChecker"; export * from "./handleOlmUnLocalMessage"; export * from "./handleOlmLocalMessage"; +export * from "./handleOlmExitNodesRequestMessage"; diff --git a/server/routers/ws/messageHandlers.ts b/server/routers/ws/messageHandlers.ts index 6b8a14127..fff2bf7c4 100644 --- a/server/routers/ws/messageHandlers.ts +++ b/server/routers/ws/messageHandlers.ts @@ -22,7 +22,8 @@ import { handleOlmDisconnectingMessage, handleOlmServerInitAddPeerHandshake, handleOlmLocalMessage, - handleOlmUnLocalMessage + handleOlmUnLocalMessage, + handleOlmExitNodesRequestMessage } from "../olm"; import { handleHealthcheckStatusMessage } from "../target"; import { handleRoundTripMessage } from "./handleRoundTripMessage"; @@ -37,6 +38,7 @@ export const messageHandlers: Record = { "olm/wg/local": handleOlmLocalMessage, "olm/wg/unlocal": handleOlmUnLocalMessage, "olm/ping": handleOlmPingMessage, + "olm/ping/request": handleOlmExitNodesRequestMessage, "olm/disconnecting": handleOlmDisconnectingMessage, "newt/disconnecting": handleNewtDisconnectingMessage, "newt/ping": handleNewtPingMessage, From 2cdeb7c104e9aea4761accfba7f47111ebe0e0b1 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 30 Jul 2026 17:34:13 -0400 Subject: [PATCH 005/296] Add inference resource type --- server/db/pg/schema/schema.ts | 7 ++++-- server/db/sqlite/schema/schema.ts | 9 ++++++-- .../routers/olm/handleOlmRegisterMessage.ts | 23 +++++++++++++++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 4e0719284..0bbf88631 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -195,7 +195,10 @@ export const resources = pgTable( postAuthPath: text("postAuthPath"), health: varchar("health").default("unknown"), // "healthy", "unhealthy", "unknown" wildcard: boolean("wildcard").notNull().default(false), - mode: text("mode").default("http").notNull(), // rdp, ssh, http, vnc + mode: text("mode") + .default("http") + .$type<"rdp" | "ssh" | "http" | "vnc" | "inference">() + .notNull(), pamMode: varchar("pamMode", { length: 32 }) .$type<"passthrough" | "push">() .default("passthrough"), @@ -429,7 +432,7 @@ export const siteResources = pgTable( name: varchar("name").notNull(), ssl: boolean("ssl").notNull().default(false), mode: varchar("mode") - .$type<"host" | "cidr" | "http" | "ssh">() + .$type<"host" | "cidr" | "http" | "ssh" | "inference">() .notNull(), // "host" | "cidr" | "http" scheme: varchar("scheme").$type<"http" | "https">(), // only for when we are doing https or http mode proxyPort: integer("proxyPort"), // only for port mode diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 1e2d822e7..f13afa2cf 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -203,7 +203,10 @@ export const resources = sqliteTable("resources", { postAuthPath: text("postAuthPath"), health: text("health").default("unknown"), // "healthy", "unhealthy", "unknown" wildcard: integer("wildcard", { mode: "boolean" }).notNull().default(false), - mode: text("mode").default("http").notNull(), // rdp, ssh, http, vnc + mode: text("mode") + .default("http") + .$type<"rdp" | "ssh" | "http" | "vnc" | "inference">() + .notNull(), // rdp, ssh, http, vnc, inference pamMode: text("pamMode") .$type<"passthrough" | "push">() .default("passthrough"), @@ -425,7 +428,9 @@ export const siteResources = sqliteTable("siteResources", { niceId: text("niceId").notNull(), name: text("name").notNull(), ssl: integer("ssl", { mode: "boolean" }).notNull().default(false), - mode: text("mode").$type<"host" | "cidr" | "http" | "ssh">().notNull(), // "host" | "cidr" | "http" + mode: text("mode") + .$type<"host" | "cidr" | "http" | "ssh" | "inference">() + .notNull(), // "host" | "cidr" | "http" scheme: text("scheme").$type<"http" | "https">(), // only for when we are doing https or http mode proxyPort: integer("proxyPort"), // only for port mode destinationPort: integer("destinationPort"), // only for port mode diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index f97c3328c..8d148380c 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -1,4 +1,4 @@ -import { db, orgs, primaryDb } from "@server/db"; +import { db, ExitNode, orgs, primaryDb } from "@server/db"; import { MessageHandler } from "@server/routers/ws"; import { clients, @@ -311,14 +311,15 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { } let clientSubnet = client.exitNodeSubnet; + let exitNode: ExitNode | null = null; if ( exitNodeId && (client.exitNodeId !== exitNodeId || !client.exitNodeSubnet) ) { - const { exitNode, hasAccess } = await verifyExitNodeOrgAccess( - exitNodeId, - client.orgId - ); + const { exitNode: exitNodeResult, hasAccess } = + await verifyExitNodeOrgAccess(exitNodeId, client.orgId); + + exitNode = exitNodeResult; if (!exitNode) { logger.warn("[handleOlmRegisterMessage] Exit node not found", { @@ -466,6 +467,18 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { sites: siteConfigurations, tunnelIP: client.subnet, utilitySubnet: org.utilitySubnet, + exitNode: + exitNode && client.exitNodeSubnet + ? { + endpoint: `${exitNode.endpoint}:${exitNode.listenPort}`, + relayPort: + config.getRawConfig().gerbil + .clients_start_port, + publicKey: exitNode.publicKey, + serverIP: exitNode.address.split("/")[0], + tunnelIP: client.exitNodeSubnet.split("/")[0] + } + : undefined, chainId: chainId } }, From 093097c6193576d3264daf60b6c774fcb842c0fc Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 31 Jul 2026 10:54:18 -0400 Subject: [PATCH 006/296] send if we need to connect to the gerbil or not --- server/db/pg/schema/schema.ts | 3 +++ server/db/sqlite/schema/schema.ts | 5 +++++ server/routers/olm/buildConfiguration.ts | 22 +++++++++++++------ .../routers/olm/handleOlmRegisterMessage.ts | 16 +++++++++----- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 0bbf88631..9f74788d7 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -428,6 +428,9 @@ export const siteResources = pgTable( onDelete: "restrict" } ), + requiresExitNodeConnection: boolean("requiresExitNodeConnection") + .notNull() + .default(false), niceId: varchar("niceId").notNull(), name: varchar("name").notNull(), ssl: boolean("ssl").notNull().default(false), diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index f13afa2cf..ec1b9e6f4 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -425,6 +425,11 @@ export const siteResources = sqliteTable("siteResources", { () => networks.networkId, { onDelete: "restrict" } ), + requiresExitNodeConnection: integer("requiresExitNodeConnection", { + mode: "boolean" + }) + .notNull() + .default(false), niceId: text("niceId").notNull(), name: text("name").notNull(), ssl: integer("ssl", { mode: "boolean" }).notNull().default(false), diff --git a/server/routers/olm/buildConfiguration.ts b/server/routers/olm/buildConfiguration.ts index ed1b0cfae..736363a5f 100644 --- a/server/routers/olm/buildConfiguration.ts +++ b/server/routers/olm/buildConfiguration.ts @@ -48,10 +48,6 @@ export async function buildSiteConfigurationForOlmClient( ) .where(eq(clientSitesAssociationsCache.clientId, client.clientId)); - if (sitesData.length === 0) { - return siteConfigurations; - } - // Batch-fetch every site resource this client has access to across ALL sites // in a single query, then group by siteId in memory. This avoids issuing one // query per site (which would be N round-trips for N sites). @@ -68,8 +64,8 @@ export async function buildSiteConfigurationForOlmClient( clientSiteResourcesAssociationsCache.siteResourceId ) ) - .innerJoin(networks, eq(siteResources.networkId, networks.networkId)) - .innerJoin(siteNetworks, eq(networks.networkId, siteNetworks.networkId)) + .leftJoin(networks, eq(siteResources.networkId, networks.networkId)) + .leftJoin(siteNetworks, eq(networks.networkId, siteNetworks.networkId)) .where( and( eq( @@ -80,8 +76,20 @@ export async function buildSiteConfigurationForOlmClient( ) ); + const haveInferenceResources = allClientSiteResources.some( + (row) => row.siteResource.requiresExitNodeConnection === true + ); + + if (sitesData.length === 0) { + return { siteConfigurations, haveInferenceResources }; + } + const siteResourcesBySiteId = new Map(); for (const row of allClientSiteResources) { + if (!row.siteId) { + // because we are doing a leftJoin above to get the inference resources without a network / sites + continue; + } const arr = siteResourcesBySiteId.get(row.siteId); if (arr) { arr.push(row.siteResource); @@ -226,5 +234,5 @@ export async function buildSiteConfigurationForOlmClient( }); } - return siteConfigurations; + return { siteConfigurations, haveInferenceResources }; } diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 8d148380c..7fa270fa2 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -337,6 +337,8 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { return; } + // TODO: IF WE DO NOT HAVE AN INFERENCE RESOURCE DO WE NEED TO BE HOLDING A SUBNET ON THE CLIENT? + const newSubnet = await getUniqueSubnetForExitNode(exitNode); if (!newSubnet) { @@ -452,12 +454,13 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { // NOTE: its important that the client here is the old client and the public key is the new key await waitForClientRebuildIdle(olm.clientId); - const siteConfigurations = await buildSiteConfigurationForOlmClient( - client, - publicKey, - relay, - jitMode - ); + const { siteConfigurations, haveInferenceResources } = + await buildSiteConfigurationForOlmClient( + client, + publicKey, + relay, + jitMode + ); // Return connect message with all site configurations return { @@ -470,6 +473,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { exitNode: exitNode && client.exitNodeSubnet ? { + connect: haveInferenceResources, // we do not need to connect to the exit node if we do not have inference resources endpoint: `${exitNode.endpoint}:${exitNode.listenPort}`, relayPort: config.getRawConfig().gerbil From deb2d5ce2a7bd61a5b5aa7a7e3ac3a46a3b2b613 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 31 Jul 2026 11:06:30 -0400 Subject: [PATCH 007/296] Update site resources to handle new inference --- .../siteResource/createSiteResource.ts | 53 +++++++++++-------- .../siteResource/updateSiteResource.ts | 28 ++++++++-- 2 files changed, 54 insertions(+), 27 deletions(-) diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index d3e32babd..d220a4b08 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -49,7 +49,7 @@ const createSiteResourceSchema = z name: z.string().min(1).max(255), niceId: z.string().optional(), // protocol: z.enum(["tcp", "udp"]).optional(), - mode: z.enum(["host", "cidr", "http", "ssh"]), + mode: z.enum(["host", "cidr", "http", "ssh", "inference"]), ssl: z.boolean().optional(), // only used for http mode scheme: z.enum(["http", "https"]).optional(), siteIds: z.array(z.int()).optional(), @@ -171,6 +171,9 @@ const createSiteResourceSchema = z ) .refine( (data) => { + if (data.mode == "inference") { + return true; + } return ( (data.siteIds !== undefined && data.siteIds.length > 0) || data.siteId !== undefined @@ -533,21 +536,24 @@ export async function createSiteResource( let newSiteResource: SiteResource | undefined; try { await db.transaction(async (trx) => { - const [network] = await trx - .insert(networks) - .values({ - scope: "resource", - orgId: orgId - }) - .returning(); + let network: typeof networks.$inferSelect | undefined; + if (mode !== "inference") { + [network] = await trx + .insert(networks) + .values({ + scope: "resource", + orgId: orgId + }) + .returning(); - if (!network) { - return next( - createHttpError( - HttpCode.INTERNAL_SERVER_ERROR, - `Failed to create network` - ) - ); + if (!network) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + `Failed to create network` + ) + ); + } } let tcpPortRangeStringAdjusted = tcpPortRangeString; @@ -566,7 +572,7 @@ export async function createSiteResource( name, mode, ssl, - networkId: network.networkId, + networkId: network ? network.networkId : null, destination: destination, // the ssh can be null scheme, destinationPort, @@ -582,7 +588,8 @@ export async function createSiteResource( (mode == "http" || mode == "ssh" ? true : false), // default to true for http resources, otherwise false domainId, subdomain: finalSubdomain, - fullDomain + fullDomain, + requiresExitNodeConnection: mode === "inference" // in the future we might want to have different modes that do this }; if (isLicensedSshPam) { if (authDaemonPort !== undefined) @@ -600,11 +607,13 @@ export async function createSiteResource( //////////////////// update the associations //////////////////// - for (const siteId of siteIds) { - await trx.insert(siteNetworks).values({ - siteId: siteId, - networkId: network.networkId - }); + if (network) { + for (const siteId of siteIds) { + await trx.insert(siteNetworks).values({ + siteId: siteId, + networkId: network.networkId + }); + } } const [adminRole] = await trx diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index 7b0ef0783..1b79e7b16 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -50,7 +50,7 @@ const updateSiteResourceSchema = z ) .optional(), // mode: z.enum(["host", "cidr", "port"]).optional(), - mode: z.enum(["host", "cidr", "http", "ssh"]).optional(), + mode: z.enum(["host", "cidr", "http", "ssh", "inference"]).optional(), ssl: z.boolean().optional(), scheme: z.enum(["http", "https"]).nullish(), destinationPort: z.int().positive().nullish(), @@ -173,6 +173,9 @@ const updateSiteResourceSchema = z ) .refine( (data) => { + if (data.mode == "inference") { + return true; + } // if neither is provided, the existing site associations are left unchanged if (data.siteIds === undefined && data.siteId === undefined) { return true; @@ -579,13 +582,15 @@ export async function updateSiteResource( disableIcmp: mode !== undefined ? disableIcmp || - (mode == "http" || mode == "ssh" - ? true - : false) + (mode == "http" || mode == "ssh" ? true : false) : disableIcmp, domainId, subdomain: finalSubdomain, fullDomain, + networkId: + mode === "inference" ? null : undefined, + requiresExitNodeConnection: + mode !== undefined ? mode === "inference" : undefined, ...sshPamSet }) .where(and(eq(siteResources.siteResourceId, siteResourceId))) @@ -593,7 +598,20 @@ export async function updateSiteResource( //////////////////// update the associations //////////////////// - if (siteIds !== undefined) { + if (mode === "inference") { + // inference resources are not attached to any site network + if (existingSiteResource.networkId) { + await trx + .delete(siteNetworks) + .where( + eq( + siteNetworks.networkId, + existingSiteResource.networkId + ) + ); + } + updatedSiteIds = []; + } else if (siteIds !== undefined) { // delete the site - site resources associations await trx .delete(siteNetworks) From a790fed297a0f8ae4f09075c99a4b8132cf808f4 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 31 Jul 2026 11:30:32 -0400 Subject: [PATCH 008/296] sync the exit node connection --- server/lib/rebuildClientAssociations.ts | 219 +++++++++++++++++++++++- 1 file changed, 217 insertions(+), 2 deletions(-) diff --git a/server/lib/rebuildClientAssociations.ts b/server/lib/rebuildClientAssociations.ts index f8b83f5f7..ce0897aab 100644 --- a/server/lib/rebuildClientAssociations.ts +++ b/server/lib/rebuildClientAssociations.ts @@ -19,7 +19,7 @@ import { userOrgRoles, userSiteResources } from "@server/db"; -import { and, count, eq, inArray, ne } from "drizzle-orm"; +import { and, count, eq, inArray, isNotNull, ne } from "drizzle-orm"; import { deletePeersBatch as newtDeletePeersBatch } from "@server/routers/newt/peers"; import { @@ -27,6 +27,9 @@ import { deletePeersBatch as olmDeletePeersBatch } from "@server/routers/olm/peers"; import { sendToExitNode } from "#dynamic/lib/exitNodes"; +import { sendToClientsBatch } from "#dynamic/routers/ws"; +import { canCompress } from "@server/lib/clientVersionChecks"; +import config from "@server/lib/config"; import logger from "@server/logger"; import { generateAliasConfig, @@ -187,7 +190,12 @@ export async function getClientSiteResourceAccess( `rebuildClientAssociations: [getClientSiteResourceAccess] siteResourceId=${siteResource.siteResourceId} networkId=${siteResource.networkId} siteCount=${sitesList.length} siteIds=[${sitesList.map((s) => s.siteId).join(", ")}]` ); - if (sitesList.length === 0) { + if (sitesList.length === 0 && siteResource.networkId !== null) { + // A site resource with a networkId is expected to have at least one + // site attached via siteNetworks. Resources with no networkId (e.g. + // inference-mode resources, which connect clients directly to the + // exit node instead of any site) are expected to have no sites, so + // don't warn for those. logger.warn( `No sites found for siteResource ${siteResource.siteResourceId} with networkId ${siteResource.networkId}` ); @@ -687,6 +695,22 @@ async function rebuildClientAssociationsFromSiteResourceImpl( clientSiteResourcesToRemove, trx ); + + // If this resource requires clients to be connected to the exit node + // (e.g. an inference resource), re-sync the connect/disconnect state for + // every client whose access to it may have changed - both those who + // currently have access and those who just lost it. + if (siteResource.requiresExitNodeConnection) { + await syncClientExitNodeConnections( + Array.from( + new Set([ + ...mergedAllClientIds, + ...existingClientSiteResourceIds + ]) + ), + trx + ); + } } async function handleMessagesForSiteClients( @@ -1052,6 +1076,179 @@ export async function updateClientSiteDestinations( } } +// Determines, for each of the given clients, whether they currently have +// access to any enabled site resource with requiresExitNodeConnection set +// (e.g. an inference-mode resource) and tells the client's olm to connect to +// or disconnect from its assigned exit node accordingly. Site resources with +// requiresExitNodeConnection don't belong to any site/network, so this can't +// be derived from the per-site peer logic above - it has to be recomputed +// from the client's full current resource access every time that access +// changes. +async function syncClientExitNodeConnections( + clientIds: number[], + trx: Transaction | typeof db = db +): Promise { + const uniqueClientIds = Array.from(new Set(clientIds)); + if (uniqueClientIds.length === 0) { + return; + } + + // Only clients with an exit node assigned can be told to connect/disconnect. + const clientsData = await trx + .select({ + clientId: clients.clientId, + exitNodeId: clients.exitNodeId, + exitNodeSubnet: clients.exitNodeSubnet + }) + .from(clients) + .where( + and( + inArray(clients.clientId, uniqueClientIds), + isNotNull(clients.exitNodeId) + ) + ); + + if (clientsData.length === 0) { + return; + } + + const clientIdsWithExitNode = clientsData.map((c) => c.clientId); + + const requiresExitNodeRows = await trx + .select({ + clientId: clientSiteResourcesAssociationsCache.clientId + }) + .from(clientSiteResourcesAssociationsCache) + .innerJoin( + siteResources, + eq( + clientSiteResourcesAssociationsCache.siteResourceId, + siteResources.siteResourceId + ) + ) + .where( + and( + inArray( + clientSiteResourcesAssociationsCache.clientId, + clientIdsWithExitNode + ), + eq(siteResources.enabled, true), + eq(siteResources.requiresExitNodeConnection, true) + ) + ); + + const needsConnectSet = new Set(requiresExitNodeRows.map((r) => r.clientId)); + + const exitNodeIds = Array.from( + new Set( + clientsData + .map((c) => c.exitNodeId) + .filter((id): id is number => id !== null) + ) + ); + + const exitNodeRows = + exitNodeIds.length > 0 + ? await trx + .select() + .from(exitNodes) + .where(inArray(exitNodes.exitNodeId, exitNodeIds)) + : []; + const exitNodeById = new Map(exitNodeRows.map((n) => [n.exitNodeId, n])); + + const olmRows = await trx + .select({ + clientId: olms.clientId, + olmId: olms.olmId, + version: olms.version + }) + .from(olms) + .where(inArray(olms.clientId, clientIdsWithExitNode)); + const olmByClientId = new Map( + olmRows + .filter((r) => r.clientId !== null) + .map((r) => [r.clientId as number, r]) + ); + + const relayPort = config.getRawConfig().gerbil.clients_start_port; + + const connectPayloads: { + clientId: string; + message: { type: string; data: any }; + options: { compress: boolean }; + }[] = []; + const disconnectPayloads: { + clientId: string; + message: { type: string; data: any }; + options: { compress: boolean }; + }[] = []; + + for (const client of clientsData) { + const olm = olmByClientId.get(client.clientId); + if (!olm) { + // No olm registered for this client yet/anymore, nothing to send. + continue; + } + + const needsConnect = needsConnectSet.has(client.clientId); + + if (needsConnect) { + const exitNode = client.exitNodeId + ? exitNodeById.get(client.exitNodeId) + : undefined; + if (!exitNode || !client.exitNodeSubnet) { + logger.warn( + `rebuildClientAssociations: [syncClientExitNodeConnections] client ${client.clientId} needs an exit node connection but has no exit node or subnet assigned` + ); + continue; + } + + connectPayloads.push({ + clientId: olm.olmId, + message: { + type: "olm/wg/exitnode/connect", + data: { + connect: true, + endpoint: `${exitNode.endpoint}:${exitNode.listenPort}`, + relayPort, + publicKey: exitNode.publicKey, + serverIP: exitNode.address.split("/")[0], + tunnelIP: client.exitNodeSubnet.split("/")[0] + } + }, + options: { compress: canCompress(olm.version, "olm") } + }); + } else { + disconnectPayloads.push({ + clientId: olm.olmId, + message: { + type: "olm/wg/exitnode/disconnect", + data: {} + }, + options: { compress: canCompress(olm.version, "olm") } + }); + } + } + + if (connectPayloads.length > 0) { + await sendToClientsBatch(connectPayloads).catch((error) => { + logger.error( + `rebuildClientAssociations: Error sending exit node connect messages:`, + error + ); + }); + } + + if (disconnectPayloads.length > 0) { + await sendToClientsBatch(disconnectPayloads).catch((error) => { + logger.error( + `rebuildClientAssociations: Error sending exit node disconnect messages:`, + error + ); + }); + } +} + async function handleSubnetProxyTargetUpdates( siteResource: SiteResource, sitesList: Site[], @@ -1709,6 +1906,20 @@ export async function handleMessagingForUpdatedSiteResource( ); } + // If this resource requires (or required) clients to be connected to the + // exit node (e.g. an inference resource), re-sync connect/disconnect + // state for every client currently associated with it - covers toggling + // requiresExitNodeConnection on update as well as enabling/disabling it. + if ( + updatedSiteResource.requiresExitNodeConnection || + existingSiteResource?.requiresExitNodeConnection + ) { + await syncClientExitNodeConnections( + mergedAllClients.map((c) => c.clientId), + trx + ); + } + logger.debug( `handleMessagingForUpdatedSiteResource: DONE siteResourceId=${updatedSiteResource.siteResourceId}` ); @@ -1990,6 +2201,10 @@ async function rebuildClientAssociationsFromClientImpl( resourcesToRemove, trx ); + + // Re-sync exit node connect/disconnect state based on this client's + // current full set of resource access (e.g. inference resources). + await syncClientExitNodeConnections([client.clientId], trx); } async function handleMessagesForClientSites( From aad2ed2719caeea5359d785eddb5ca04467a9070 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 31 Jul 2026 15:17:16 -0400 Subject: [PATCH 009/296] Get exit node every time --- .../routers/olm/handleOlmRegisterMessage.ts | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 7fa270fa2..374c37631 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -1,4 +1,4 @@ -import { db, ExitNode, orgs, primaryDb } from "@server/db"; +import { db, ExitNode, exitNodes, orgs, primaryDb } from "@server/db"; import { MessageHandler } from "@server/routers/ws"; import { clients, @@ -311,15 +311,14 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { } let clientSubnet = client.exitNodeSubnet; - let exitNode: ExitNode | null = null; if ( exitNodeId && (client.exitNodeId !== exitNodeId || !client.exitNodeSubnet) ) { - const { exitNode: exitNodeResult, hasAccess } = - await verifyExitNodeOrgAccess(exitNodeId, client.orgId); - - exitNode = exitNodeResult; + const { exitNode, hasAccess } = await verifyExitNodeOrgAccess( + exitNodeId, + client.orgId + ); if (!exitNode) { logger.warn("[handleOlmRegisterMessage] Exit node not found", { @@ -451,6 +450,15 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { return; } + let exitNode: ExitNode | null = null; + if (exitNodeId) { + [exitNode] = await db + .select() + .from(exitNodes) + .where(eq(exitNodes.exitNodeId, exitNodeId)) + .limit(1); + } + // NOTE: its important that the client here is the old client and the public key is the new key await waitForClientRebuildIdle(olm.clientId); From 32ac8db8032f2b8c6d2a1b2551d87d27645f09c2 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 31 Jul 2026 16:21:55 -0400 Subject: [PATCH 010/296] Handle aliases when registering --- server/lib/ip.ts | 29 ++++++++++++----- server/routers/olm/buildConfiguration.ts | 31 +++++++++++++------ .../routers/olm/handleOlmRegisterMessage.ts | 12 ++++--- 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/server/lib/ip.ts b/server/lib/ip.ts index e8f589424..bb21d28d6 100644 --- a/server/lib/ip.ts +++ b/server/lib/ip.ts @@ -528,7 +528,10 @@ export function generateRemoteSubnets( export type Alias = { alias: string | null; aliasAddress: string | null }; -export function generateAliasConfig(allSiteResources: SiteResource[]): Alias[] { +export function generateAliasConfig( + allSiteResources: SiteResource[], + overrideIp?: string +): Alias[] { return allSiteResources .filter( (sr) => @@ -539,7 +542,7 @@ export function generateAliasConfig(allSiteResources: SiteResource[]): Alias[] { ) .map((sr) => ({ alias: sr.alias || sr.fullDomain, - aliasAddress: sr.aliasAddress + aliasAddress: overrideIp || sr.aliasAddress })); } @@ -660,9 +663,10 @@ export type CertRef = { id: string; cert: string; key: string }; * certificate (e.g. a wildcard cert used by thousands of site resources) * only need that certificate sent once per sync message. */ -export function dedupeCertsForTargets( - targetsV2: SubnetProxyTargetV2[] -): { targets: SubnetProxyTargetV2[]; certs: CertRef[] } { +export function dedupeCertsForTargets(targetsV2: SubnetProxyTargetV2[]): { + targets: SubnetProxyTargetV2[]; + certs: CertRef[]; +} { const idByContent = new Map(); const certs: CertRef[] = []; @@ -674,7 +678,10 @@ export function dedupeCertsForTargets( const contentKey = `${target.tlsCert}|${target.tlsKey}`; let id = idByContent.get(contentKey); if (!id) { - id = createHash("sha1").update(contentKey).digest("hex").slice(0, 16); + id = createHash("sha1") + .update(contentKey) + .digest("hex") + .slice(0, 16); idByContent.set(contentKey, id); certs.push({ id, cert: target.tlsCert, key: target.tlsKey }); } @@ -708,7 +715,9 @@ export async function batchFetchCertsForSiteResources( ): Promise { const domains = new Set( allSiteResources - .filter((r) => r.enabled && r.mode === "http" && r.ssl && r.fullDomain) + .filter( + (r) => r.enabled && r.mode === "http" && r.ssl && r.fullDomain + ) .map((r) => r.fullDomain as string) ); @@ -852,7 +861,11 @@ export async function generateSubnetProxyTargetV2( new Set([siteResource.fullDomain]), true ); - if (certs.length > 0 && certs[0].certFile && certs[0].keyFile) { + if ( + certs.length > 0 && + certs[0].certFile && + certs[0].keyFile + ) { tlsCert = certs[0].certFile; tlsKey = certs[0].keyFile; } else { diff --git a/server/routers/olm/buildConfiguration.ts b/server/routers/olm/buildConfiguration.ts index 736363a5f..af309f89a 100644 --- a/server/routers/olm/buildConfiguration.ts +++ b/server/routers/olm/buildConfiguration.ts @@ -19,6 +19,7 @@ import logger from "@server/logger"; import { and, eq, inArray } from "drizzle-orm"; import { addPeer, deletePeer } from "../newt/peers"; import config from "@server/lib/config"; +import { SiR } from "react-icons/si"; export async function buildSiteConfigurationForOlmClient( client: Client, @@ -38,6 +39,8 @@ export async function buildSiteConfigurationForOlmClient( aliases: Alias[]; }[] = []; + let exitNodeAliases: string[] = []; + // Get all sites data const sitesData = await db .select() @@ -76,16 +79,12 @@ export async function buildSiteConfigurationForOlmClient( ) ); - const haveInferenceResources = allClientSiteResources.some( - (row) => row.siteResource.requiresExitNodeConnection === true - ); - - if (sitesData.length === 0) { - return { siteConfigurations, haveInferenceResources }; - } - const siteResourcesBySiteId = new Map(); + let siteResourcesForExitNode = []; for (const row of allClientSiteResources) { + if (row.siteResource.requiresExitNodeConnection) { + siteResourcesForExitNode.push(row.siteResource); + } if (!row.siteId) { // because we are doing a leftJoin above to get the inference resources without a network / sites continue; @@ -98,6 +97,17 @@ export async function buildSiteConfigurationForOlmClient( } } + exitNodeAliases = siteResourcesForExitNode + .map((sr) => sr.alias) + .filter((a) => a != null); + + if (sitesData.length == 0) { + return { + siteConfigurations, + exitNodeAliases + }; + } + // Batch-fetch exit nodes for all sites in one query (only needed in relay mode). const exitNodesById = new Map(); if (!jitMode && relay) { @@ -234,5 +244,8 @@ export async function buildSiteConfigurationForOlmClient( }); } - return { siteConfigurations, haveInferenceResources }; + return { + siteConfigurations, + exitNodeAliases + }; } diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 374c37631..95c5e0d47 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -462,7 +462,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { // NOTE: its important that the client here is the old client and the public key is the new key await waitForClientRebuildIdle(olm.clientId); - const { siteConfigurations, haveInferenceResources } = + const { siteConfigurations, exitNodeAliases } = await buildSiteConfigurationForOlmClient( client, publicKey, @@ -470,6 +470,10 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { jitMode ); + logger.info( + `+++++++++++++++++++++++++++++++ ExitNode Aliases: ${exitNodeAliases}` + ); + // Return connect message with all site configurations return { message: { @@ -481,11 +485,9 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { exitNode: exitNode && client.exitNodeSubnet ? { - connect: haveInferenceResources, // we do not need to connect to the exit node if we do not have inference resources + aliases: exitNodeAliases, + connect: exitNodeAliases.length > 0, // we do not need to connect to the exit node if we do not have inference resources and right now all site resources on the exit node have an alias endpoint: `${exitNode.endpoint}:${exitNode.listenPort}`, - relayPort: - config.getRawConfig().gerbil - .clients_start_port, publicKey: exitNode.publicKey, serverIP: exitNode.address.split("/")[0], tunnelIP: client.exitNodeSubnet.split("/")[0] From 730078597e9921725f59b2fb87f0323db51c38be Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 31 Jul 2026 15:23:19 -0400 Subject: [PATCH 011/296] add basic schema --- server/db/pg/schema/schema.ts | 43 +++++++++++++++++++++++++++++ server/db/sqlite/schema/schema.ts | 46 +++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 9f74788d7..83f1e19bd 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1547,6 +1547,47 @@ export const statusHistory = pgTable( ] ); +export const aiProviders = pgTable("aiProviders", { + providerId: serial("providerId").primaryKey(), + orgId: varchar("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + name: varchar("name").notNull(), + type: varchar("type") + .$type<"openai" | "anthropic" | "bedrock" | "custom">() + .notNull(), + upstreamUrl: text("upstreamUrl"), + apiKey: text("apiKey"), + apiKeyLastChars: varchar("apiKeyLastChars"), + authType: varchar("authType").$type<"bearer">(), + skipTlsVerification: boolean("skipTlsVerification") + .notNull() + .default(false), + budgetAmount: real("budgetAmount"), + budgetUnit: varchar("budgetUnit").$type<"usd" | "tokens">(), + enabled: boolean("enabled").notNull().default(true), + createdAt: bigint("createdAt", { mode: "number" }).notNull(), + updatedAt: bigint("updatedAt", { mode: "number" }).notNull() +}); + +export const aiModels = pgTable( + "aiModels", + { + modelId: serial("modelId").primaryKey(), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + modelKey: varchar("modelKey").notNull(), + name: varchar("name").notNull(), + budgetAmount: real("budgetAmount"), + budgetUnit: varchar("budgetUnit").$type<"usd" | "tokens">(), + enabled: boolean("enabled").notNull().default(true), + createdAt: bigint("createdAt", { mode: "number" }).notNull(), + updatedAt: bigint("updatedAt", { mode: "number" }).notNull() + }, + (t) => [unique("ai_model_provider_key_uniq").on(t.providerId, t.modelKey)] +); + export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1631,3 +1672,5 @@ export type ResourcePolicy = InferSelectModel; export type RolePolicy = InferSelectModel; export type UserPolicy = InferSelectModel; export type ResourcePolicyRule = InferSelectModel; +export type AiProvider = InferSelectModel; +export type AiModel = InferSelectModel; diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index ec1b9e6f4..97059d3cb 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -4,6 +4,7 @@ import { index, integer, primaryKey, + real, sqliteTable, text, unique @@ -1537,6 +1538,49 @@ export const statusHistory = sqliteTable( ] ); +export const aiProviders = sqliteTable("aiProviders", { + providerId: integer("providerId").primaryKey({ autoIncrement: true }), + orgId: text("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + name: text("name").notNull(), + type: text("type") + .$type<"openai" | "anthropic" | "bedrock" | "custom">() + .notNull(), + upstreamUrl: text("upstreamUrl"), + apiKey: text("apiKey"), + apiKeyLastChars: text("apiKeyLastChars"), + authType: text("authType").$type<"bearer">(), + skipTlsVerification: integer("skipTlsVerification", { mode: "boolean" }) + .notNull() + .default(false), + budgetAmount: real("budgetAmount"), + budgetUnit: text("budgetUnit").$type<"usd" | "tokens">(), + enabled: integer("enabled", { mode: "boolean" }).notNull().default(true), + createdAt: integer("createdAt").notNull(), + updatedAt: integer("updatedAt").notNull() +}); + +export const aiModels = sqliteTable( + "aiModels", + { + modelId: integer("modelId").primaryKey({ autoIncrement: true }), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + modelKey: text("modelKey").notNull(), + name: text("name").notNull(), + budgetAmount: real("budgetAmount"), + budgetUnit: text("budgetUnit").$type<"usd" | "tokens">(), + enabled: integer("enabled", { mode: "boolean" }) + .notNull() + .default(true), + createdAt: integer("createdAt").notNull(), + updatedAt: integer("updatedAt").notNull() + }, + (t) => [unique("ai_model_provider_key_uniq").on(t.providerId, t.modelKey)] +); + export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1619,3 +1663,5 @@ export type ResourcePolicyHeaderAuth = InferSelectModel< >; export type RolePolicy = InferSelectModel; export type UserPolicy = InferSelectModel; +export type AiProvider = InferSelectModel; +export type AiModel = InferSelectModel; From 694fe98131f1a839ffdc8d72c2fc64447f8a3193 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 31 Jul 2026 15:44:17 -0400 Subject: [PATCH 012/296] add crud for providers and models --- messages/en-US.json | 10 + server/auth/actions.ts | 12 +- server/index.ts | 4 + server/lib/aiProviderDefaults.ts | 48 ++++ server/middlewares/index.ts | 2 + server/middlewares/integration/index.ts | 2 + .../integration/verifyApiKeyAiModelAccess.ts | 113 ++++++++++ .../verifyApiKeyAiProviderAccess.ts | 115 ++++++++++ server/middlewares/verifyAiModelAccess.ts | 125 +++++++++++ server/middlewares/verifyAiProviderAccess.ts | 108 +++++++++ server/openApi.ts | 4 +- server/routers/aiProvider/createAiModel.ts | 157 +++++++++++++ server/routers/aiProvider/createAiProvider.ts | 141 ++++++++++++ server/routers/aiProvider/deleteAiModel.ts | 98 +++++++++ server/routers/aiProvider/deleteAiProvider.ts | 92 ++++++++ server/routers/aiProvider/getAiModel.ts | 94 ++++++++ server/routers/aiProvider/getAiProvider.ts | 88 ++++++++ server/routers/aiProvider/index.ts | 11 + server/routers/aiProvider/listAiModels.ts | 166 ++++++++++++++ server/routers/aiProvider/listAiProviders.ts | 152 +++++++++++++ server/routers/aiProvider/types.ts | 51 +++++ server/routers/aiProvider/updateAiModel.ts | 185 ++++++++++++++++ server/routers/aiProvider/updateAiProvider.ts | 208 ++++++++++++++++++ server/routers/aiProvider/validation.ts | 66 ++++++ server/routers/external.ts | 89 +++++++- server/routers/integration.ts | 87 ++++++++ src/components/PermissionsSelectBox.tsx | 16 ++ 27 files changed, 2241 insertions(+), 3 deletions(-) create mode 100644 server/lib/aiProviderDefaults.ts create mode 100644 server/middlewares/integration/verifyApiKeyAiModelAccess.ts create mode 100644 server/middlewares/integration/verifyApiKeyAiProviderAccess.ts create mode 100644 server/middlewares/verifyAiModelAccess.ts create mode 100644 server/middlewares/verifyAiProviderAccess.ts create mode 100644 server/routers/aiProvider/createAiModel.ts create mode 100644 server/routers/aiProvider/createAiProvider.ts create mode 100644 server/routers/aiProvider/deleteAiModel.ts create mode 100644 server/routers/aiProvider/deleteAiProvider.ts create mode 100644 server/routers/aiProvider/getAiModel.ts create mode 100644 server/routers/aiProvider/getAiProvider.ts create mode 100644 server/routers/aiProvider/index.ts create mode 100644 server/routers/aiProvider/listAiModels.ts create mode 100644 server/routers/aiProvider/listAiProviders.ts create mode 100644 server/routers/aiProvider/types.ts create mode 100644 server/routers/aiProvider/updateAiModel.ts create mode 100644 server/routers/aiProvider/updateAiProvider.ts create mode 100644 server/routers/aiProvider/validation.ts diff --git a/messages/en-US.json b/messages/en-US.json index aff3be28b..24d10ab95 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1424,6 +1424,16 @@ "actionDeleteSite": "Delete Site", "actionGetSite": "Get Site", "actionListSites": "List Sites", + "actionCreateAiProvider": "Create AI Provider", + "actionDeleteAiProvider": "Delete AI Provider", + "actionGetAiProvider": "Get AI Provider", + "actionListAiProviders": "List AI Providers", + "actionUpdateAiProvider": "Update AI Provider", + "actionCreateAiModel": "Create AI Model", + "actionDeleteAiModel": "Delete AI Model", + "actionGetAiModel": "Get AI Model", + "actionListAiModels": "List AI Models", + "actionUpdateAiModel": "Update AI Model", "actionApplyBlueprint": "Apply Blueprint", "actionListBlueprints": "List Blueprints", "actionGetBlueprint": "Get Blueprint", diff --git a/server/auth/actions.ts b/server/auth/actions.ts index 741d7a057..944adbff4 100644 --- a/server/auth/actions.ts +++ b/server/auth/actions.ts @@ -182,7 +182,17 @@ export enum ActionsEnum { setResourcePolicyHeaderAuth = "setResourcePolicyHeaderAuth", setResourcePolicyWhitelist = "setResourcePolicyWhitelist", setResourcePolicyRules = "setResourcePolicyRules", - createOrgWideLauncherView = "createOrgWideLauncherView" + createOrgWideLauncherView = "createOrgWideLauncherView", + createAiProvider = "createAiProvider", + deleteAiProvider = "deleteAiProvider", + getAiProvider = "getAiProvider", + listAiProviders = "listAiProviders", + updateAiProvider = "updateAiProvider", + createAiModel = "createAiModel", + deleteAiModel = "deleteAiModel", + getAiModel = "getAiModel", + listAiModels = "listAiModels", + updateAiModel = "updateAiModel" } export async function checkUserActionPermission( diff --git a/server/index.ts b/server/index.ts index 53b3e9a69..88612b86a 100644 --- a/server/index.ts +++ b/server/index.ts @@ -9,6 +9,8 @@ import { createIntegrationApiServer } from "./integrationApiServer"; import { ApiKey, ApiKeyOrg, + AiModel, + AiProvider, RemoteExitNode, Session, SiteResource, @@ -83,6 +85,8 @@ declare global { userOrgIds?: string[]; remoteExitNode?: RemoteExitNode; siteResource?: SiteResource; + aiProvider?: AiProvider; + aiModel?: AiModel; orgPolicyAllowed?: boolean; } } diff --git a/server/lib/aiProviderDefaults.ts b/server/lib/aiProviderDefaults.ts new file mode 100644 index 000000000..d89be99a0 --- /dev/null +++ b/server/lib/aiProviderDefaults.ts @@ -0,0 +1,48 @@ +export type AiProviderType = "openai" | "anthropic" | "bedrock" | "custom"; +export type AiProviderAuthType = "bearer"; +export type AiBudgetUnit = "usd" | "tokens"; + +type AiProviderDefaults = { + upstreamUrl: string; + authType: AiProviderAuthType; +}; + +export const AI_PROVIDER_DEFAULTS: Record< + Exclude, + AiProviderDefaults +> = { + openai: { + upstreamUrl: "https://api.openai.com/v1", + authType: "bearer" + }, + anthropic: { + upstreamUrl: "https://api.anthropic.com", + authType: "bearer" + }, + bedrock: { + upstreamUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", + authType: "bearer" + } +}; + +export function resolveAiProviderConfig(input: { + type: AiProviderType; + upstreamUrl: string | null; + authType: AiProviderAuthType | null; +}): { + upstreamUrl: string | null; + authType: AiProviderAuthType | null; +} { + if (input.type === "custom") { + return { + upstreamUrl: input.upstreamUrl, + authType: input.authType + }; + } + + const defaults = AI_PROVIDER_DEFAULTS[input.type]; + return { + upstreamUrl: input.upstreamUrl ?? defaults.upstreamUrl, + authType: input.authType ?? defaults.authType + }; +} diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts index a7f3ae125..8f6b50e33 100644 --- a/server/middlewares/index.ts +++ b/server/middlewares/index.ts @@ -27,6 +27,8 @@ export * from "./verifyUserHasAction"; export * from "./verifyApiKeyAccess"; export * from "./verifySiteProvisioningKeyAccess"; export * from "./verifyDomainAccess"; +export * from "./verifyAiProviderAccess"; +export * from "./verifyAiModelAccess"; export * from "./verifyUserIsOrgOwner"; export * from "./verifyUserFromResourceSession"; export * from "./verifySiteResourceAccess"; diff --git a/server/middlewares/integration/index.ts b/server/middlewares/integration/index.ts index be9d400cf..63df4b2bb 100644 --- a/server/middlewares/integration/index.ts +++ b/server/middlewares/integration/index.ts @@ -16,5 +16,7 @@ export * from "./verifyApiKeyClientAccess"; export * from "./verifyApiKeySiteResourceAccess"; export * from "./verifyApiKeyIdpAccess"; export * from "./verifyApiKeyDomainAccess"; +export * from "./verifyApiKeyAiProviderAccess"; +export * from "./verifyApiKeyAiModelAccess"; export * from "./verifyApiKeyResourcePolicyAccess"; export * from "./verifyApiKeySiteProvisioningKeyAccess"; diff --git a/server/middlewares/integration/verifyApiKeyAiModelAccess.ts b/server/middlewares/integration/verifyApiKeyAiModelAccess.ts new file mode 100644 index 000000000..75a56f37e --- /dev/null +++ b/server/middlewares/integration/verifyApiKeyAiModelAccess.ts @@ -0,0 +1,113 @@ +import { Request, Response, NextFunction } from "express"; +import { aiModels, aiProviders, apiKeyOrg, db } from "@server/db"; +import { and, eq } from "drizzle-orm"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { getFirstString } from "@server/lib/requestParams"; + +export async function verifyApiKeyAiModelAccess( + req: Request, + res: Response, + next: NextFunction +) { + try { + const apiKey = req.apiKey; + const modelIdRaw = getFirstString(req.params.modelId); + const modelId = Number.parseInt(modelIdRaw ?? "", 10); + const providerIdRaw = getFirstString(req.params.providerId); + const providerId = Number.parseInt(providerIdRaw ?? "", 10); + const orgId = getFirstString(req.params.orgId); + + if (!apiKey) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "Key not authenticated") + ); + } + + if (!orgId) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") + ); + } + + if (Number.isNaN(providerId)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") + ); + } + + if (Number.isNaN(modelId)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid model ID") + ); + } + + const [row] = await db + .select({ + model: aiModels, + provider: aiProviders + }) + .from(aiModels) + .innerJoin( + aiProviders, + eq(aiModels.providerId, aiProviders.providerId) + ) + .where( + and( + eq(aiModels.modelId, modelId), + eq(aiModels.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!row) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI model with ID ${modelId} not found` + ) + ); + } + + if (apiKey.isRoot) { + req.aiProvider = row.provider; + req.aiModel = row.model; + return next(); + } + + if (!req.apiKeyOrg) { + const apiKeyOrgRes = await db + .select() + .from(apiKeyOrg) + .where( + and( + eq(apiKeyOrg.apiKeyId, apiKey.apiKeyId), + eq(apiKeyOrg.orgId, orgId) + ) + ) + .limit(1); + req.apiKeyOrg = apiKeyOrgRes[0]; + } + + if (!req.apiKeyOrg) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "Key does not have access to this organization" + ) + ); + } + + req.aiProvider = row.provider; + req.aiModel = row.model; + return next(); + } catch (error) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Error verifying AI model access" + ) + ); + } +} diff --git a/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts b/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts new file mode 100644 index 000000000..6b9f9b0ec --- /dev/null +++ b/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts @@ -0,0 +1,115 @@ +import { Request, Response, NextFunction } from "express"; +import { aiProviders, apiKeyOrg, db } from "@server/db"; +import { and, eq } from "drizzle-orm"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { getFirstString } from "@server/lib/requestParams"; + +export async function verifyApiKeyAiProviderAccess( + req: Request, + res: Response, + next: NextFunction +) { + try { + const apiKey = req.apiKey; + const providerIdRaw = getFirstString(req.params.providerId); + const providerId = Number.parseInt(providerIdRaw ?? "", 10); + const orgId = getFirstString(req.params.orgId); + + if (!apiKey) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "Key not authenticated") + ); + } + + if (!orgId) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") + ); + } + + if (Number.isNaN(providerId)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") + ); + } + + if (apiKey.isRoot) { + const [provider] = await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + req.aiProvider = provider; + return next(); + } + + const [provider] = await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + if (!req.apiKeyOrg) { + const apiKeyOrgRes = await db + .select() + .from(apiKeyOrg) + .where( + and( + eq(apiKeyOrg.apiKeyId, apiKey.apiKeyId), + eq(apiKeyOrg.orgId, orgId) + ) + ) + .limit(1); + req.apiKeyOrg = apiKeyOrgRes[0]; + } + + if (!req.apiKeyOrg) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "Key does not have access to this organization" + ) + ); + } + + req.aiProvider = provider; + return next(); + } catch (error) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Error verifying AI provider access" + ) + ); + } +} diff --git a/server/middlewares/verifyAiModelAccess.ts b/server/middlewares/verifyAiModelAccess.ts new file mode 100644 index 000000000..99693b982 --- /dev/null +++ b/server/middlewares/verifyAiModelAccess.ts @@ -0,0 +1,125 @@ +import { Request, Response, NextFunction } from "express"; +import { aiModels, aiProviders, db, userOrgs } from "@server/db"; +import { and, eq } from "drizzle-orm"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy"; +import { getUserOrgRoleIds } from "@server/lib/userOrgRoles"; +import { getFirstString } from "@server/lib/requestParams"; + +export async function verifyAiModelAccess( + req: Request, + res: Response, + next: NextFunction +) { + try { + const userId = req.user!.userId; + const modelIdRaw = getFirstString(req.params.modelId); + const modelId = Number.parseInt(modelIdRaw ?? "", 10); + const providerIdRaw = getFirstString(req.params.providerId); + const providerId = Number.parseInt(providerIdRaw ?? "", 10); + const orgId = getFirstString(req.params.orgId); + + if (!userId) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "User not authenticated") + ); + } + + if (!orgId) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") + ); + } + + if (Number.isNaN(providerId)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") + ); + } + + if (Number.isNaN(modelId)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid model ID") + ); + } + + const [row] = await db + .select({ + model: aiModels, + provider: aiProviders + }) + .from(aiModels) + .innerJoin( + aiProviders, + eq(aiModels.providerId, aiProviders.providerId) + ) + .where( + and( + eq(aiModels.modelId, modelId), + eq(aiModels.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!row) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI model with ID ${modelId} not found` + ) + ); + } + + if (!req.userOrg) { + const userOrgRole = await db + .select() + .from(userOrgs) + .where( + and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)) + ) + .limit(1); + req.userOrg = userOrgRole[0]; + } + + if (!req.userOrg) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + if (req.orgPolicyAllowed === undefined && req.userOrg.orgId) { + const policyCheck = await checkOrgAccessPolicy({ + orgId: req.userOrg.orgId, + userId, + session: req.session + }); + req.orgPolicyAllowed = policyCheck.allowed; + if (!policyCheck.allowed || policyCheck.error) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "" + (policyCheck.error || "Unknown error") + ) + ); + } + } + + req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId); + req.aiProvider = row.provider; + req.aiModel = row.model; + + return next(); + } catch (error) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Error verifying AI model access" + ) + ); + } +} diff --git a/server/middlewares/verifyAiProviderAccess.ts b/server/middlewares/verifyAiProviderAccess.ts new file mode 100644 index 000000000..604e3e893 --- /dev/null +++ b/server/middlewares/verifyAiProviderAccess.ts @@ -0,0 +1,108 @@ +import { Request, Response, NextFunction } from "express"; +import { aiProviders, db, userOrgs } from "@server/db"; +import { and, eq } from "drizzle-orm"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy"; +import { getUserOrgRoleIds } from "@server/lib/userOrgRoles"; +import { getFirstString } from "@server/lib/requestParams"; + +export async function verifyAiProviderAccess( + req: Request, + res: Response, + next: NextFunction +) { + try { + const userId = req.user!.userId; + const providerIdRaw = getFirstString(req.params.providerId); + const providerId = Number.parseInt(providerIdRaw ?? "", 10); + const orgId = getFirstString(req.params.orgId); + + if (!userId) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "User not authenticated") + ); + } + + if (!orgId) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") + ); + } + + if (Number.isNaN(providerId)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") + ); + } + + const [provider] = await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + if (!req.userOrg) { + const userOrgRole = await db + .select() + .from(userOrgs) + .where( + and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)) + ) + .limit(1); + req.userOrg = userOrgRole[0]; + } + + if (!req.userOrg) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + if (req.orgPolicyAllowed === undefined && req.userOrg.orgId) { + const policyCheck = await checkOrgAccessPolicy({ + orgId: req.userOrg.orgId, + userId, + session: req.session + }); + req.orgPolicyAllowed = policyCheck.allowed; + if (!policyCheck.allowed || policyCheck.error) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "" + (policyCheck.error || "Unknown error") + ) + ); + } + } + + req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId); + req.aiProvider = provider; + + return next(); + } catch (error) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Error verifying AI provider access" + ) + ); + } +} diff --git a/server/openApi.ts b/server/openApi.ts index 27c26b2da..1bcaa23ab 100644 --- a/server/openApi.ts +++ b/server/openApi.ts @@ -28,7 +28,9 @@ export enum OpenAPITags { HealthCheck = "Health Check", PublicResourcePolicyLegacy = "Public Resource Policy (Legacy)", PublicResourceLegacy = "Public Resource (Legacy)", - PrivateResourceLegacy = "Private Resource (Legacy)" + PrivateResourceLegacy = "Private Resource (Legacy)", + AiProvider = "AI Provider", + AiModel = "AI Model" } // Order here controls the order tags are displayed in Swagger UI diff --git a/server/routers/aiProvider/createAiModel.ts b/server/routers/aiProvider/createAiModel.ts new file mode 100644 index 000000000..a770a5ed8 --- /dev/null +++ b/server/routers/aiProvider/createAiModel.ts @@ -0,0 +1,157 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiModels, aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq } from "drizzle-orm"; +import type { CreateOrEditAiModelResponse } from "@server/routers/aiProvider/types"; +import { + aiBudgetUnitSchema, + refineBudgetFields +} from "@server/routers/aiProvider/validation"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty(), + providerId: z.coerce.number().int().positive() +}); + +const bodySchema = z + .strictObject({ + modelKey: z.string().nonempty(), + name: z.string().nonempty(), + budgetAmount: z.number().positive().optional().nullable(), + budgetUnit: aiBudgetUnitSchema.optional().nullable(), + enabled: z.boolean().optional() + }) + .superRefine((data, ctx) => { + refineBudgetFields(data, ctx); + }); + +registry.registerPath({ + method: "put", + path: "/org/{orgId}/ai-provider/{providerId}/model", + description: "Create an AI model under a provider.", + tags: [OpenAPITags.AiModel], + request: { + params: paramsSchema, + body: { + content: { + "application/json": { + schema: bodySchema + } + } + } + }, + responses: { + 201: { + description: "Successful response" + } + } +}); + +export async function createAiModel( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedBody = bodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { orgId, providerId } = parsedParams.data; + const { modelKey, name, budgetAmount, budgetUnit, enabled } = + parsedBody.data; + + const [provider] = + req.aiProvider && req.aiProvider.providerId === providerId + ? [req.aiProvider] + : await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + const [existing] = await db + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .where( + and( + eq(aiModels.providerId, providerId), + eq(aiModels.modelKey, modelKey) + ) + ) + .limit(1); + + if (existing) { + return next( + createHttpError( + HttpCode.CONFLICT, + `Model with key ${modelKey} already exists for this provider` + ) + ); + } + + const now = Date.now(); + const [model] = await db + .insert(aiModels) + .values({ + providerId, + modelKey, + name, + budgetAmount: budgetAmount ?? null, + budgetUnit: budgetUnit ?? null, + enabled: enabled ?? true, + createdAt: now, + updatedAt: now + }) + .returning(); + + return response(res, { + data: { model }, + success: true, + error: false, + message: "AI model created successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts new file mode 100644 index 000000000..dad11d18d --- /dev/null +++ b/server/routers/aiProvider/createAiProvider.ts @@ -0,0 +1,141 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { encrypt } from "@server/lib/crypto"; +import config from "@server/lib/config"; +import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; +import { toPublicAiProvider } from "@server/routers/aiProvider/types"; +import { + aiAuthTypeSchema, + aiBudgetUnitSchema, + aiProviderTypeSchema, + refineBudgetFields, + refineCustomProviderFields +} from "@server/routers/aiProvider/validation"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +const bodySchema = z + .strictObject({ + name: z.string().nonempty(), + type: aiProviderTypeSchema, + upstreamUrl: z.url().optional().nullable(), + apiKey: z.string().optional(), + authType: aiAuthTypeSchema.optional().nullable(), + skipTlsVerification: z.boolean().optional(), + budgetAmount: z.number().positive().optional().nullable(), + budgetUnit: aiBudgetUnitSchema.optional().nullable(), + enabled: z.boolean().optional() + }) + .superRefine((data, ctx) => { + refineCustomProviderFields(data, ctx); + refineBudgetFields(data, ctx); + }); + +registry.registerPath({ + method: "put", + path: "/org/{orgId}/ai-provider", + description: "Create an AI provider for an organization.", + tags: [OpenAPITags.AiProvider], + request: { + params: paramsSchema, + body: { + content: { + "application/json": { + schema: bodySchema + } + } + } + }, + responses: { + 201: { + description: "Successful response" + } + } +}); + +export async function createAiProvider( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedBody = bodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { orgId } = parsedParams.data; + const { + name, + type, + upstreamUrl, + apiKey, + authType, + skipTlsVerification, + budgetAmount, + budgetUnit, + enabled + } = parsedBody.data; + + const key = config.getRawConfig().server.secret!; + const encryptedApiKey = apiKey ? encrypt(apiKey, key) : null; + const apiKeyLastChars = apiKey ? apiKey.slice(-4) : null; + const now = Date.now(); + + const [provider] = await db + .insert(aiProviders) + .values({ + orgId, + name, + type, + upstreamUrl: upstreamUrl ?? null, + apiKey: encryptedApiKey, + apiKeyLastChars, + authType: authType ?? null, + skipTlsVerification: skipTlsVerification ?? false, + budgetAmount: budgetAmount ?? null, + budgetUnit: budgetUnit ?? null, + enabled: enabled ?? true, + createdAt: now, + updatedAt: now + }) + .returning(); + + return response(res, { + data: { provider: toPublicAiProvider(provider) }, + success: true, + error: false, + message: "AI provider created successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/deleteAiModel.ts b/server/routers/aiProvider/deleteAiModel.ts new file mode 100644 index 000000000..be784275e --- /dev/null +++ b/server/routers/aiProvider/deleteAiModel.ts @@ -0,0 +1,98 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiModels, aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq } from "drizzle-orm"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty(), + providerId: z.coerce.number().int().positive(), + modelId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "delete", + path: "/org/{orgId}/ai-provider/{providerId}/model/{modelId}", + description: "Delete an AI model.", + tags: [OpenAPITags.AiModel], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function deleteAiModel( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { orgId, providerId, modelId } = parsedParams.data; + + const [existing] = await db + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .innerJoin( + aiProviders, + eq(aiModels.providerId, aiProviders.providerId) + ) + .where( + and( + eq(aiModels.modelId, modelId), + eq(aiModels.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!existing) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI model with ID ${modelId} not found` + ) + ); + } + + await db + .delete(aiModels) + .where( + and( + eq(aiModels.modelId, modelId), + eq(aiModels.providerId, providerId) + ) + ); + + return response(res, { + data: null, + success: true, + error: false, + message: "AI model deleted successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/deleteAiProvider.ts b/server/routers/aiProvider/deleteAiProvider.ts new file mode 100644 index 000000000..b5416c23c --- /dev/null +++ b/server/routers/aiProvider/deleteAiProvider.ts @@ -0,0 +1,92 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq } from "drizzle-orm"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty(), + providerId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "delete", + path: "/org/{orgId}/ai-provider/{providerId}", + description: "Delete an AI provider.", + tags: [OpenAPITags.AiProvider], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function deleteAiProvider( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { orgId, providerId } = parsedParams.data; + + const [existing] = await db + .select({ providerId: aiProviders.providerId }) + .from(aiProviders) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!existing) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + await db + .delete(aiProviders) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ); + + return response(res, { + data: null, + success: true, + error: false, + message: "AI provider deleted successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/getAiModel.ts b/server/routers/aiProvider/getAiModel.ts new file mode 100644 index 000000000..6ea8795ee --- /dev/null +++ b/server/routers/aiProvider/getAiModel.ts @@ -0,0 +1,94 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiModels, aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq } from "drizzle-orm"; +import type { GetAiModelResponse } from "@server/routers/aiProvider/types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty(), + providerId: z.coerce.number().int().positive(), + modelId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/ai-provider/{providerId}/model/{modelId}", + description: "Get an AI model by ID.", + tags: [OpenAPITags.AiModel], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function getAiModel( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { orgId, providerId, modelId } = parsedParams.data; + + const [model] = + req.aiModel && req.aiModel.modelId === modelId + ? [req.aiModel] + : await db + .select({ model: aiModels }) + .from(aiModels) + .innerJoin( + aiProviders, + eq(aiModels.providerId, aiProviders.providerId) + ) + .where( + and( + eq(aiModels.modelId, modelId), + eq(aiModels.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1) + .then((rows) => rows.map((r) => r.model)); + + if (!model) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI model with ID ${modelId} not found` + ) + ); + } + + return response(res, { + data: { model }, + success: true, + error: false, + message: "AI model retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/getAiProvider.ts b/server/routers/aiProvider/getAiProvider.ts new file mode 100644 index 000000000..e2abd3948 --- /dev/null +++ b/server/routers/aiProvider/getAiProvider.ts @@ -0,0 +1,88 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq } from "drizzle-orm"; +import type { GetAiProviderResponse } from "@server/routers/aiProvider/types"; +import { toPublicAiProvider } from "@server/routers/aiProvider/types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty(), + providerId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/ai-provider/{providerId}", + description: "Get an AI provider by ID.", + tags: [OpenAPITags.AiProvider], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function getAiProvider( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { orgId, providerId } = parsedParams.data; + + const [provider] = + req.aiProvider && req.aiProvider.providerId === providerId + ? [req.aiProvider] + : await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + return response(res, { + data: { provider: toPublicAiProvider(provider) }, + success: true, + error: false, + message: "AI provider retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/index.ts b/server/routers/aiProvider/index.ts new file mode 100644 index 000000000..34c900139 --- /dev/null +++ b/server/routers/aiProvider/index.ts @@ -0,0 +1,11 @@ +export * from "./createAiProvider"; +export * from "./listAiProviders"; +export * from "./getAiProvider"; +export * from "./updateAiProvider"; +export * from "./deleteAiProvider"; +export * from "./createAiModel"; +export * from "./listAiModels"; +export * from "./getAiModel"; +export * from "./updateAiModel"; +export * from "./deleteAiModel"; +export * from "./types"; diff --git a/server/routers/aiProvider/listAiModels.ts b/server/routers/aiProvider/listAiModels.ts new file mode 100644 index 000000000..a7794efca --- /dev/null +++ b/server/routers/aiProvider/listAiModels.ts @@ -0,0 +1,166 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiModels, aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, asc, eq, like, sql } from "drizzle-orm"; +import type { ListAiModelsResponse } from "@server/routers/aiProvider/types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty(), + providerId: z.coerce.number().int().positive() +}); + +const listSchema = z.object({ + pageSize: z.coerce + .number() + .int() + .positive() + .optional() + .catch(20) + .default(20) + .openapi({ + type: "integer", + default: 20, + description: "Number of items per page" + }), + page: z.coerce + .number() + .int() + .min(0) + .optional() + .catch(1) + .default(1) + .openapi({ + type: "integer", + default: 1, + description: "Page number to retrieve" + }), + query: z.string().optional() +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/ai-provider/{providerId}/models", + description: "List AI models for a provider.", + tags: [OpenAPITags.AiModel], + request: { + params: paramsSchema, + query: listSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listAiModels( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = listSchema.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error).toString() + ) + ); + } + + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { orgId, providerId } = parsedParams.data; + + const [provider] = + req.aiProvider && req.aiProvider.providerId === providerId + ? [req.aiProvider] + : await db + .select({ providerId: aiProviders.providerId }) + .from(aiProviders) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + const { pageSize, page, query } = parsedQuery.data; + const conditions = [eq(aiModels.providerId, providerId)]; + + if (query) { + conditions.push( + like( + sql`LOWER(${aiModels.name})`, + "%" + query.toLowerCase() + "%" + ) + ); + } + + const baseQuery = db + .select() + .from(aiModels) + .where(and(...conditions)); + + const countQuery = db.$count( + db + .select() + .from(aiModels) + .where(and(...conditions)) + .as("filtered_ai_models") + ); + + const [totalCount, rows] = await Promise.all([ + countQuery, + baseQuery + .limit(pageSize) + .offset(pageSize * (page - 1)) + .orderBy(asc(aiModels.name)) + ]); + + return response(res, { + data: { + models: rows, + pagination: { + total: totalCount, + pageSize, + page + } + }, + success: true, + error: false, + message: "AI models retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/listAiProviders.ts b/server/routers/aiProvider/listAiProviders.ts new file mode 100644 index 000000000..7ec89df51 --- /dev/null +++ b/server/routers/aiProvider/listAiProviders.ts @@ -0,0 +1,152 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, asc, eq, like, sql } from "drizzle-orm"; +import type { ListAiProvidersResponse } from "@server/routers/aiProvider/types"; +import { toPublicAiProvider } from "@server/routers/aiProvider/types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +const listSchema = z.object({ + pageSize: z.coerce + .number() + .int() + .positive() + .optional() + .catch(20) + .default(20) + .openapi({ + type: "integer", + default: 20, + description: "Number of items per page" + }), + page: z.coerce + .number() + .int() + .min(0) + .optional() + .catch(1) + .default(1) + .openapi({ + type: "integer", + default: 1, + description: "Page number to retrieve" + }), + query: z.string().optional() +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/ai-providers", + description: "List AI providers for an organization.", + tags: [OpenAPITags.AiProvider], + request: { + params: paramsSchema, + query: listSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listAiProviders( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = listSchema.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error).toString() + ) + ); + } + + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { orgId } = parsedParams.data; + + if (req.user && orgId && orgId !== req.userOrgId) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + const { pageSize, page, query } = parsedQuery.data; + const conditions = [eq(aiProviders.orgId, orgId)]; + + if (query) { + conditions.push( + like( + sql`LOWER(${aiProviders.name})`, + "%" + query.toLowerCase() + "%" + ) + ); + } + + const baseQuery = db + .select() + .from(aiProviders) + .where(and(...conditions)); + + const countQuery = db.$count( + db + .select() + .from(aiProviders) + .where(and(...conditions)) + .as("filtered_ai_providers") + ); + + const [totalCount, rows] = await Promise.all([ + countQuery, + baseQuery + .limit(pageSize) + .offset(pageSize * (page - 1)) + .orderBy(asc(aiProviders.name)) + ]); + + return response(res, { + data: { + providers: rows.map(toPublicAiProvider), + pagination: { + total: totalCount, + pageSize, + page + } + }, + success: true, + error: false, + message: "AI providers retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/types.ts b/server/routers/aiProvider/types.ts new file mode 100644 index 000000000..4b8461b5e --- /dev/null +++ b/server/routers/aiProvider/types.ts @@ -0,0 +1,51 @@ +import type { AiModel, AiProvider } from "@server/db"; +import type { PaginatedResponse } from "@server/types/Pagination"; +import { + resolveAiProviderConfig, + type AiProviderAuthType, + type AiProviderType +} from "@server/lib/aiProviderDefaults"; + +export type AiProviderPublic = Omit & { + effectiveUpstreamUrl: string | null; + effectiveAuthType: AiProviderAuthType | null; +}; + +export type ListAiProvidersResponse = PaginatedResponse<{ + providers: AiProviderPublic[]; +}>; + +export type GetAiProviderResponse = { + provider: AiProviderPublic; +}; + +export type CreateOrEditAiProviderResponse = { + provider: AiProviderPublic; +}; + +export type ListAiModelsResponse = PaginatedResponse<{ + models: AiModel[]; +}>; + +export type GetAiModelResponse = { + model: AiModel; +}; + +export type CreateOrEditAiModelResponse = { + model: AiModel; +}; + +export function toPublicAiProvider(provider: AiProvider): AiProviderPublic { + const { apiKey: _apiKey, ...rest } = provider; + const resolved = resolveAiProviderConfig({ + type: provider.type as AiProviderType, + upstreamUrl: provider.upstreamUrl, + authType: provider.authType as AiProviderAuthType | null + }); + + return { + ...rest, + effectiveUpstreamUrl: resolved.upstreamUrl, + effectiveAuthType: resolved.authType + }; +} diff --git a/server/routers/aiProvider/updateAiModel.ts b/server/routers/aiProvider/updateAiModel.ts new file mode 100644 index 000000000..ad09ee823 --- /dev/null +++ b/server/routers/aiProvider/updateAiModel.ts @@ -0,0 +1,185 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiModels, aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq, ne } from "drizzle-orm"; +import type { CreateOrEditAiModelResponse } from "@server/routers/aiProvider/types"; +import { + aiBudgetUnitSchema, + refineBudgetFields +} from "@server/routers/aiProvider/validation"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty(), + providerId: z.coerce.number().int().positive(), + modelId: z.coerce.number().int().positive() +}); + +const bodySchema = z + .strictObject({ + modelKey: z.string().nonempty().optional(), + name: z.string().nonempty().optional(), + budgetAmount: z.number().positive().optional().nullable(), + budgetUnit: aiBudgetUnitSchema.optional().nullable(), + enabled: z.boolean().optional() + }) + .superRefine((data, ctx) => { + refineBudgetFields(data, ctx); + }); + +registry.registerPath({ + method: "post", + path: "/org/{orgId}/ai-provider/{providerId}/model/{modelId}", + description: "Update an AI model.", + tags: [OpenAPITags.AiModel], + request: { + params: paramsSchema, + body: { + content: { + "application/json": { + schema: bodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function updateAiModel( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedBody = bodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { orgId, providerId, modelId } = parsedParams.data; + const body = parsedBody.data; + + const [existing] = + req.aiModel && req.aiModel.modelId === modelId + ? [req.aiModel] + : await db + .select({ model: aiModels }) + .from(aiModels) + .innerJoin( + aiProviders, + eq(aiModels.providerId, aiProviders.providerId) + ) + .where( + and( + eq(aiModels.modelId, modelId), + eq(aiModels.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1) + .then((rows) => rows.map((r) => r.model)); + + if (!existing) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI model with ID ${modelId} not found` + ) + ); + } + + if ( + body.modelKey !== undefined && + body.modelKey !== existing.modelKey + ) { + const [conflict] = await db + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .where( + and( + eq(aiModels.providerId, providerId), + eq(aiModels.modelKey, body.modelKey), + ne(aiModels.modelId, modelId) + ) + ) + .limit(1); + + if (conflict) { + return next( + createHttpError( + HttpCode.CONFLICT, + `Model with key ${body.modelKey} already exists for this provider` + ) + ); + } + } + + const updateData: Partial = { + updatedAt: Date.now() + }; + + if (body.modelKey !== undefined) { + updateData.modelKey = body.modelKey; + } + if (body.name !== undefined) { + updateData.name = body.name; + } + if (body.budgetAmount !== undefined) { + updateData.budgetAmount = body.budgetAmount; + } + if (body.budgetUnit !== undefined) { + updateData.budgetUnit = body.budgetUnit; + } + if (body.enabled !== undefined) { + updateData.enabled = body.enabled; + } + + const [model] = await db + .update(aiModels) + .set(updateData) + .where( + and( + eq(aiModels.modelId, modelId), + eq(aiModels.providerId, providerId) + ) + ) + .returning(); + + return response(res, { + data: { model }, + success: true, + error: false, + message: "AI model updated successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts new file mode 100644 index 000000000..8525e1dd3 --- /dev/null +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -0,0 +1,208 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq } from "drizzle-orm"; +import { encrypt } from "@server/lib/crypto"; +import config from "@server/lib/config"; +import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; +import { toPublicAiProvider } from "@server/routers/aiProvider/types"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import { + aiAuthTypeSchema, + aiBudgetUnitSchema, + refineBudgetFields, + refineCustomProviderFields +} from "@server/routers/aiProvider/validation"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty(), + providerId: z.coerce.number().int().positive() +}); + +const bodySchema = z + .strictObject({ + name: z.string().nonempty().optional(), + upstreamUrl: z.url().optional().nullable(), + apiKey: z.string().optional(), + authType: aiAuthTypeSchema.optional().nullable(), + skipTlsVerification: z.boolean().optional(), + budgetAmount: z.number().positive().optional().nullable(), + budgetUnit: aiBudgetUnitSchema.optional().nullable(), + enabled: z.boolean().optional() + }) + .superRefine((data, ctx) => { + refineBudgetFields(data, ctx); + }); + +registry.registerPath({ + method: "post", + path: "/org/{orgId}/ai-provider/{providerId}", + description: "Update an AI provider.", + tags: [OpenAPITags.AiProvider], + request: { + params: paramsSchema, + body: { + content: { + "application/json": { + schema: bodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function updateAiProvider( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedBody = bodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { orgId, providerId } = parsedParams.data; + const body = parsedBody.data; + + const [existing] = + req.aiProvider && req.aiProvider.providerId === providerId + ? [req.aiProvider] + : await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + + if (!existing) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + const providerType = existing.type as AiProviderType; + + if (providerType === "custom") { + const nextUpstreamUrl = + body.upstreamUrl !== undefined + ? body.upstreamUrl + : existing.upstreamUrl; + const nextAuthType = + body.authType !== undefined ? body.authType : existing.authType; + + const validation = z + .object({ + type: z.literal("custom"), + upstreamUrl: z.string().nullable().optional(), + authType: aiAuthTypeSchema.nullable().optional() + }) + .superRefine((data, ctx) => + refineCustomProviderFields(data, ctx) + ) + .safeParse({ + type: "custom", + upstreamUrl: nextUpstreamUrl, + authType: nextAuthType + }); + + if (!validation.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(validation.error).toString() + ) + ); + } + } + + const updateData: Partial = { + updatedAt: Date.now() + }; + + if (body.name !== undefined) { + updateData.name = body.name; + } + if (body.skipTlsVerification !== undefined) { + updateData.skipTlsVerification = body.skipTlsVerification; + } + if (body.enabled !== undefined) { + updateData.enabled = body.enabled; + } + if (body.budgetAmount !== undefined) { + updateData.budgetAmount = body.budgetAmount; + } + if (body.budgetUnit !== undefined) { + updateData.budgetUnit = body.budgetUnit; + } + if (body.upstreamUrl !== undefined) { + updateData.upstreamUrl = body.upstreamUrl; + } + if (body.authType !== undefined) { + updateData.authType = body.authType; + } + + if (body.apiKey !== undefined) { + const key = config.getRawConfig().server.secret!; + updateData.apiKey = encrypt(body.apiKey, key); + updateData.apiKeyLastChars = body.apiKey.slice(-4); + } + + const [provider] = await db + .update(aiProviders) + .set(updateData) + .where( + and( + eq(aiProviders.providerId, providerId), + eq(aiProviders.orgId, orgId) + ) + ) + .returning(); + + return response(res, { + data: { provider: toPublicAiProvider(provider) }, + success: true, + error: false, + message: "AI provider updated successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts new file mode 100644 index 000000000..62b442c59 --- /dev/null +++ b/server/routers/aiProvider/validation.ts @@ -0,0 +1,66 @@ +import { z } from "zod"; +import type { + AiBudgetUnit, + AiProviderType +} from "@server/lib/aiProviderDefaults"; + +export const aiProviderTypeSchema = z.enum([ + "openai", + "anthropic", + "bedrock", + "custom" +]); + +export const aiBudgetUnitSchema = z.enum(["usd", "tokens"]); + +export const aiAuthTypeSchema = z.enum(["bearer"]); + +export function refineBudgetFields( + data: { + budgetAmount?: number | null; + budgetUnit?: AiBudgetUnit | null; + }, + ctx: z.RefinementCtx +) { + const hasAmount = + data.budgetAmount !== undefined && data.budgetAmount !== null; + const hasUnit = data.budgetUnit !== undefined && data.budgetUnit !== null; + + if (hasAmount !== hasUnit) { + ctx.addIssue({ + code: "custom", + message: + "budgetAmount and budgetUnit must both be set or both omitted", + path: hasAmount ? ["budgetUnit"] : ["budgetAmount"] + }); + } +} + +export function refineCustomProviderFields( + data: { + type: AiProviderType; + upstreamUrl?: string | null; + authType?: "bearer" | null; + }, + ctx: z.RefinementCtx +) { + if (data.type !== "custom") { + return; + } + + if (!data.upstreamUrl) { + ctx.addIssue({ + code: "custom", + message: "upstreamUrl is required for custom providers", + path: ["upstreamUrl"] + }); + } + + if (!data.authType) { + ctx.addIssue({ + code: "custom", + message: "authType is required for custom providers", + path: ["authType"] + }); + } +} diff --git a/server/routers/external.ts b/server/routers/external.ts index ffe0a809b..1d33d6166 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -45,7 +45,9 @@ import { verifySiteResourceAccess, verifyOlmAccess, verifyLimits, - verifyResourcePolicyAccess + verifyResourcePolicyAccess, + verifyAiProviderAccess, + verifyAiModelAccess } from "@server/middlewares"; import { ActionsEnum } from "@server/auth/actions"; import rateLimit, { ipKeyGenerator } from "express-rate-limit"; @@ -55,6 +57,7 @@ import { createStore } from "#dynamic/lib/rateLimitStore"; import { logActionAudit } from "#dynamic/middlewares"; import { checkRoundTripMessage } from "./ws"; import * as labels from "@server/routers/labels"; +import * as aiProvider from "@server/routers/aiProvider"; // Root routes export const unauthenticated = Router(); @@ -1366,6 +1369,90 @@ authenticated.get( authenticated.get("/ws/round-trip-message/:messageId", checkRoundTripMessage); +authenticated.put( + "/org/:orgId/ai-provider", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.createAiProvider), + logActionAudit(ActionsEnum.createAiProvider), + aiProvider.createAiProvider +); + +authenticated.get( + "/org/:orgId/ai-providers", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.listAiProviders), + aiProvider.listAiProviders +); + +authenticated.get( + "/org/:orgId/ai-provider/:providerId", + verifyOrgAccess, + verifyAiProviderAccess, + verifyUserHasAction(ActionsEnum.getAiProvider), + aiProvider.getAiProvider +); + +authenticated.post( + "/org/:orgId/ai-provider/:providerId", + verifyOrgAccess, + verifyAiProviderAccess, + verifyUserHasAction(ActionsEnum.updateAiProvider), + logActionAudit(ActionsEnum.updateAiProvider), + aiProvider.updateAiProvider +); + +authenticated.delete( + "/org/:orgId/ai-provider/:providerId", + verifyOrgAccess, + verifyAiProviderAccess, + verifyUserHasAction(ActionsEnum.deleteAiProvider), + logActionAudit(ActionsEnum.deleteAiProvider), + aiProvider.deleteAiProvider +); + +authenticated.put( + "/org/:orgId/ai-provider/:providerId/model", + verifyOrgAccess, + verifyAiProviderAccess, + verifyUserHasAction(ActionsEnum.createAiModel), + logActionAudit(ActionsEnum.createAiModel), + aiProvider.createAiModel +); + +authenticated.get( + "/org/:orgId/ai-provider/:providerId/models", + verifyOrgAccess, + verifyAiProviderAccess, + verifyUserHasAction(ActionsEnum.listAiModels), + aiProvider.listAiModels +); + +authenticated.get( + "/org/:orgId/ai-provider/:providerId/model/:modelId", + verifyOrgAccess, + verifyAiModelAccess, + verifyUserHasAction(ActionsEnum.getAiModel), + aiProvider.getAiModel +); + +authenticated.post( + "/org/:orgId/ai-provider/:providerId/model/:modelId", + verifyOrgAccess, + verifyAiModelAccess, + verifyUserHasAction(ActionsEnum.updateAiModel), + logActionAudit(ActionsEnum.updateAiModel), + aiProvider.updateAiModel +); + +authenticated.delete( + "/org/:orgId/ai-provider/:providerId/model/:modelId", + verifyOrgAccess, + verifyAiModelAccess, + verifyUserHasAction(ActionsEnum.deleteAiModel), + logActionAudit(ActionsEnum.deleteAiModel), + aiProvider.deleteAiModel +); + authenticated.get( "/org/:orgId/labels", verifyOrgAccess, diff --git a/server/routers/integration.ts b/server/routers/integration.ts index 13498ad25..3a17079c6 100644 --- a/server/routers/integration.ts +++ b/server/routers/integration.ts @@ -13,6 +13,7 @@ import * as apiKeys from "./apiKeys"; import * as idp from "./idp"; import * as logs from "./auditLogs"; import * as siteResource from "./siteResource"; +import * as aiProvider from "./aiProvider"; import { verifyApiKey, verifyApiKeyOrgAccess, @@ -31,6 +32,8 @@ import { verifyLimits, verifyApiKeyDomainAccess, verifyApiKeyResourcePolicyAccess, + verifyApiKeyAiProviderAccess, + verifyApiKeyAiModelAccess, verifyUserHasAction } from "@server/middlewares"; import HttpCode from "@server/types/HttpCode"; @@ -1366,3 +1369,87 @@ authenticated.get( verifyApiKeyHasAction(ActionsEnum.listResources), resource.listAllResourceNames ); + +authenticated.put( + "/org/:orgId/ai-provider", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.createAiProvider), + logActionAudit(ActionsEnum.createAiProvider), + aiProvider.createAiProvider +); + +authenticated.get( + "/org/:orgId/ai-providers", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.listAiProviders), + aiProvider.listAiProviders +); + +authenticated.get( + "/org/:orgId/ai-provider/:providerId", + verifyApiKeyOrgAccess, + verifyApiKeyAiProviderAccess, + verifyApiKeyHasAction(ActionsEnum.getAiProvider), + aiProvider.getAiProvider +); + +authenticated.post( + "/org/:orgId/ai-provider/:providerId", + verifyApiKeyOrgAccess, + verifyApiKeyAiProviderAccess, + verifyApiKeyHasAction(ActionsEnum.updateAiProvider), + logActionAudit(ActionsEnum.updateAiProvider), + aiProvider.updateAiProvider +); + +authenticated.delete( + "/org/:orgId/ai-provider/:providerId", + verifyApiKeyOrgAccess, + verifyApiKeyAiProviderAccess, + verifyApiKeyHasAction(ActionsEnum.deleteAiProvider), + logActionAudit(ActionsEnum.deleteAiProvider), + aiProvider.deleteAiProvider +); + +authenticated.put( + "/org/:orgId/ai-provider/:providerId/model", + verifyApiKeyOrgAccess, + verifyApiKeyAiProviderAccess, + verifyApiKeyHasAction(ActionsEnum.createAiModel), + logActionAudit(ActionsEnum.createAiModel), + aiProvider.createAiModel +); + +authenticated.get( + "/org/:orgId/ai-provider/:providerId/models", + verifyApiKeyOrgAccess, + verifyApiKeyAiProviderAccess, + verifyApiKeyHasAction(ActionsEnum.listAiModels), + aiProvider.listAiModels +); + +authenticated.get( + "/org/:orgId/ai-provider/:providerId/model/:modelId", + verifyApiKeyOrgAccess, + verifyApiKeyAiModelAccess, + verifyApiKeyHasAction(ActionsEnum.getAiModel), + aiProvider.getAiModel +); + +authenticated.post( + "/org/:orgId/ai-provider/:providerId/model/:modelId", + verifyApiKeyOrgAccess, + verifyApiKeyAiModelAccess, + verifyApiKeyHasAction(ActionsEnum.updateAiModel), + logActionAudit(ActionsEnum.updateAiModel), + aiProvider.updateAiModel +); + +authenticated.delete( + "/org/:orgId/ai-provider/:providerId/model/:modelId", + verifyApiKeyOrgAccess, + verifyApiKeyAiModelAccess, + verifyApiKeyHasAction(ActionsEnum.deleteAiModel), + logActionAudit(ActionsEnum.deleteAiModel), + aiProvider.deleteAiModel +); diff --git a/src/components/PermissionsSelectBox.tsx b/src/components/PermissionsSelectBox.tsx index 48868722e..fe33337d4 100644 --- a/src/components/PermissionsSelectBox.tsx +++ b/src/components/PermissionsSelectBox.tsx @@ -150,6 +150,22 @@ function getActionsCategories(root: boolean) { [t("actionListSiteProvisioningKeys")]: "listSiteProvisioningKeys", [t("actionUpdateSiteProvisioningKey")]: "updateSiteProvisioningKey", [t("actionDeleteSiteProvisioningKey")]: "deleteSiteProvisioningKey" + }, + + "AI Provider": { + [t("actionCreateAiProvider")]: "createAiProvider", + [t("actionDeleteAiProvider")]: "deleteAiProvider", + [t("actionGetAiProvider")]: "getAiProvider", + [t("actionListAiProviders")]: "listAiProviders", + [t("actionUpdateAiProvider")]: "updateAiProvider" + }, + + "AI Model": { + [t("actionCreateAiModel")]: "createAiModel", + [t("actionDeleteAiModel")]: "deleteAiModel", + [t("actionGetAiModel")]: "getAiModel", + [t("actionListAiModels")]: "listAiModels", + [t("actionUpdateAiModel")]: "updateAiModel" } }; From e5ac6ec7cd701bd186db12f813de59e2bc4005fd Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 31 Jul 2026 15:55:10 -0400 Subject: [PATCH 013/296] add default providers and support overrides --- server/db/pg/schema/schema.ts | 12 +++++- server/db/sqlite/schema/schema.ts | 12 +++++- server/lib/aiProviderDefaults.ts | 41 ++++++++++++++++++- server/routers/aiProvider/createAiProvider.ts | 4 +- server/routers/aiProvider/updateAiProvider.ts | 33 +++++++++------ server/routers/aiProvider/validation.ts | 24 ++++++----- 6 files changed, 96 insertions(+), 30 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 83f1e19bd..2f8199aa8 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1554,7 +1554,17 @@ export const aiProviders = pgTable("aiProviders", { .references(() => orgs.orgId, { onDelete: "cascade" }), name: varchar("name").notNull(), type: varchar("type") - .$type<"openai" | "anthropic" | "bedrock" | "custom">() + .$type< + | "openai" + | "anthropic" + | "googleGemini" + | "vertexAi" + | "bedrock" + | "microsoftFoundry" + | "openRouter" + | "vercelAiGateway" + | "custom" + >() .notNull(), upstreamUrl: text("upstreamUrl"), apiKey: text("apiKey"), diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 97059d3cb..4cc533475 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1545,7 +1545,17 @@ export const aiProviders = sqliteTable("aiProviders", { .references(() => orgs.orgId, { onDelete: "cascade" }), name: text("name").notNull(), type: text("type") - .$type<"openai" | "anthropic" | "bedrock" | "custom">() + .$type< + | "openai" + | "anthropic" + | "googleGemini" + | "vertexAi" + | "bedrock" + | "microsoftFoundry" + | "openRouter" + | "vercelAiGateway" + | "custom" + >() .notNull(), upstreamUrl: text("upstreamUrl"), apiKey: text("apiKey"), diff --git a/server/lib/aiProviderDefaults.ts b/server/lib/aiProviderDefaults.ts index d89be99a0..63c060b62 100644 --- a/server/lib/aiProviderDefaults.ts +++ b/server/lib/aiProviderDefaults.ts @@ -1,9 +1,19 @@ -export type AiProviderType = "openai" | "anthropic" | "bedrock" | "custom"; +export type AiProviderType = + | "openai" + | "anthropic" + | "googleGemini" + | "vertexAi" + | "bedrock" + | "microsoftFoundry" + | "openRouter" + | "vercelAiGateway" + | "custom"; + export type AiProviderAuthType = "bearer"; export type AiBudgetUnit = "usd" | "tokens"; type AiProviderDefaults = { - upstreamUrl: string; + upstreamUrl: string | null; authType: AiProviderAuthType; }; @@ -19,12 +29,39 @@ export const AI_PROVIDER_DEFAULTS: Record< upstreamUrl: "https://api.anthropic.com", authType: "bearer" }, + googleGemini: { + upstreamUrl: "https://generativelanguage.googleapis.com/v1beta/openai/", + authType: "bearer" + }, + vertexAi: { + upstreamUrl: null, + authType: "bearer" + }, bedrock: { upstreamUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", authType: "bearer" + }, + microsoftFoundry: { + upstreamUrl: null, + authType: "bearer" + }, + openRouter: { + upstreamUrl: "https://openrouter.ai/api/v1", + authType: "bearer" + }, + vercelAiGateway: { + upstreamUrl: "https://ai-gateway.vercel.sh/v1", + authType: "bearer" } }; +export function providerRequiresUpstreamUrl(type: AiProviderType): boolean { + if (type === "custom") { + return true; + } + return AI_PROVIDER_DEFAULTS[type].upstreamUrl === null; +} + export function resolveAiProviderConfig(input: { type: AiProviderType; upstreamUrl: string | null; diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index dad11d18d..27420bd34 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -16,7 +16,7 @@ import { aiBudgetUnitSchema, aiProviderTypeSchema, refineBudgetFields, - refineCustomProviderFields + refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; const paramsSchema = z.strictObject({ @@ -36,7 +36,7 @@ const bodySchema = z enabled: z.boolean().optional() }) .superRefine((data, ctx) => { - refineCustomProviderFields(data, ctx); + refineProviderUpstreamFields(data, ctx); refineBudgetFields(data, ctx); }); diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts index 8525e1dd3..186b5bcd8 100644 --- a/server/routers/aiProvider/updateAiProvider.ts +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -12,13 +12,17 @@ import { encrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import { toPublicAiProvider } from "@server/routers/aiProvider/types"; -import type { AiProviderType } from "@server/lib/aiProviderDefaults"; import { aiAuthTypeSchema, aiBudgetUnitSchema, + aiProviderTypeSchema, refineBudgetFields, - refineCustomProviderFields + refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; +import { + providerRequiresUpstreamUrl, + type AiProviderType +} from "@server/lib/aiProviderDefaults"; const paramsSchema = z.strictObject({ orgId: z.string().nonempty(), @@ -115,26 +119,29 @@ export async function updateAiProvider( } const providerType = existing.type as AiProviderType; + const nextUpstreamUrl = + body.upstreamUrl !== undefined + ? body.upstreamUrl + : existing.upstreamUrl; + const nextAuthType = + body.authType !== undefined ? body.authType : existing.authType; - if (providerType === "custom") { - const nextUpstreamUrl = - body.upstreamUrl !== undefined - ? body.upstreamUrl - : existing.upstreamUrl; - const nextAuthType = - body.authType !== undefined ? body.authType : existing.authType; - + if ( + providerRequiresUpstreamUrl(providerType) || + body.upstreamUrl !== undefined || + body.authType !== undefined + ) { const validation = z .object({ - type: z.literal("custom"), + type: aiProviderTypeSchema, upstreamUrl: z.string().nullable().optional(), authType: aiAuthTypeSchema.nullable().optional() }) .superRefine((data, ctx) => - refineCustomProviderFields(data, ctx) + refineProviderUpstreamFields(data, ctx) ) .safeParse({ - type: "custom", + type: providerType, upstreamUrl: nextUpstreamUrl, authType: nextAuthType }); diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts index 62b442c59..ab70800b5 100644 --- a/server/routers/aiProvider/validation.ts +++ b/server/routers/aiProvider/validation.ts @@ -1,13 +1,19 @@ import { z } from "zod"; -import type { - AiBudgetUnit, - AiProviderType +import { + providerRequiresUpstreamUrl, + type AiBudgetUnit, + type AiProviderType } from "@server/lib/aiProviderDefaults"; export const aiProviderTypeSchema = z.enum([ "openai", "anthropic", + "googleGemini", + "vertexAi", "bedrock", + "microsoftFoundry", + "openRouter", + "vercelAiGateway", "custom" ]); @@ -36,7 +42,7 @@ export function refineBudgetFields( } } -export function refineCustomProviderFields( +export function refineProviderUpstreamFields( data: { type: AiProviderType; upstreamUrl?: string | null; @@ -44,19 +50,15 @@ export function refineCustomProviderFields( }, ctx: z.RefinementCtx ) { - if (data.type !== "custom") { - return; - } - - if (!data.upstreamUrl) { + if (providerRequiresUpstreamUrl(data.type) && !data.upstreamUrl) { ctx.addIssue({ code: "custom", - message: "upstreamUrl is required for custom providers", + message: `upstreamUrl is required for ${data.type} providers`, path: ["upstreamUrl"] }); } - if (!data.authType) { + if (data.type === "custom" && !data.authType) { ctx.addIssue({ code: "custom", message: "authType is required for custom providers", From 9d581f3897c3491790f81f3f9846f5af59102b6b Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 31 Jul 2026 16:50:35 -0400 Subject: [PATCH 014/296] Fix types --- messages/en-US.json | 1 + server/db/pg/schema/schema.ts | 4 ++- server/db/sqlite/schema/schema.ts | 2 +- .../private/lib/traefik/getTraefikConfig.ts | 3 +- .../routers/launcher/formatLauncherAccess.ts | 3 +- server/routers/resource/createResource.ts | 4 +-- server/routers/resource/listResources.ts | 3 +- .../settings/resources/private/page.tsx | 4 +-- .../settings/resources/public/create/page.tsx | 2 +- src/components/PrivateResourcesTable.tsx | 36 +++++++++---------- src/components/SiteResourceInfoBox.tsx | 3 +- src/components/SiteResourcesOverview.tsx | 3 +- src/lib/formatSiteResourceAccess.ts | 4 ++- src/lib/privateResourceForm.ts | 7 ++-- 14 files changed, 44 insertions(+), 35 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 24d10ab95..fd28a5469 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -2411,6 +2411,7 @@ "editInternalResourceDialogModeCidr": "CIDR", "editInternalResourceDialogModeHttp": "HTTP", "editInternalResourceDialogModeHttps": "HTTPS", + "editInternalResourceDialogModeInference": "Inference", "editInternalResourceDialogModeSsh": "SSH", "editInternalResourceDialogScheme": "Scheme", "editInternalResourceDialogEnableSsl": "Enable TLS", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 2f8199aa8..43a84ab5d 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -197,7 +197,9 @@ export const resources = pgTable( wildcard: boolean("wildcard").notNull().default(false), mode: text("mode") .default("http") - .$type<"rdp" | "ssh" | "http" | "vnc" | "inference">() + .$type< + "rdp" | "ssh" | "http" | "vnc" | "inference" | "tcp" | "udp" + >() .notNull(), pamMode: varchar("pamMode", { length: 32 }) .$type<"passthrough" | "push">() diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 4cc533475..8265a2855 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -206,7 +206,7 @@ export const resources = sqliteTable("resources", { wildcard: integer("wildcard", { mode: "boolean" }).notNull().default(false), mode: text("mode") .default("http") - .$type<"rdp" | "ssh" | "http" | "vnc" | "inference">() + .$type<"rdp" | "ssh" | "http" | "vnc" | "inference" | "tcp" | "udp">() .notNull(), // rdp, ssh, http, vnc, inference pamMode: text("pamMode") .$type<"passthrough" | "push">() diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 4be4c6d2b..0544cafe7 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -18,6 +18,7 @@ import { domains, exitNodes, loginPage, + SiteResource, targetHealthCheck } from "@server/db"; import { @@ -361,7 +362,7 @@ export async function getTraefikConfig( let siteResourcesWithFullDomain: { siteResourceId: number; fullDomain: string | null; - mode: "http" | "host" | "cidr" | "ssh"; + mode: SiteResource["mode"]; }[] = []; if ( build == "enterprise" && diff --git a/server/routers/launcher/formatLauncherAccess.ts b/server/routers/launcher/formatLauncherAccess.ts index c12767ce1..ff5a5bf37 100644 --- a/server/routers/launcher/formatLauncherAccess.ts +++ b/server/routers/launcher/formatLauncherAccess.ts @@ -1,7 +1,8 @@ +import { SiteResource } from "@server/db"; import { formatEndpoint, parseEndpoint } from "@server/lib/ip"; export type SiteResourceDestinationInput = { - mode: "host" | "cidr" | "http" | "ssh"; + mode: SiteResource["mode"]; destination: string | null; destinationPort: number | null; scheme: "http" | "https" | null; diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index ab758cc1c..2ff7b9ae3 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -44,11 +44,11 @@ const createResourceParamsSchema = z.strictObject({ }); function resolveModeFromLegacyFields(data: { - mode?: "http" | "ssh" | "rdp" | "vnc" | "tcp" | "udp"; + mode?: "http" | "ssh" | "rdp" | "vnc" | "tcp" | "udp" | "inference"; http?: boolean; protocol?: "tcp" | "udp"; }): { - mode?: "http" | "ssh" | "rdp" | "vnc" | "tcp" | "udp"; + mode?: "http" | "ssh" | "rdp" | "vnc" | "tcp" | "udp" | "inference"; error?: string; } { if (data.mode) { diff --git a/server/routers/resource/listResources.ts b/server/routers/resource/listResources.ts index cc075f23b..e5408f630 100644 --- a/server/routers/resource/listResources.ts +++ b/server/routers/resource/listResources.ts @@ -637,11 +637,12 @@ export async function listResources( ${resourcePassword.passwordId} ) `; - const browserGatewayModes = ["http", "ssh", "rdp", "vnc"]; + const browserGatewayModes = ["http", "ssh", "rdp", "vnc"] as const; switch (authState) { case "none": conditions.push( + // TODO: Does inference belong here? or(eq(resources.mode, "tcp"), eq(resources.mode, "udp")) ); break; diff --git a/src/app/[orgId]/settings/resources/private/page.tsx b/src/app/[orgId]/settings/resources/private/page.tsx index cbd9ec7ef..86bba120d 100644 --- a/src/app/[orgId]/settings/resources/private/page.tsx +++ b/src/app/[orgId]/settings/resources/private/page.tsx @@ -1,5 +1,5 @@ import PrivateResourcesBanner from "@app/components/PrivateResourcesBanner"; -import type { InternalResourceRow } from "@app/components/PrivateResourcesTable"; +import type { PrivateResourceRow } from "@app/components/PrivateResourcesTable"; import PrivateResourcesTable from "@app/components/PrivateResourcesTable"; import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; import { internal } from "@app/lib/api"; @@ -61,7 +61,7 @@ export default async function ClientResourcesPage( redirect(`/${params.orgId}/settings/resources`); } - const internalResourceRows: InternalResourceRow[] = siteResources.map( + const internalResourceRows: PrivateResourceRow[] = siteResources.map( (siteResource) => { return { id: siteResource.siteResourceId, diff --git a/src/app/[orgId]/settings/resources/public/create/page.tsx b/src/app/[orgId]/settings/resources/public/create/page.tsx index 43d9528b9..eaf4e5527 100644 --- a/src/app/[orgId]/settings/resources/public/create/page.tsx +++ b/src/app/[orgId]/settings/resources/public/create/page.tsx @@ -206,7 +206,7 @@ function createAddTargetSchema(t: TranslateFn) { ); } -type NewResourceType = "http" | "ssh" | "rdp" | "vnc" | "tcp" | "udp"; +type NewResourceType = "http" | "ssh" | "rdp" | "vnc" | "tcp" | "udp" | "inference"; type CreateBgTargetFormValues = SshSettingsFormValues; diff --git a/src/components/PrivateResourcesTable.tsx b/src/components/PrivateResourcesTable.tsx index 5ee23f424..87eddb921 100644 --- a/src/components/PrivateResourcesTable.tsx +++ b/src/components/PrivateResourcesTable.tsx @@ -69,20 +69,21 @@ import { LabelColumnFilterButton } from "./LabelColumnFilterButton"; import { LabelsTableCell } from "./LabelsTableCell"; import { ControlledDataTable } from "./ui/controlled-data-table"; import { SitesColumnFilterButton } from "./SitesColumnFilterButton"; +import { SiteResource } from "@server/db"; -export type InternalResourceSiteRow = ResourceSiteRow; +export type PrivateResourceSiteRow = ResourceSiteRow; -export type InternalResourceRow = { +export type PrivateResourceRow = { id: number; name: string; orgId: string; - sites: InternalResourceSiteRow[]; + sites: PrivateResourceSiteRow[]; siteNames: string[]; siteAddresses: (string | null)[]; siteIds: number[]; siteNiceIds: string[]; // mode: "host" | "cidr" | "port"; - mode: "host" | "cidr" | "http" | "ssh"; + mode: SiteResource["mode"]; scheme: "http" | "https" | null; ssl: boolean; // protocol: string | null; @@ -109,7 +110,7 @@ export type InternalResourceRow = { }>; }; -function formatDestinationDisplay(row: InternalResourceRow): string { +function formatDestinationDisplay(row: PrivateResourceRow): string { return formatSiteResourceDestinationDisplay({ mode: row.mode, destination: row.destination, @@ -133,7 +134,7 @@ const booleanSearchFilterSchema = z .catch(undefined); type ClientResourcesTableProps = { - internalResources: InternalResourceRow[]; + internalResources: PrivateResourceRow[]; orgId: string; pagination: PaginationState; rowCount: number; @@ -164,10 +165,10 @@ export default function PrivateResourcesTable({ const [isNavigatingToAddPage, startNavigation] = useTransition(); const [selectedInternalResource, setSelectedInternalResource] = - useState(null); + useState(null); const [isEditDialogOpen, setIsEditDialogOpen] = useState(false); const [editingResource, setEditingResource] = - useState(); + useState(); const [isCreateDialogOpen, setIsCreateDialogOpen] = useState(false); const [isRefreshing, startRefreshTransition] = useTransition(); @@ -241,9 +242,9 @@ export default function PrivateResourcesTable({ }; const internalColumns = useMemo< - ExtendedColumnDef[] + ExtendedColumnDef[] >(() => { - const cols: ExtendedColumnDef[] = [ + const cols: ExtendedColumnDef[] = [ { accessorKey: "name", enableHiding: false, @@ -364,15 +365,12 @@ export default function PrivateResourcesTable({ ), cell: ({ row }) => { const resourceRow = row.original; - const modeLabels: Record< - "host" | "cidr" | "port" | "http" | "ssh", - string - > = { + const modeLabels: Record = { host: t("editInternalResourceDialogModeHost"), cidr: t("editInternalResourceDialogModeCidr"), - port: t("editInternalResourceDialogModePort"), http: t("editInternalResourceDialogModeHttp"), - ssh: t("editInternalResourceDialogModeSsh") + ssh: t("editInternalResourceDialogModeSsh"), + inference: t("editInternalResourceDialogModeInference") }; return {modeLabels[resourceRow.mode]}; } @@ -521,7 +519,7 @@ export default function PrivateResourcesTable({ className="p-3" /> ), - cell: ({ row }: { row: { original: InternalResourceRow } }) => ( + cell: ({ row }: { row: { original: PrivateResourceRow } }) => ( Date: Fri, 31 Jul 2026 17:22:08 -0400 Subject: [PATCH 015/296] Handle the alias create and update special case exit nodes --- server/lib/rebuildClientAssociations.ts | 79 ++++++++++++++++++- .../siteResource/createSiteResource.ts | 9 ++- .../siteResource/updateSiteResource.ts | 9 ++- 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/server/lib/rebuildClientAssociations.ts b/server/lib/rebuildClientAssociations.ts index ce0897aab..194bfe446 100644 --- a/server/lib/rebuildClientAssociations.ts +++ b/server/lib/rebuildClientAssociations.ts @@ -1137,7 +1137,9 @@ async function syncClientExitNodeConnections( ) ); - const needsConnectSet = new Set(requiresExitNodeRows.map((r) => r.clientId)); + const needsConnectSet = new Set( + requiresExitNodeRows.map((r) => r.clientId) + ); const exitNodeIds = Array.from( new Set( @@ -1249,6 +1251,61 @@ async function syncClientExitNodeConnections( } } +// Notifies the olms of every given client that the alias of the site resource +// they're using an exit node connection for has changed, via the dedicated +// exit node data-update message. Unlike syncClientExitNodeConnections, this +// doesn't touch connect/disconnect state - it's purely a rename for clients +// that are (and remain) connected to the exit node for this resource. +async function syncClientExitNodeAliasUpdate( + clientIds: number[], + oldAlias: string | null, + newAlias: string | null, + trx: Transaction | typeof db = db +): Promise { + const uniqueClientIds = Array.from(new Set(clientIds)); + if (uniqueClientIds.length === 0) { + return; + } + + const oldAliases = oldAlias ? [oldAlias] : []; + const newAliases = newAlias ? [newAlias] : []; + if (oldAliases.length === 0 && newAliases.length === 0) { + return; + } + + const olmRows = await trx + .select({ + clientId: olms.clientId, + olmId: olms.olmId, + version: olms.version + }) + .from(olms) + .where(inArray(olms.clientId, uniqueClientIds)); + + const updatePayloads = olmRows + .filter((r) => r.clientId !== null) + .map((olm) => ({ + clientId: olm.olmId, + message: { + type: "olm/wg/exitnode/data/update", + data: { + oldAliases, + newAliases + } + }, + options: { compress: canCompress(olm.version, "olm") } + })); + + if (updatePayloads.length > 0) { + await sendToClientsBatch(updatePayloads).catch((error) => { + logger.error( + `rebuildClientAssociations: Error sending exit node alias update messages:`, + error + ); + }); + } +} + async function handleSubnetProxyTargetUpdates( siteResource: SiteResource, sitesList: Site[], @@ -1479,7 +1536,7 @@ export async function handleMessagingForUpdatedSiteResource( `handleMessagingForUpdatedSiteResource: fetched newts for ${newtsForSites.length}/${allSiteIds.length} site(s)` ); - // WARNING: THIS RELIES ON THE CACHE TABLES BEING UP TO DATE, SO CALL THIS AFTER THE ASSOCIATION CACHE IS UPDATED + // !!!!!!!!!!!!!!!!!! WARNING: THIS RELIES ON THE CACHE TABLES BEING UP TO DATE, SO CALL THIS AFTER THE ASSOCIATION CACHE IS UPDATED !!!!!!!!!!!!!!!!!! const mergedAllClients = await trx .select({ clientId: clientSiteResourcesAssociationsCache.clientId, @@ -1906,6 +1963,24 @@ export async function handleMessagingForUpdatedSiteResource( ); } + // For a resource that stays on an exit node connection across the update, + // the alias is the only field that affects already-connected clients (the + // exit node itself, its endpoint, etc. are not per-resource). Tell those + // clients' olms about the rename directly via the exit node data-update + // message rather than a full connect/disconnect cycle. + if ( + existingSiteResource?.requiresExitNodeConnection && + updatedSiteResource.requiresExitNodeConnection && + aliasChanged + ) { + await syncClientExitNodeAliasUpdate( + mergedAllClients.map((c) => c.clientId), + existingSiteResource.alias, + updatedSiteResource.alias, + trx + ); + } + // If this resource requires (or required) clients to be connected to the // exit node (e.g. an inference resource), re-sync connect/disconnect // state for every client currently associated with it - covers toggling diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index d220a4b08..1a4cd9a4d 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -29,7 +29,7 @@ import response from "@server/lib/response"; import logger from "@server/logger"; import { OpenAPITags, registry } from "@server/openApi"; import HttpCode from "@server/types/HttpCode"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, ne } from "drizzle-orm"; import { NextFunction, Request, Response } from "express"; import createHttpError from "http-errors"; import { z } from "zod"; @@ -490,7 +490,11 @@ export async function createSiteResource( .where( and( eq(siteResources.orgId, orgId), - eq(siteResources.alias, alias.trim()) + eq(siteResources.alias, alias.trim()), + ne( + siteResources.requiresExitNodeConnection, + mode == "inference" + ) // exclude looking at the ones on exit nodes if this is an inference resource ) ) .limit(1); @@ -527,6 +531,7 @@ export async function createSiteResource( let aliasAddress: string | null = null; let releaseAliasLock: (() => Promise) | null = null; if (mode === "host" || mode === "http" || mode === "ssh") { + // no alias address but we do have an alias for inference const { value, release } = await getNextAvailableAliasAddress(orgId); aliasAddress = value; diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index 1b79e7b16..0ab928d04 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -507,7 +507,11 @@ export async function updateSiteResource( and( eq(siteResources.orgId, existingSiteResource.orgId), eq(siteResources.alias, alias.trim()), - ne(siteResources.siteResourceId, siteResourceId) // exclude self + ne(siteResources.siteResourceId, siteResourceId), // exclude self + ne( + siteResources.requiresExitNodeConnection, + mode == "inference" + ) // exclude looking at the ones on exit nodes if this is an inference resource ) ) .limit(1); @@ -587,8 +591,7 @@ export async function updateSiteResource( domainId, subdomain: finalSubdomain, fullDomain, - networkId: - mode === "inference" ? null : undefined, + networkId: mode === "inference" ? null : undefined, requiresExitNodeConnection: mode !== undefined ? mode === "inference" : undefined, ...sshPamSet From 42c0abedb7195bffad45cdfce909e433c8240143 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 31 Jul 2026 17:47:45 -0400 Subject: [PATCH 016/296] Add basic page to configure the alias --- messages/en-US.json | 1 + .../private/[niceId]/inference/page.tsx | 107 ++++++++++++++++++ .../resources/private/[niceId]/layout.tsx | 3 +- .../PrivateResourceDestinationFields.tsx | 31 +++++ src/lib/privateResourceForm.ts | 10 ++ 5 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx diff --git a/messages/en-US.json b/messages/en-US.json index fd28a5469..94dd49f96 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -2256,6 +2256,7 @@ "requireDeviceApproval": "Require Device Approvals", "requireDeviceApprovalDescription": "Users with this role need new devices approved by an admin before they can connect and access resources.", "sshSettings": "SSH Settings", + "inferenceSettings": "Inference Settings", "sshAccess": "SSH Access", "rdpSettings": "RDP Settings", "vncSettings": "VNC Settings", diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx new file mode 100644 index 000000000..3f27f1172 --- /dev/null +++ b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx @@ -0,0 +1,107 @@ +"use client"; + +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { Button } from "@app/components/ui/button"; +import { Form } from "@app/components/ui/form"; +import { createInferenceFormSchema } from "@app/lib/privateResourceForm"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useTranslations } from "next-intl"; +import { useActionState, useMemo, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; +import { PrivateResourceSitesField } from "@app/components/PrivateResourceSitesField"; +import { PrivateResourceInferenceDestinationFields } from "@app/components/PrivateResourceDestinationFields"; +import { PrivateResourcePortRanges } from "@app/components/PrivateResourcePortRanges"; +import { useSaveSiteResource } from "@app/hooks/useSaveSiteResource"; +import { + asAnyControl, + asAnySetValue, + asAnyWatch +} from "@app/lib/formControlUtils"; +import { buildSelectedSitesForResource } from "@app/lib/privateResourceUtils"; + +export default function PrivateResourceInferencePage() { + const t = useTranslations(); + const { save, siteResource } = useSaveSiteResource(); + const [selectedSites, setSelectedSites] = useState(() => + buildSelectedSitesForResource(siteResource) + ); + + const formSchema = useMemo(() => createInferenceFormSchema(t), [t]); + type FormValues = z.infer; + + const form = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + mode: "inference", + alias: siteResource.alias ?? null + } + }); + + const [, formAction, saveLoading] = useActionState(async () => { + const isValid = await form.trigger(); + if (!isValid) return; + + const data = form.getValues(); + await save({ + mode: "inference", + alias: data.alias + }); + }, null); + + return ( + + + + + {t("hostSettings")} + + + {t("editInternalResourceDialogDestinationDescription")} + + + + + +
+ + + + + + +
+ + + + + + + + + + ); +} diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx index a1e97e188..f07b92673 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx @@ -52,7 +52,8 @@ export default async function PrivateResourceLayout( | "hostSettings" | "cidrSettings" | "httpSettings" - | "sshSettings"; + | "sshSettings" + | "inferenceSettings"; const navItems = [ { diff --git a/src/components/PrivateResourceDestinationFields.tsx b/src/components/PrivateResourceDestinationFields.tsx index 642ee0a5b..a10ce3161 100644 --- a/src/components/PrivateResourceDestinationFields.tsx +++ b/src/components/PrivateResourceDestinationFields.tsx @@ -76,6 +76,37 @@ type PrivateResourceHostDestinationFieldsProps = { hideAlias?: boolean; }; +type PrivateResourceInferenceDestinationFieldsProps = { + control: Control; + watch: UseFormWatch; + labelPrefix?: "create" | "edit"; + hideAlias?: boolean; +}; + +export function PrivateResourceInferenceDestinationFields({ + control, + watch, + labelPrefix = "edit" +}: PrivateResourceInferenceDestinationFieldsProps) { + const t = useTranslations(); + const destinationLabelKey = + labelPrefix === "create" + ? "createInternalResourceDialogDestination" + : "editInternalResourceDialogDestination"; + + return ( + + + + + + ); +} + export function PrivateResourceHostDestinationFields({ control, watch, diff --git a/src/lib/privateResourceForm.ts b/src/lib/privateResourceForm.ts index 844ff96cb..f2ff103f6 100644 --- a/src/lib/privateResourceForm.ts +++ b/src/lib/privateResourceForm.ts @@ -485,6 +485,7 @@ function destinationRefine( const isNativeSsh = data.mode === "ssh" && data.authDaemonMode === "native"; const trimmedDestination = data.destination?.trim(); if ( + data.mode !== "inference" && !isNativeSsh && (!trimmedDestination || trimmedDestination.length < 1) ) { @@ -555,6 +556,15 @@ export function createHostFormSchema(t: TranslateFn) { .superRefine((data, ctx) => destinationRefine(data, ctx, t)); } +export function createInferenceFormSchema(t: TranslateFn) { + return z + .object({ + mode: z.literal("inference"), + alias: z.string().nullish() + }) + .superRefine((data, ctx) => destinationRefine(data, ctx, t)); +} + export function createCidrFormSchema(t: TranslateFn) { return z .object({ From 6fa0009ebf3d3699a9ab15366f95025765fbe296 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 31 Jul 2026 17:36:04 -0400 Subject: [PATCH 017/296] add targets and refactor endpoints --- server/db/pg/schema/schema.ts | 20 +- server/db/sqlite/schema/schema.ts | 15 +- server/lib/aiProviderDefaults.ts | 27 ++- server/lib/alerts/events/healthCheckEvents.ts | 8 +- server/lib/deleteResource.ts | 9 +- server/lib/deleteSiteAssociatedResources.ts | 8 +- .../integration/verifyApiKeyAiModelAccess.ts | 27 +-- .../verifyApiKeyAiProviderAccess.ts | 48 +---- .../integration/verifyApiKeyTargetAccess.ts | 80 +++++--- server/middlewares/verifyAiModelAccess.ts | 28 +-- server/middlewares/verifyAiProviderAccess.ts | 19 +- server/middlewares/verifyTargetAccess.ts | 127 +++++++----- server/routers/aiProvider/createAiModel.ts | 12 +- server/routers/aiProvider/createAiProvider.ts | 11 +- server/routers/aiProvider/deleteAiModel.ts | 31 +-- server/routers/aiProvider/deleteAiProvider.ts | 21 +- server/routers/aiProvider/getAiModel.ts | 27 +-- server/routers/aiProvider/getAiProvider.ts | 14 +- server/routers/aiProvider/listAiModels.ts | 12 +- server/routers/aiProvider/types.ts | 4 +- server/routers/aiProvider/updateAiModel.ts | 34 +-- server/routers/aiProvider/updateAiProvider.ts | 89 ++++---- server/routers/aiProvider/validation.ts | 21 +- server/routers/external.ts | 41 ++-- server/routers/integration.ts | 40 ++-- server/routers/newt/buildConfiguration.ts | 16 +- server/routers/target/createTarget.ts | 194 +++++++++++++++--- server/routers/target/deleteTarget.ts | 80 +++++--- server/routers/target/listTargets.ts | 61 +++++- server/routers/target/updateTarget.ts | 106 ++++++++-- .../public/ProxyResourceTargetsForm.tsx | 1 + 31 files changed, 741 insertions(+), 490 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 43a84ab5d..0f59b1c26 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -323,11 +323,18 @@ export const targets = pgTable( "targets", { targetId: serial("targetId").primaryKey(), - resourceId: integer("resourceId") - .references(() => resources.resourceId, { + resourceId: integer("resourceId").references( + () => resources.resourceId, + { onDelete: "cascade" - }) - .notNull(), + } + ), + providerId: integer("providerId").references( + () => aiProviders.providerId, + { + onDelete: "cascade" + } + ), siteId: integer("siteId") .references(() => sites.siteId, { onDelete: "cascade" @@ -351,6 +358,7 @@ export const targets = pgTable( }, (t) => [ index("idx_targets_resourceid_siteid").on(t.resourceId, t.siteId), + index("idx_targets_providerid_siteid").on(t.providerId, t.siteId), index("idx_targets_site_enabled_priority_target_resource") .on(t.siteId, t.priority.desc(), t.targetId, t.resourceId) .where(sql`${t.enabled} = true`) @@ -1572,6 +1580,10 @@ export const aiProviders = pgTable("aiProviders", { apiKey: text("apiKey"), apiKeyLastChars: varchar("apiKeyLastChars"), authType: varchar("authType").$type<"bearer">(), + routingMode: varchar("routingMode") + .$type<"url" | "target">() + .notNull() + .default("url"), skipTlsVerification: boolean("skipTlsVerification") .notNull() .default(false), diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 8265a2855..73e73f04a 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -326,11 +326,12 @@ export const clientLabels = sqliteTable( export const targets = sqliteTable("targets", { targetId: integer("targetId").primaryKey({ autoIncrement: true }), - resourceId: integer("resourceId") - .references(() => resources.resourceId, { - onDelete: "cascade" - }) - .notNull(), + resourceId: integer("resourceId").references(() => resources.resourceId, { + onDelete: "cascade" + }), + providerId: integer("providerId").references(() => aiProviders.providerId, { + onDelete: "cascade" + }), siteId: integer("siteId") .references(() => sites.siteId, { onDelete: "cascade" @@ -1561,6 +1562,10 @@ export const aiProviders = sqliteTable("aiProviders", { apiKey: text("apiKey"), apiKeyLastChars: text("apiKeyLastChars"), authType: text("authType").$type<"bearer">(), + routingMode: text("routingMode") + .$type<"url" | "target">() + .notNull() + .default("url"), skipTlsVerification: integer("skipTlsVerification", { mode: "boolean" }) .notNull() .default(false), diff --git a/server/lib/aiProviderDefaults.ts b/server/lib/aiProviderDefaults.ts index 63c060b62..6cfb816b7 100644 --- a/server/lib/aiProviderDefaults.ts +++ b/server/lib/aiProviderDefaults.ts @@ -11,6 +11,7 @@ export type AiProviderType = export type AiProviderAuthType = "bearer"; export type AiBudgetUnit = "usd" | "tokens"; +export type AiProviderRoutingMode = "url" | "target"; type AiProviderDefaults = { upstreamUrl: string | null; @@ -55,7 +56,13 @@ export const AI_PROVIDER_DEFAULTS: Record< } }; -export function providerRequiresUpstreamUrl(type: AiProviderType): boolean { +export function providerRequiresUpstreamUrl( + type: AiProviderType, + routingMode: AiProviderRoutingMode = "url" +): boolean { + if (routingMode === "target") { + return false; + } if (type === "custom") { return true; } @@ -66,20 +73,34 @@ export function resolveAiProviderConfig(input: { type: AiProviderType; upstreamUrl: string | null; authType: AiProviderAuthType | null; + routingMode?: AiProviderRoutingMode | null; }): { upstreamUrl: string | null; authType: AiProviderAuthType | null; + routingMode: AiProviderRoutingMode; } { + const routingMode = input.routingMode ?? "url"; + + if (routingMode === "target") { + return { + upstreamUrl: null, + authType: input.authType ?? "bearer", + routingMode + }; + } + if (input.type === "custom") { return { upstreamUrl: input.upstreamUrl, - authType: input.authType + authType: input.authType, + routingMode }; } const defaults = AI_PROVIDER_DEFAULTS[input.type]; return { upstreamUrl: input.upstreamUrl ?? defaults.upstreamUrl, - authType: input.authType ?? defaults.authType + authType: input.authType ?? defaults.authType, + routingMode }; } diff --git a/server/lib/alerts/events/healthCheckEvents.ts b/server/lib/alerts/events/healthCheckEvents.ts index 429cf73b0..b7b5a8ff2 100644 --- a/server/lib/alerts/events/healthCheckEvents.ts +++ b/server/lib/alerts/events/healthCheckEvents.ts @@ -202,6 +202,10 @@ async function handleResource( return; } + if (!target.resourceId) { + return; + } + const [resource] = await trx .select() .from(resources) @@ -227,9 +231,7 @@ async function handleResource( let health = "healthy"; const allUnknown = monitoredTargets.length === 0; - const allHealthy = monitoredTargets.every( - (t) => t.hcHealth === "healthy" - ); + const allHealthy = monitoredTargets.every((t) => t.hcHealth === "healthy"); const allUnhealthy = monitoredTargets.every( (t) => t.hcHealth === "unhealthy" ); diff --git a/server/lib/deleteResource.ts b/server/lib/deleteResource.ts index 3f33c7400..b71818b17 100644 --- a/server/lib/deleteResource.ts +++ b/server/lib/deleteResource.ts @@ -64,13 +64,20 @@ export async function performDeleteResources( const targetsByResourceId = new Map(); for (const target of targetsToBeRemoved) { + if (target.resourceId == null) { + continue; + } const existing = targetsByResourceId.get(target.resourceId) ?? []; existing.push(target); targetsByResourceId.set(target.resourceId, existing); } const targetIdToResourceId = new Map( - targetsToBeRemoved.map((target) => [target.targetId, target.resourceId]) + targetsToBeRemoved.flatMap((target) => + target.resourceId == null + ? [] + : [[target.targetId, target.resourceId] as const] + ) ); const healthChecksByResourceId = new Map(); diff --git a/server/lib/deleteSiteAssociatedResources.ts b/server/lib/deleteSiteAssociatedResources.ts index e69585d1a..c0e520846 100644 --- a/server/lib/deleteSiteAssociatedResources.ts +++ b/server/lib/deleteSiteAssociatedResources.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, sql } from "drizzle-orm"; +import { and, eq, inArray, isNotNull, sql } from "drizzle-orm"; import { db, resources, @@ -33,9 +33,11 @@ export async function getResourceIdsForSite( const rows = await trx .selectDistinct({ resourceId: targets.resourceId }) .from(targets) - .where(eq(targets.siteId, siteId)); + .where(and(eq(targets.siteId, siteId), isNotNull(targets.resourceId))); - return rows.map((row) => row.resourceId); + return rows + .map((row) => row.resourceId) + .filter((resourceId): resourceId is number => resourceId != null); } export async function getSiteResourceIdsForSite( diff --git a/server/middlewares/integration/verifyApiKeyAiModelAccess.ts b/server/middlewares/integration/verifyApiKeyAiModelAccess.ts index 75a56f37e..eec772ac3 100644 --- a/server/middlewares/integration/verifyApiKeyAiModelAccess.ts +++ b/server/middlewares/integration/verifyApiKeyAiModelAccess.ts @@ -14,9 +14,6 @@ export async function verifyApiKeyAiModelAccess( const apiKey = req.apiKey; const modelIdRaw = getFirstString(req.params.modelId); const modelId = Number.parseInt(modelIdRaw ?? "", 10); - const providerIdRaw = getFirstString(req.params.providerId); - const providerId = Number.parseInt(providerIdRaw ?? "", 10); - const orgId = getFirstString(req.params.orgId); if (!apiKey) { return next( @@ -24,18 +21,6 @@ export async function verifyApiKeyAiModelAccess( ); } - if (!orgId) { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") - ); - } - - if (Number.isNaN(providerId)) { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") - ); - } - if (Number.isNaN(modelId)) { return next( createHttpError(HttpCode.BAD_REQUEST, "Invalid model ID") @@ -52,13 +37,7 @@ export async function verifyApiKeyAiModelAccess( aiProviders, eq(aiModels.providerId, aiProviders.providerId) ) - .where( - and( - eq(aiModels.modelId, modelId), - eq(aiModels.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiModels.modelId, modelId)) .limit(1); if (!row) { @@ -76,7 +55,9 @@ export async function verifyApiKeyAiModelAccess( return next(); } - if (!req.apiKeyOrg) { + const orgId = row.provider.orgId; + + if (!req.apiKeyOrg || req.apiKeyOrg.orgId !== orgId) { const apiKeyOrgRes = await db .select() .from(apiKeyOrg) diff --git a/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts b/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts index 6b9f9b0ec..1668b5727 100644 --- a/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts +++ b/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts @@ -14,7 +14,6 @@ export async function verifyApiKeyAiProviderAccess( const apiKey = req.apiKey; const providerIdRaw = getFirstString(req.params.providerId); const providerId = Number.parseInt(providerIdRaw ?? "", 10); - const orgId = getFirstString(req.params.orgId); if (!apiKey) { return next( @@ -22,52 +21,16 @@ export async function verifyApiKeyAiProviderAccess( ); } - if (!orgId) { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") - ); - } - if (Number.isNaN(providerId)) { return next( createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") ); } - if (apiKey.isRoot) { - const [provider] = await db - .select() - .from(aiProviders) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) - .limit(1); - - if (!provider) { - return next( - createHttpError( - HttpCode.NOT_FOUND, - `AI provider with ID ${providerId} not found` - ) - ); - } - - req.aiProvider = provider; - return next(); - } - const [provider] = await db .select() .from(aiProviders) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiProviders.providerId, providerId)) .limit(1); if (!provider) { @@ -79,7 +42,14 @@ export async function verifyApiKeyAiProviderAccess( ); } - if (!req.apiKeyOrg) { + if (apiKey.isRoot) { + req.aiProvider = provider; + return next(); + } + + const orgId = provider.orgId; + + if (!req.apiKeyOrg || req.apiKeyOrg.orgId !== orgId) { const apiKeyOrgRes = await db .select() .from(apiKeyOrg) diff --git a/server/middlewares/integration/verifyApiKeyTargetAccess.ts b/server/middlewares/integration/verifyApiKeyTargetAccess.ts index dac36b1e2..98d5349df 100644 --- a/server/middlewares/integration/verifyApiKeyTargetAccess.ts +++ b/server/middlewares/integration/verifyApiKeyTargetAccess.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { db } from "@server/db"; -import { resources, targets, apiKeyOrg } from "@server/db"; +import { aiProviders, resources, targets, apiKeyOrg } from "@server/db"; import { and, eq } from "drizzle-orm"; import createHttpError from "http-errors"; import HttpCode from "@server/types/HttpCode"; @@ -43,43 +43,65 @@ export async function verifyApiKeyTargetAccess( ); } - const resourceId = target.resourceId; - if (!resourceId) { + const { resourceId, providerId } = target; + if ((!resourceId && !providerId) || (resourceId && providerId)) { return next( createHttpError( HttpCode.INTERNAL_SERVER_ERROR, - `Target with ID ${targetId} does not have a resource ID` - ) - ); - } - - const [resource] = await db - .select() - .from(resources) - .where(eq(resources.resourceId, resourceId)) - .limit(1); - - if (!resource) { - return next( - createHttpError( - HttpCode.NOT_FOUND, - `Resource with ID ${resourceId} not found` + `Target with ID ${targetId} has invalid ownership` ) ); } if (apiKey.isRoot) { - // Root keys can access any key in any org + // Root keys can access any target return next(); } - if (!resource.orgId) { - return next( - createHttpError( - HttpCode.INTERNAL_SERVER_ERROR, - `Resource with ID ${resourceId} does not have an organization ID` - ) - ); + let orgId: string; + if (resourceId) { + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Resource with ID ${resourceId} not found` + ) + ); + } + + if (!resource.orgId) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + `Resource with ID ${resourceId} does not have an organization ID` + ) + ); + } + + orgId = resource.orgId; + } else { + const [provider] = await db + .select() + .from(aiProviders) + .where(eq(aiProviders.providerId, providerId!)) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + orgId = provider.orgId; } if (!req.apiKeyOrg) { @@ -89,7 +111,7 @@ export async function verifyApiKeyTargetAccess( .where( and( eq(apiKeyOrg.apiKeyId, apiKey.apiKeyId), - eq(apiKeyOrg.orgId, resource.orgId) + eq(apiKeyOrg.orgId, orgId) ) ) .limit(1); @@ -98,7 +120,7 @@ export async function verifyApiKeyTargetAccess( } } - if (!req.apiKeyOrg) { + if (!req.apiKeyOrg || req.apiKeyOrg.orgId !== orgId) { return next( createHttpError( HttpCode.FORBIDDEN, diff --git a/server/middlewares/verifyAiModelAccess.ts b/server/middlewares/verifyAiModelAccess.ts index 99693b982..0d5761fc4 100644 --- a/server/middlewares/verifyAiModelAccess.ts +++ b/server/middlewares/verifyAiModelAccess.ts @@ -16,9 +16,6 @@ export async function verifyAiModelAccess( const userId = req.user!.userId; const modelIdRaw = getFirstString(req.params.modelId); const modelId = Number.parseInt(modelIdRaw ?? "", 10); - const providerIdRaw = getFirstString(req.params.providerId); - const providerId = Number.parseInt(providerIdRaw ?? "", 10); - const orgId = getFirstString(req.params.orgId); if (!userId) { return next( @@ -26,18 +23,6 @@ export async function verifyAiModelAccess( ); } - if (!orgId) { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") - ); - } - - if (Number.isNaN(providerId)) { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") - ); - } - if (Number.isNaN(modelId)) { return next( createHttpError(HttpCode.BAD_REQUEST, "Invalid model ID") @@ -54,13 +39,7 @@ export async function verifyAiModelAccess( aiProviders, eq(aiModels.providerId, aiProviders.providerId) ) - .where( - and( - eq(aiModels.modelId, modelId), - eq(aiModels.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiModels.modelId, modelId)) .limit(1); if (!row) { @@ -72,7 +51,9 @@ export async function verifyAiModelAccess( ); } - if (!req.userOrg) { + const orgId = row.provider.orgId; + + if (!req.userOrg || req.userOrg.orgId !== orgId) { const userOrgRole = await db .select() .from(userOrgs) @@ -109,6 +90,7 @@ export async function verifyAiModelAccess( } } + req.userOrgId = orgId; req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId); req.aiProvider = row.provider; req.aiModel = row.model; diff --git a/server/middlewares/verifyAiProviderAccess.ts b/server/middlewares/verifyAiProviderAccess.ts index 604e3e893..22edd6f86 100644 --- a/server/middlewares/verifyAiProviderAccess.ts +++ b/server/middlewares/verifyAiProviderAccess.ts @@ -16,7 +16,6 @@ export async function verifyAiProviderAccess( const userId = req.user!.userId; const providerIdRaw = getFirstString(req.params.providerId); const providerId = Number.parseInt(providerIdRaw ?? "", 10); - const orgId = getFirstString(req.params.orgId); if (!userId) { return next( @@ -24,12 +23,6 @@ export async function verifyAiProviderAccess( ); } - if (!orgId) { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") - ); - } - if (Number.isNaN(providerId)) { return next( createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") @@ -39,12 +32,7 @@ export async function verifyAiProviderAccess( const [provider] = await db .select() .from(aiProviders) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiProviders.providerId, providerId)) .limit(1); if (!provider) { @@ -56,7 +44,9 @@ export async function verifyAiProviderAccess( ); } - if (!req.userOrg) { + const orgId = provider.orgId; + + if (!req.userOrg || req.userOrg.orgId !== orgId) { const userOrgRole = await db .select() .from(userOrgs) @@ -93,6 +83,7 @@ export async function verifyAiProviderAccess( } } + req.userOrgId = orgId; req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId); req.aiProvider = provider; diff --git a/server/middlewares/verifyTargetAccess.ts b/server/middlewares/verifyTargetAccess.ts index 24b8abd22..b495cf0e9 100644 --- a/server/middlewares/verifyTargetAccess.ts +++ b/server/middlewares/verifyTargetAccess.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { db } from "@server/db"; -import { resources, targets, userOrgs } from "@server/db"; +import { aiProviders, resources, targets, userOrgs } from "@server/db"; import { and, eq } from "drizzle-orm"; import createHttpError from "http-errors"; import HttpCode from "@server/types/HttpCode"; @@ -25,9 +25,7 @@ export async function verifyTargetAccess( } if (isNaN(targetId)) { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") - ); + return next(createHttpError(HttpCode.BAD_REQUEST, "Invalid target ID")); } const target = await db @@ -45,73 +43,88 @@ export async function verifyTargetAccess( ); } - const resourceId = target[0].resourceId; + const { resourceId, providerId } = target[0]; - if (!resourceId) { + if ((!resourceId && !providerId) || (resourceId && providerId)) { return next( createHttpError( HttpCode.INTERNAL_SERVER_ERROR, - `Target with ID ${targetId} does not have a resource ID` + `Target with ID ${targetId} has invalid ownership` ) ); } try { - const resource = await db - .select() - .from(resources) - .where(eq(resources.resourceId, resourceId!)) - .limit(1); + let orgId: string; - if (resource.length === 0) { - return next( - createHttpError( - HttpCode.NOT_FOUND, - `Resource with ID ${resourceId} not found` - ) - ); - } + if (resourceId) { + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); - if (!resource[0].orgId) { - return next( - createHttpError( - HttpCode.INTERNAL_SERVER_ERROR, - `resource with ID ${resourceId} does not have an organization ID` - ) - ); + if (!resource) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Resource with ID ${resourceId} not found` + ) + ); + } + + if (!resource.orgId) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + `Resource with ID ${resourceId} does not have an organization ID` + ) + ); + } + + orgId = resource.orgId; + } else { + const [provider] = await db + .select() + .from(aiProviders) + .where(eq(aiProviders.providerId, providerId!)) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + orgId = provider.orgId; } if (!req.userOrg) { - const res = await db + const userOrgResult = await db .select() .from(userOrgs) .where( - and( - eq(userOrgs.userId, userId), - eq(userOrgs.orgId, resource[0].orgId) - ) + and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)) ); - req.userOrg = res[0]; + req.userOrg = userOrgResult[0]; } - if (!req.userOrg) { - next( + if (!req.userOrg || req.userOrg.orgId !== orgId) { + return next( createHttpError( HttpCode.FORBIDDEN, "User does not have access to this organization" ) ); - } else { - req.userOrgRoleIds = await getUserOrgRoleIds( - req.userOrg.userId, - resource[0].orgId! - ); - req.userOrgId = resource[0].orgId!; } - const orgId = req.userOrg.orgId; + req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId); + req.userOrgId = orgId; - if (req.orgPolicyAllowed === undefined && orgId) { + if (req.orgPolicyAllowed === undefined) { const policyCheck = await checkOrgAccessPolicy({ orgId, userId, @@ -128,22 +141,24 @@ export async function verifyTargetAccess( } } - const resourceAllowed = await canUserAccessResource({ - userId, - resourceId, - roleIds: req.userOrgRoleIds ?? [] - }); + if (resourceId) { + const resourceAllowed = await canUserAccessResource({ + userId, + resourceId, + roleIds: req.userOrgRoleIds ?? [] + }); - if (!resourceAllowed) { - return next( - createHttpError( - HttpCode.FORBIDDEN, - "User does not have access to this resource" - ) - ); + if (!resourceAllowed) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this resource" + ) + ); + } } - next(); + return next(); } catch (e) { return next( createHttpError( diff --git a/server/routers/aiProvider/createAiModel.ts b/server/routers/aiProvider/createAiModel.ts index a770a5ed8..7fda989d2 100644 --- a/server/routers/aiProvider/createAiModel.ts +++ b/server/routers/aiProvider/createAiModel.ts @@ -15,7 +15,6 @@ import { } from "@server/routers/aiProvider/validation"; const paramsSchema = z.strictObject({ - orgId: z.string().nonempty(), providerId: z.coerce.number().int().positive() }); @@ -33,7 +32,7 @@ const bodySchema = z registry.registerPath({ method: "put", - path: "/org/{orgId}/ai-provider/{providerId}/model", + path: "/ai-provider/{providerId}/model", description: "Create an AI model under a provider.", tags: [OpenAPITags.AiModel], request: { @@ -79,7 +78,7 @@ export async function createAiModel( ); } - const { orgId, providerId } = parsedParams.data; + const { providerId } = parsedParams.data; const { modelKey, name, budgetAmount, budgetUnit, enabled } = parsedBody.data; @@ -89,12 +88,7 @@ export async function createAiModel( : await db .select() .from(aiProviders) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiProviders.providerId, providerId)) .limit(1); if (!provider) { diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index 27420bd34..947810ee1 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -15,6 +15,7 @@ import { aiAuthTypeSchema, aiBudgetUnitSchema, aiProviderTypeSchema, + aiRoutingModeSchema, refineBudgetFields, refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; @@ -30,6 +31,7 @@ const bodySchema = z upstreamUrl: z.url().optional().nullable(), apiKey: z.string().optional(), authType: aiAuthTypeSchema.optional().nullable(), + routingMode: aiRoutingModeSchema.optional(), skipTlsVerification: z.boolean().optional(), budgetAmount: z.number().positive().optional().nullable(), budgetUnit: aiBudgetUnitSchema.optional().nullable(), @@ -95,6 +97,7 @@ export async function createAiProvider( upstreamUrl, apiKey, authType, + routingMode, skipTlsVerification, budgetAmount, budgetUnit, @@ -105,6 +108,8 @@ export async function createAiProvider( const encryptedApiKey = apiKey ? encrypt(apiKey, key) : null; const apiKeyLastChars = apiKey ? apiKey.slice(-4) : null; const now = Date.now(); + const resolvedRoutingMode = + type === "custom" ? (routingMode ?? "url") : "url"; const [provider] = await db .insert(aiProviders) @@ -112,10 +117,14 @@ export async function createAiProvider( orgId, name, type, - upstreamUrl: upstreamUrl ?? null, + upstreamUrl: + resolvedRoutingMode === "target" + ? null + : (upstreamUrl ?? null), apiKey: encryptedApiKey, apiKeyLastChars, authType: authType ?? null, + routingMode: resolvedRoutingMode, skipTlsVerification: skipTlsVerification ?? false, budgetAmount: budgetAmount ?? null, budgetUnit: budgetUnit ?? null, diff --git a/server/routers/aiProvider/deleteAiModel.ts b/server/routers/aiProvider/deleteAiModel.ts index be784275e..e9c58da79 100644 --- a/server/routers/aiProvider/deleteAiModel.ts +++ b/server/routers/aiProvider/deleteAiModel.ts @@ -1,23 +1,21 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { aiModels, aiProviders, db } from "@server/db"; +import { aiModels, db } from "@server/db"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { and, eq } from "drizzle-orm"; +import { eq } from "drizzle-orm"; const paramsSchema = z.strictObject({ - orgId: z.string().nonempty(), - providerId: z.coerce.number().int().positive(), modelId: z.coerce.number().int().positive() }); registry.registerPath({ method: "delete", - path: "/org/{orgId}/ai-provider/{providerId}/model/{modelId}", + path: "/ai-model/{modelId}", description: "Delete an AI model.", tags: [OpenAPITags.AiModel], request: { @@ -46,22 +44,12 @@ export async function deleteAiModel( ); } - const { orgId, providerId, modelId } = parsedParams.data; + const { modelId } = parsedParams.data; const [existing] = await db .select({ modelId: aiModels.modelId }) .from(aiModels) - .innerJoin( - aiProviders, - eq(aiModels.providerId, aiProviders.providerId) - ) - .where( - and( - eq(aiModels.modelId, modelId), - eq(aiModels.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiModels.modelId, modelId)) .limit(1); if (!existing) { @@ -73,14 +61,7 @@ export async function deleteAiModel( ); } - await db - .delete(aiModels) - .where( - and( - eq(aiModels.modelId, modelId), - eq(aiModels.providerId, providerId) - ) - ); + await db.delete(aiModels).where(eq(aiModels.modelId, modelId)); return response(res, { data: null, diff --git a/server/routers/aiProvider/deleteAiProvider.ts b/server/routers/aiProvider/deleteAiProvider.ts index b5416c23c..86a606b7a 100644 --- a/server/routers/aiProvider/deleteAiProvider.ts +++ b/server/routers/aiProvider/deleteAiProvider.ts @@ -7,16 +7,15 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { and, eq } from "drizzle-orm"; +import { eq } from "drizzle-orm"; const paramsSchema = z.strictObject({ - orgId: z.string().nonempty(), providerId: z.coerce.number().int().positive() }); registry.registerPath({ method: "delete", - path: "/org/{orgId}/ai-provider/{providerId}", + path: "/ai-provider/{providerId}", description: "Delete an AI provider.", tags: [OpenAPITags.AiProvider], request: { @@ -45,17 +44,12 @@ export async function deleteAiProvider( ); } - const { orgId, providerId } = parsedParams.data; + const { providerId } = parsedParams.data; const [existing] = await db .select({ providerId: aiProviders.providerId }) .from(aiProviders) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiProviders.providerId, providerId)) .limit(1); if (!existing) { @@ -69,12 +63,7 @@ export async function deleteAiProvider( await db .delete(aiProviders) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ); + .where(eq(aiProviders.providerId, providerId)); return response(res, { data: null, diff --git a/server/routers/aiProvider/getAiModel.ts b/server/routers/aiProvider/getAiModel.ts index 6ea8795ee..45ffbe0ee 100644 --- a/server/routers/aiProvider/getAiModel.ts +++ b/server/routers/aiProvider/getAiModel.ts @@ -1,24 +1,22 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { aiModels, aiProviders, db } from "@server/db"; +import { aiModels, db } from "@server/db"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { and, eq } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import type { GetAiModelResponse } from "@server/routers/aiProvider/types"; const paramsSchema = z.strictObject({ - orgId: z.string().nonempty(), - providerId: z.coerce.number().int().positive(), modelId: z.coerce.number().int().positive() }); registry.registerPath({ method: "get", - path: "/org/{orgId}/ai-provider/{providerId}/model/{modelId}", + path: "/ai-model/{modelId}", description: "Get an AI model by ID.", tags: [OpenAPITags.AiModel], request: { @@ -47,27 +45,16 @@ export async function getAiModel( ); } - const { orgId, providerId, modelId } = parsedParams.data; + const { modelId } = parsedParams.data; const [model] = req.aiModel && req.aiModel.modelId === modelId ? [req.aiModel] : await db - .select({ model: aiModels }) + .select() .from(aiModels) - .innerJoin( - aiProviders, - eq(aiModels.providerId, aiProviders.providerId) - ) - .where( - and( - eq(aiModels.modelId, modelId), - eq(aiModels.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) - .limit(1) - .then((rows) => rows.map((r) => r.model)); + .where(eq(aiModels.modelId, modelId)) + .limit(1); if (!model) { return next( diff --git a/server/routers/aiProvider/getAiProvider.ts b/server/routers/aiProvider/getAiProvider.ts index e2abd3948..32a337c85 100644 --- a/server/routers/aiProvider/getAiProvider.ts +++ b/server/routers/aiProvider/getAiProvider.ts @@ -7,18 +7,17 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { and, eq } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import type { GetAiProviderResponse } from "@server/routers/aiProvider/types"; import { toPublicAiProvider } from "@server/routers/aiProvider/types"; const paramsSchema = z.strictObject({ - orgId: z.string().nonempty(), providerId: z.coerce.number().int().positive() }); registry.registerPath({ method: "get", - path: "/org/{orgId}/ai-provider/{providerId}", + path: "/ai-provider/{providerId}", description: "Get an AI provider by ID.", tags: [OpenAPITags.AiProvider], request: { @@ -47,7 +46,7 @@ export async function getAiProvider( ); } - const { orgId, providerId } = parsedParams.data; + const { providerId } = parsedParams.data; const [provider] = req.aiProvider && req.aiProvider.providerId === providerId @@ -55,12 +54,7 @@ export async function getAiProvider( : await db .select() .from(aiProviders) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiProviders.providerId, providerId)) .limit(1); if (!provider) { diff --git a/server/routers/aiProvider/listAiModels.ts b/server/routers/aiProvider/listAiModels.ts index a7794efca..753d51499 100644 --- a/server/routers/aiProvider/listAiModels.ts +++ b/server/routers/aiProvider/listAiModels.ts @@ -11,7 +11,6 @@ import { and, asc, eq, like, sql } from "drizzle-orm"; import type { ListAiModelsResponse } from "@server/routers/aiProvider/types"; const paramsSchema = z.strictObject({ - orgId: z.string().nonempty(), providerId: z.coerce.number().int().positive() }); @@ -45,7 +44,7 @@ const listSchema = z.object({ registry.registerPath({ method: "get", - path: "/org/{orgId}/ai-provider/{providerId}/models", + path: "/ai-provider/{providerId}/models", description: "List AI models for a provider.", tags: [OpenAPITags.AiModel], request: { @@ -85,7 +84,7 @@ export async function listAiModels( ); } - const { orgId, providerId } = parsedParams.data; + const { providerId } = parsedParams.data; const [provider] = req.aiProvider && req.aiProvider.providerId === providerId @@ -93,12 +92,7 @@ export async function listAiModels( : await db .select({ providerId: aiProviders.providerId }) .from(aiProviders) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiProviders.providerId, providerId)) .limit(1); if (!provider) { diff --git a/server/routers/aiProvider/types.ts b/server/routers/aiProvider/types.ts index 4b8461b5e..2a6047e23 100644 --- a/server/routers/aiProvider/types.ts +++ b/server/routers/aiProvider/types.ts @@ -3,6 +3,7 @@ import type { PaginatedResponse } from "@server/types/Pagination"; import { resolveAiProviderConfig, type AiProviderAuthType, + type AiProviderRoutingMode, type AiProviderType } from "@server/lib/aiProviderDefaults"; @@ -40,7 +41,8 @@ export function toPublicAiProvider(provider: AiProvider): AiProviderPublic { const resolved = resolveAiProviderConfig({ type: provider.type as AiProviderType, upstreamUrl: provider.upstreamUrl, - authType: provider.authType as AiProviderAuthType | null + authType: provider.authType as AiProviderAuthType | null, + routingMode: provider.routingMode as AiProviderRoutingMode | null }); return { diff --git a/server/routers/aiProvider/updateAiModel.ts b/server/routers/aiProvider/updateAiModel.ts index ad09ee823..fa94e8da4 100644 --- a/server/routers/aiProvider/updateAiModel.ts +++ b/server/routers/aiProvider/updateAiModel.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { aiModels, aiProviders, db } from "@server/db"; +import { aiModels, db } from "@server/db"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; @@ -15,8 +15,6 @@ import { } from "@server/routers/aiProvider/validation"; const paramsSchema = z.strictObject({ - orgId: z.string().nonempty(), - providerId: z.coerce.number().int().positive(), modelId: z.coerce.number().int().positive() }); @@ -34,7 +32,7 @@ const bodySchema = z registry.registerPath({ method: "post", - path: "/org/{orgId}/ai-provider/{providerId}/model/{modelId}", + path: "/ai-model/{modelId}", description: "Update an AI model.", tags: [OpenAPITags.AiModel], request: { @@ -80,28 +78,17 @@ export async function updateAiModel( ); } - const { orgId, providerId, modelId } = parsedParams.data; + const { modelId } = parsedParams.data; const body = parsedBody.data; const [existing] = req.aiModel && req.aiModel.modelId === modelId ? [req.aiModel] : await db - .select({ model: aiModels }) + .select() .from(aiModels) - .innerJoin( - aiProviders, - eq(aiModels.providerId, aiProviders.providerId) - ) - .where( - and( - eq(aiModels.modelId, modelId), - eq(aiModels.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) - .limit(1) - .then((rows) => rows.map((r) => r.model)); + .where(eq(aiModels.modelId, modelId)) + .limit(1); if (!existing) { return next( @@ -121,7 +108,7 @@ export async function updateAiModel( .from(aiModels) .where( and( - eq(aiModels.providerId, providerId), + eq(aiModels.providerId, existing.providerId), eq(aiModels.modelKey, body.modelKey), ne(aiModels.modelId, modelId) ) @@ -161,12 +148,7 @@ export async function updateAiModel( const [model] = await db .update(aiModels) .set(updateData) - .where( - and( - eq(aiModels.modelId, modelId), - eq(aiModels.providerId, providerId) - ) - ) + .where(eq(aiModels.modelId, modelId)) .returning(); return response(res, { diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts index 186b5bcd8..494e459f6 100644 --- a/server/routers/aiProvider/updateAiProvider.ts +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -7,7 +7,7 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { and, eq } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import { encrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; @@ -16,16 +16,16 @@ import { aiAuthTypeSchema, aiBudgetUnitSchema, aiProviderTypeSchema, + aiRoutingModeSchema, refineBudgetFields, refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; -import { - providerRequiresUpstreamUrl, - type AiProviderType +import type { + AiProviderRoutingMode, + AiProviderType } from "@server/lib/aiProviderDefaults"; const paramsSchema = z.strictObject({ - orgId: z.string().nonempty(), providerId: z.coerce.number().int().positive() }); @@ -35,6 +35,7 @@ const bodySchema = z upstreamUrl: z.url().optional().nullable(), apiKey: z.string().optional(), authType: aiAuthTypeSchema.optional().nullable(), + routingMode: aiRoutingModeSchema.optional(), skipTlsVerification: z.boolean().optional(), budgetAmount: z.number().positive().optional().nullable(), budgetUnit: aiBudgetUnitSchema.optional().nullable(), @@ -46,7 +47,7 @@ const bodySchema = z registry.registerPath({ method: "post", - path: "/org/{orgId}/ai-provider/{providerId}", + path: "/ai-provider/{providerId}", description: "Update an AI provider.", tags: [OpenAPITags.AiProvider], request: { @@ -92,7 +93,7 @@ export async function updateAiProvider( ); } - const { orgId, providerId } = parsedParams.data; + const { providerId } = parsedParams.data; const body = parsedBody.data; const [existing] = @@ -101,12 +102,7 @@ export async function updateAiProvider( : await db .select() .from(aiProviders) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiProviders.providerId, providerId)) .limit(1); if (!existing) { @@ -119,6 +115,11 @@ export async function updateAiProvider( } const providerType = existing.type as AiProviderType; + const nextRoutingMode: AiProviderRoutingMode = + providerType === "custom" + ? ((body.routingMode ?? + existing.routingMode) as AiProviderRoutingMode) + : "url"; const nextUpstreamUrl = body.upstreamUrl !== undefined ? body.upstreamUrl @@ -126,38 +127,33 @@ export async function updateAiProvider( const nextAuthType = body.authType !== undefined ? body.authType : existing.authType; - if ( - providerRequiresUpstreamUrl(providerType) || - body.upstreamUrl !== undefined || - body.authType !== undefined - ) { - const validation = z - .object({ - type: aiProviderTypeSchema, - upstreamUrl: z.string().nullable().optional(), - authType: aiAuthTypeSchema.nullable().optional() - }) - .superRefine((data, ctx) => - refineProviderUpstreamFields(data, ctx) - ) - .safeParse({ - type: providerType, - upstreamUrl: nextUpstreamUrl, - authType: nextAuthType - }); + const validation = z + .object({ + type: aiProviderTypeSchema, + upstreamUrl: z.string().nullable().optional(), + authType: aiAuthTypeSchema.nullable().optional(), + routingMode: aiRoutingModeSchema.optional() + }) + .superRefine((data, ctx) => refineProviderUpstreamFields(data, ctx)) + .safeParse({ + type: providerType, + upstreamUrl: nextUpstreamUrl, + authType: nextAuthType, + routingMode: nextRoutingMode + }); - if (!validation.success) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - fromError(validation.error).toString() - ) - ); - } + if (!validation.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(validation.error).toString() + ) + ); } const updateData: Partial = { - updatedAt: Date.now() + updatedAt: Date.now(), + routingMode: nextRoutingMode }; if (body.name !== undefined) { @@ -175,7 +171,9 @@ export async function updateAiProvider( if (body.budgetUnit !== undefined) { updateData.budgetUnit = body.budgetUnit; } - if (body.upstreamUrl !== undefined) { + if (nextRoutingMode === "target") { + updateData.upstreamUrl = null; + } else if (body.upstreamUrl !== undefined) { updateData.upstreamUrl = body.upstreamUrl; } if (body.authType !== undefined) { @@ -191,12 +189,7 @@ export async function updateAiProvider( const [provider] = await db .update(aiProviders) .set(updateData) - .where( - and( - eq(aiProviders.providerId, providerId), - eq(aiProviders.orgId, orgId) - ) - ) + .where(eq(aiProviders.providerId, providerId)) .returning(); return response(res, { diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts index ab70800b5..a0b0aa684 100644 --- a/server/routers/aiProvider/validation.ts +++ b/server/routers/aiProvider/validation.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { providerRequiresUpstreamUrl, type AiBudgetUnit, + type AiProviderRoutingMode, type AiProviderType } from "@server/lib/aiProviderDefaults"; @@ -21,6 +22,8 @@ export const aiBudgetUnitSchema = z.enum(["usd", "tokens"]); export const aiAuthTypeSchema = z.enum(["bearer"]); +export const aiRoutingModeSchema = z.enum(["url", "target"]); + export function refineBudgetFields( data: { budgetAmount?: number | null; @@ -47,10 +50,24 @@ export function refineProviderUpstreamFields( type: AiProviderType; upstreamUrl?: string | null; authType?: "bearer" | null; + routingMode?: AiProviderRoutingMode | null; }, ctx: z.RefinementCtx ) { - if (providerRequiresUpstreamUrl(data.type) && !data.upstreamUrl) { + const routingMode = data.routingMode ?? "url"; + + if (data.type !== "custom" && routingMode === "target") { + ctx.addIssue({ + code: "custom", + message: "routingMode target is only allowed for custom providers", + path: ["routingMode"] + }); + } + + if ( + providerRequiresUpstreamUrl(data.type, routingMode) && + !data.upstreamUrl + ) { ctx.addIssue({ code: "custom", message: `upstreamUrl is required for ${data.type} providers`, @@ -58,7 +75,7 @@ export function refineProviderUpstreamFields( }); } - if (data.type === "custom" && !data.authType) { + if (data.type === "custom" && routingMode === "url" && !data.authType) { ctx.addIssue({ code: "custom", message: "authType is required for custom providers", diff --git a/server/routers/external.ts b/server/routers/external.ts index 1d33d6166..99bca3a8e 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1385,16 +1385,31 @@ authenticated.get( ); authenticated.get( - "/org/:orgId/ai-provider/:providerId", - verifyOrgAccess, + "/ai-provider/:providerId", verifyAiProviderAccess, verifyUserHasAction(ActionsEnum.getAiProvider), aiProvider.getAiProvider ); +authenticated.put( + "/ai-provider/:providerId/target", + verifyAiProviderAccess, + verifySiteAccess, + verifyLimits, + verifyUserHasAction(ActionsEnum.createTarget), + logActionAudit(ActionsEnum.createTarget), + target.createTarget +); + +authenticated.get( + "/ai-provider/:providerId/targets", + verifyAiProviderAccess, + verifyUserHasAction(ActionsEnum.listTargets), + target.listTargets +); + authenticated.post( - "/org/:orgId/ai-provider/:providerId", - verifyOrgAccess, + "/ai-provider/:providerId", verifyAiProviderAccess, verifyUserHasAction(ActionsEnum.updateAiProvider), logActionAudit(ActionsEnum.updateAiProvider), @@ -1402,8 +1417,7 @@ authenticated.post( ); authenticated.delete( - "/org/:orgId/ai-provider/:providerId", - verifyOrgAccess, + "/ai-provider/:providerId", verifyAiProviderAccess, verifyUserHasAction(ActionsEnum.deleteAiProvider), logActionAudit(ActionsEnum.deleteAiProvider), @@ -1411,8 +1425,7 @@ authenticated.delete( ); authenticated.put( - "/org/:orgId/ai-provider/:providerId/model", - verifyOrgAccess, + "/ai-provider/:providerId/model", verifyAiProviderAccess, verifyUserHasAction(ActionsEnum.createAiModel), logActionAudit(ActionsEnum.createAiModel), @@ -1420,24 +1433,21 @@ authenticated.put( ); authenticated.get( - "/org/:orgId/ai-provider/:providerId/models", - verifyOrgAccess, + "/ai-provider/:providerId/models", verifyAiProviderAccess, verifyUserHasAction(ActionsEnum.listAiModels), aiProvider.listAiModels ); authenticated.get( - "/org/:orgId/ai-provider/:providerId/model/:modelId", - verifyOrgAccess, + "/ai-model/:modelId", verifyAiModelAccess, verifyUserHasAction(ActionsEnum.getAiModel), aiProvider.getAiModel ); authenticated.post( - "/org/:orgId/ai-provider/:providerId/model/:modelId", - verifyOrgAccess, + "/ai-model/:modelId", verifyAiModelAccess, verifyUserHasAction(ActionsEnum.updateAiModel), logActionAudit(ActionsEnum.updateAiModel), @@ -1445,8 +1455,7 @@ authenticated.post( ); authenticated.delete( - "/org/:orgId/ai-provider/:providerId/model/:modelId", - verifyOrgAccess, + "/ai-model/:modelId", verifyAiModelAccess, verifyUserHasAction(ActionsEnum.deleteAiModel), logActionAudit(ActionsEnum.deleteAiModel), diff --git a/server/routers/integration.ts b/server/routers/integration.ts index 3a17079c6..384214e4c 100644 --- a/server/routers/integration.ts +++ b/server/routers/integration.ts @@ -1386,16 +1386,30 @@ authenticated.get( ); authenticated.get( - "/org/:orgId/ai-provider/:providerId", - verifyApiKeyOrgAccess, + "/ai-provider/:providerId", verifyApiKeyAiProviderAccess, verifyApiKeyHasAction(ActionsEnum.getAiProvider), aiProvider.getAiProvider ); +authenticated.put( + "/ai-provider/:providerId/target", + verifyApiKeyAiProviderAccess, + verifyLimits, + verifyApiKeyHasAction(ActionsEnum.createTarget), + logActionAudit(ActionsEnum.createTarget), + target.createTarget +); + +authenticated.get( + "/ai-provider/:providerId/targets", + verifyApiKeyAiProviderAccess, + verifyApiKeyHasAction(ActionsEnum.listTargets), + target.listTargets +); + authenticated.post( - "/org/:orgId/ai-provider/:providerId", - verifyApiKeyOrgAccess, + "/ai-provider/:providerId", verifyApiKeyAiProviderAccess, verifyApiKeyHasAction(ActionsEnum.updateAiProvider), logActionAudit(ActionsEnum.updateAiProvider), @@ -1403,8 +1417,7 @@ authenticated.post( ); authenticated.delete( - "/org/:orgId/ai-provider/:providerId", - verifyApiKeyOrgAccess, + "/ai-provider/:providerId", verifyApiKeyAiProviderAccess, verifyApiKeyHasAction(ActionsEnum.deleteAiProvider), logActionAudit(ActionsEnum.deleteAiProvider), @@ -1412,8 +1425,7 @@ authenticated.delete( ); authenticated.put( - "/org/:orgId/ai-provider/:providerId/model", - verifyApiKeyOrgAccess, + "/ai-provider/:providerId/model", verifyApiKeyAiProviderAccess, verifyApiKeyHasAction(ActionsEnum.createAiModel), logActionAudit(ActionsEnum.createAiModel), @@ -1421,24 +1433,21 @@ authenticated.put( ); authenticated.get( - "/org/:orgId/ai-provider/:providerId/models", - verifyApiKeyOrgAccess, + "/ai-provider/:providerId/models", verifyApiKeyAiProviderAccess, verifyApiKeyHasAction(ActionsEnum.listAiModels), aiProvider.listAiModels ); authenticated.get( - "/org/:orgId/ai-provider/:providerId/model/:modelId", - verifyApiKeyOrgAccess, + "/ai-model/:modelId", verifyApiKeyAiModelAccess, verifyApiKeyHasAction(ActionsEnum.getAiModel), aiProvider.getAiModel ); authenticated.post( - "/org/:orgId/ai-provider/:providerId/model/:modelId", - verifyApiKeyOrgAccess, + "/ai-model/:modelId", verifyApiKeyAiModelAccess, verifyApiKeyHasAction(ActionsEnum.updateAiModel), logActionAudit(ActionsEnum.updateAiModel), @@ -1446,8 +1455,7 @@ authenticated.post( ); authenticated.delete( - "/org/:orgId/ai-provider/:providerId/model/:modelId", - verifyApiKeyOrgAccess, + "/ai-model/:modelId", verifyApiKeyAiModelAccess, verifyApiKeyHasAction(ActionsEnum.deleteAiModel), logActionAudit(ActionsEnum.deleteAiModel), diff --git a/server/routers/newt/buildConfiguration.ts b/server/routers/newt/buildConfiguration.ts index ee33a2c94..e1782fd40 100644 --- a/server/routers/newt/buildConfiguration.ts +++ b/server/routers/newt/buildConfiguration.ts @@ -15,7 +15,7 @@ import { } from "@server/db"; import logger from "@server/logger"; import { initPeerAddHandshake, updatePeer } from "../olm/peers"; -import { eq, and, inArray } from "drizzle-orm"; +import { eq, and, inArray, or, isNotNull, sql } from "drizzle-orm"; import config from "@server/lib/config"; import { decrypt } from "@server/lib/crypto"; import { @@ -211,7 +211,8 @@ export async function buildClientConfigurationForNewtClient( // call rather than letting each resource fetch its own — with thousands // of resources this avoids a concurrent DB/cache stampede for what is // often the very same (e.g. wildcard) certificate. - const certByDomain = await batchFetchCertsForSiteResources(allSiteResources); + const certByDomain = + await batchFetchCertsForSiteResources(allSiteResources); const resourceTargetsArr = await Promise.all( allSiteResources.map((resource) => @@ -240,7 +241,7 @@ export async function buildTargetConfigurationForNewtClient( version?: string | null, remoteExitNodeId?: string ) { - // Get all enabled targets with their resource mode information + // Get enabled HTTP/TCP/UDP targets for resources and AI providers const allTargets = await db .select({ resourceId: targets.resourceId, @@ -250,15 +251,18 @@ export async function buildTargetConfigurationForNewtClient( port: targets.port, internalPort: targets.internalPort, enabled: targets.enabled, - mode: resources.mode + mode: sql`COALESCE(${resources.mode}, ${targets.mode})`.mapWith( + String + ) }) .from(targets) - .innerJoin(resources, eq(targets.resourceId, resources.resourceId)) + .leftJoin(resources, eq(targets.resourceId, resources.resourceId)) .where( and( eq(targets.siteId, siteId), eq(targets.enabled, true), - inArray(targets.mode, ["http", "udp", "tcp"]) + inArray(targets.mode, ["http", "udp", "tcp"]), + or(isNotNull(targets.resourceId), isNotNull(targets.providerId)) ) ); diff --git a/server/routers/target/createTarget.ts b/server/routers/target/createTarget.ts index 84fa00cec..fbcaa8439 100644 --- a/server/routers/target/createTarget.ts +++ b/server/routers/target/createTarget.ts @@ -6,7 +6,14 @@ import { TargetHealthCheck, targetHealthCheck } from "@server/db"; -import { newts, resources, sites, Target, targets } from "@server/db"; +import { + aiProviders, + newts, + resources, + sites, + Target, + targets +} from "@server/db"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; @@ -29,10 +36,19 @@ import { generateId } from "@server/auth/sessions/app"; import config from "@server/lib/config"; import { sendBrowserGatewayTargets } from "@server/routers/newt/targets"; -const createTargetParamsSchema = z.strictObject({ +const resourceTargetParamsSchema = z.strictObject({ resourceId: z.coerce.number().int().positive() }); +const providerTargetParamsSchema = z.strictObject({ + providerId: z.coerce.number().int().positive() +}); + +const createTargetParamsSchema = z.union([ + resourceTargetParamsSchema, + providerTargetParamsSchema +]); + const createTargetSchema = z .strictObject({ siteId: z.int().positive(), @@ -95,7 +111,7 @@ registry.registerPath({ description: "Create a target for a resource.", tags: [OpenAPITags.PublicResourceLegacy], request: { - params: createTargetParamsSchema, + params: resourceTargetParamsSchema, body: { content: { "application/json": { @@ -128,7 +144,40 @@ registry.registerPath({ description: "Create a target for a resource.", tags: [OpenAPITags.PublicResource, OpenAPITags.Target], request: { - params: createTargetParamsSchema, + params: resourceTargetParamsSchema, + body: { + content: { + "application/json": { + schema: createTargetSchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +registry.registerPath({ + method: "put", + path: "/ai-provider/{providerId}/target", + description: "Create a target for an AI provider.", + tags: [OpenAPITags.AiProvider], + request: { + params: providerTargetParamsSchema, body: { content: { "application/json": { @@ -183,21 +232,74 @@ export async function createTarget( ); } - const { resourceId } = parsedParams.data; + let resource: typeof resources.$inferSelect | undefined; + let provider: typeof aiProviders.$inferSelect | undefined; - // get the resource - const [resource] = await db - .select() - .from(resources) - .where(eq(resources.resourceId, resourceId)); + if ("providerId" in parsedParams.data) { + const { providerId } = parsedParams.data; + [provider] = + req.aiProvider && req.aiProvider.providerId === providerId + ? [req.aiProvider] + : await db + .select() + .from(aiProviders) + .where(eq(aiProviders.providerId, providerId)) + .limit(1); - if (!resource) { - return next( - createHttpError( - HttpCode.NOT_FOUND, - `Resource with ID ${resourceId} not found` - ) - ); + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + if (provider.routingMode !== "target") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI provider must use target routing mode" + ) + ); + } + + if (provider.type !== "custom") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Only custom AI providers support targets" + ) + ); + } + + if ( + targetData.method && + !["http", "https"].includes(targetData.method.toLowerCase()) + ) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI provider target method must be http or https" + ) + ); + } + } else { + const { resourceId } = parsedParams.data; + [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Resource with ID ${resourceId} not found` + ) + ); + } } const siteId = targetData.siteId; @@ -217,6 +319,24 @@ export async function createTarget( ); } + if (provider && site.orgId && site.orgId !== provider.orgId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Site must belong to the AI provider organization" + ) + ); + } + + const resourceId = resource?.resourceId ?? null; + const providerId = provider?.providerId ?? null; + const targetMode = provider + ? "http" + : (targetData.mode ?? resource?.mode ?? "http"); + const targetMethod = provider + ? (targetData.method?.toLowerCase() ?? "https") + : targetData.method; + const plainToken = generateId(48); const encryptedToken = encrypt( plainToken, @@ -230,20 +350,24 @@ export async function createTarget( const existingTargets = await trx .select() .from(targets) - .where(eq(targets.resourceId, resourceId)); + .where( + providerId + ? eq(targets.providerId, providerId) + : eq(targets.resourceId, resourceId!) + ); const existingTarget = existingTargets.find( (target) => target.ip === targetData.ip && target.port === targetData.port && - target.method === targetData.method && + target.method === targetMethod && target.siteId === targetData.siteId ); if (existingTarget) { // log a warning logger.warn( - `Target with IP ${targetData.ip}, port ${targetData.port}, method ${targetData.method} already exists for resource ID ${resourceId}` + `Target with IP ${targetData.ip}, port ${targetData.port}, method ${targetMethod} already exists for ${providerId ? `AI provider ID ${providerId}` : `resource ID ${resourceId}`}` ); } @@ -252,10 +376,10 @@ export async function createTarget( .insert(targets) .values({ resourceId, + providerId, ...targetData, - mode: (targetData.mode ?? - resource.mode ?? - "http") as Target["mode"], + mode: targetMode as Target["mode"], + method: targetMethod, priority: targetData.priority || 100 }) .returning(); @@ -289,13 +413,12 @@ export async function createTarget( .insert(targets) .values({ resourceId, + providerId, siteId: site.siteId, ip: targetData.ip, - mode: (targetData.mode ?? - resource.mode ?? - "http") as Target["mode"], + mode: targetMode as Target["mode"], authToken: encryptedToken, - method: targetData.method, + method: targetMethod, port: targetData.port, internalPort, enabled: targetData.enabled, @@ -321,10 +444,12 @@ export async function createTarget( healthCheck = await trx .insert(targetHealthCheck) .values({ - orgId: resource.orgId, + orgId: provider?.orgId ?? resource!.orgId, targetId: newTarget[0].targetId, siteId: targetData.siteId, - name: `Resource ${resource.name} - ${targetData.ip}:${targetData.port}`, + name: provider + ? `AI Provider ${provider.name} - ${targetData.ip}:${targetData.port}` + : `Resource ${resource!.name} - ${targetData.ip}:${targetData.port}`, hcEnabled: targetData.hcEnabled ?? false, hcPath: targetData.hcPath ?? null, hcScheme: targetData.hcScheme ?? null, @@ -399,10 +524,17 @@ export async function createTarget( newt.newtId, newTarget, healthCheck, - resource.mode === "udp" ? "udp" : "tcp", + provider + ? "tcp" + : (resource!.mode as string) === "udp" + ? "udp" + : "tcp", newt.version ); - } else if (["ssh", "rdp", "vnc"].includes(newTarget[0].mode)) { + } else if ( + !provider && + ["ssh", "rdp", "vnc"].includes(newTarget[0].mode) + ) { await sendBrowserGatewayTargets( newt.newtId, newTarget, diff --git a/server/routers/target/deleteTarget.ts b/server/routers/target/deleteTarget.ts index a959e10eb..8baf17d66 100644 --- a/server/routers/target/deleteTarget.ts +++ b/server/routers/target/deleteTarget.ts @@ -79,38 +79,54 @@ export async function deleteTarget( ) ); } - // get the resource - const [resource] = await db - .select() - .from(resources) - .where(eq(resources.resourceId, deletedTarget.resourceId!)); - if (!resource) { + if ( + (!deletedTarget.resourceId && !deletedTarget.providerId) || + (deletedTarget.resourceId && deletedTarget.providerId) + ) { return next( createHttpError( - HttpCode.NOT_FOUND, - `Resource with ID ${deletedTarget.resourceId} not found` + HttpCode.INTERNAL_SERVER_ERROR, + `Target with ID ${targetId} has invalid ownership` ) ); } - // check if there are other targets on the resource - const otherTargets = await db - .select() - .from(targets) - .where( - and( - eq(targets.resourceId, resource.resourceId), - ne(targets.targetId, targetId) - ) - ); + let resource: typeof resources.$inferSelect | undefined; + if (deletedTarget.resourceId) { + [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, deletedTarget.resourceId)) + .limit(1); - if (otherTargets.length == 0) { - // set the resource status - await db - .update(resources) - .set({ health: "unknown" }) - .where(eq(resources.resourceId, resource.resourceId)); + if (!resource) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Resource with ID ${deletedTarget.resourceId} not found` + ) + ); + } + + // check if there are other targets on the resource + const otherTargets = await db + .select() + .from(targets) + .where( + and( + eq(targets.resourceId, resource.resourceId), + ne(targets.targetId, targetId) + ) + ); + + if (otherTargets.length == 0) { + // set the resource status + await db + .update(resources) + .set({ health: "unknown" }) + .where(eq(resources.resourceId, resource.resourceId)); + } } const [site] = await db @@ -137,16 +153,26 @@ export async function deleteTarget( .where(eq(newts.siteId, site.siteId)) .limit(1); - if (["http", "tcp", "udp"].includes(deletedTarget.mode)) { + if ( + deletedTarget.providerId || + ["http", "tcp", "udp"].includes(deletedTarget.mode) + ) { await removeTargets( newt.newtId, // [deletedTarget], [], // deleting the target from newt causes issues because we cant unbind the port. this needs to be fixed in newt before we can do this [deletedHealthCheck], - resource.mode === "udp" ? "udp" : "tcp", + deletedTarget.providerId + ? "tcp" + : (resource!.mode as string) === "udp" + ? "udp" + : "tcp", newt.version ); - } else if (["ssh", "rdp", "vnc"].includes(deletedTarget.mode)) { + } else if ( + !deletedTarget.providerId && + ["ssh", "rdp", "vnc"].includes(deletedTarget.mode) + ) { await removeBrowserGatewayTarget( newt.newtId, deletedTarget.targetId, diff --git a/server/routers/target/listTargets.ts b/server/routers/target/listTargets.ts index f2d16dd21..fc6058429 100644 --- a/server/routers/target/listTargets.ts +++ b/server/routers/target/listTargets.ts @@ -10,10 +10,19 @@ import { fromError } from "zod-validation-error"; import logger from "@server/logger"; import { OpenAPITags, registry } from "@server/openApi"; -const listTargetsParamsSchema = z.strictObject({ +const resourceTargetsParamsSchema = z.strictObject({ resourceId: z.coerce.number().int().positive() }); +const providerTargetsParamsSchema = z.strictObject({ + providerId: z.coerce.number().int().positive() +}); + +const listTargetsParamsSchema = z.union([ + resourceTargetsParamsSchema, + providerTargetsParamsSchema +]); + const listTargetsSchema = z.strictObject({ limit: z .string() @@ -29,7 +38,7 @@ const listTargetsSchema = z.strictObject({ .pipe(z.int().nonnegative()) }); -function queryTargets(resourceId: number) { +function queryTargets(owner: { resourceId: number } | { providerId: number }) { const baseQuery = db .select({ targetId: targets.targetId, @@ -39,6 +48,7 @@ function queryTargets(resourceId: number) { port: targets.port, enabled: targets.enabled, resourceId: targets.resourceId, + providerId: targets.providerId, siteId: targets.siteId, siteType: sites.type, siteName: sites.name, @@ -71,7 +81,11 @@ function queryTargets(resourceId: number) { targetHealthCheck, eq(targetHealthCheck.targetId, targets.targetId) ) - .where(eq(targets.resourceId, resourceId)); + .where( + "providerId" in owner + ? eq(targets.providerId, owner.providerId) + : eq(targets.resourceId, owner.resourceId) + ); return baseQuery; } @@ -94,7 +108,7 @@ registry.registerPath({ description: "List targets for a resource.", tags: [OpenAPITags.PublicResourceLegacy], request: { - params: listTargetsParamsSchema, + params: resourceTargetsParamsSchema, query: listTargetsSchema }, responses: { @@ -121,7 +135,34 @@ registry.registerPath({ description: "List targets for a resource.", tags: [OpenAPITags.PublicResource, OpenAPITags.Target], request: { - params: listTargetsParamsSchema, + params: resourceTargetsParamsSchema, + query: listTargetsSchema + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +registry.registerPath({ + method: "get", + path: "/ai-provider/{providerId}/targets", + description: "List targets for an AI provider.", + tags: [OpenAPITags.AiProvider], + request: { + params: providerTargetsParamsSchema, query: listTargetsSchema }, responses: { @@ -168,14 +209,18 @@ export async function listTargets( ) ); } - const { resourceId } = parsedParams.data; + const owner = parsedParams.data; + const ownerCondition = + "providerId" in owner + ? eq(targets.providerId, owner.providerId) + : eq(targets.resourceId, owner.resourceId); - const baseQuery = queryTargets(resourceId); + const baseQuery = queryTargets(owner); const countQuery = db .select({ count: sql`cast(count(*) as integer)` }) .from(targets) - .where(eq(targets.resourceId, resourceId)); + .where(ownerCondition); const targetsList = await baseQuery.limit(limit).offset(offset); const totalCountResult = await countQuery; diff --git a/server/routers/target/updateTarget.ts b/server/routers/target/updateTarget.ts index 52bf3e578..e1ca95f5f 100644 --- a/server/routers/target/updateTarget.ts +++ b/server/routers/target/updateTarget.ts @@ -1,7 +1,7 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; import { db, targetHealthCheck } from "@server/db"; -import { newts, resources, sites, targets } from "@server/db"; +import { aiProviders, newts, resources, sites, targets } from "@server/db"; import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -147,21 +147,68 @@ export async function updateTarget( ); } - // get the resource - const [resource] = await db - .select() - .from(resources) - .where(eq(resources.resourceId, target.resourceId!)); - - if (!resource) { + if ( + (!target.resourceId && !target.providerId) || + (target.resourceId && target.providerId) + ) { return next( createHttpError( - HttpCode.NOT_FOUND, - `Resource with ID ${target.resourceId} not found` + HttpCode.INTERNAL_SERVER_ERROR, + `Target with ID ${targetId} has invalid ownership` ) ); } + let resource: typeof resources.$inferSelect | undefined; + let provider: typeof aiProviders.$inferSelect | undefined; + + if (target.resourceId) { + [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, target.resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Resource with ID ${target.resourceId} not found` + ) + ); + } + } else { + [provider] = await db + .select() + .from(aiProviders) + .where(eq(aiProviders.providerId, target.providerId!)) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${target.providerId} not found` + ) + ); + } + + if ( + parsedBody.data.method !== undefined && + (!parsedBody.data.method || + !["http", "https"].includes( + parsedBody.data.method.toLowerCase() + )) + ) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI provider target method must be http or https" + ) + ); + } + } + const [site] = await db .select() .from(sites) @@ -177,6 +224,15 @@ export async function updateTarget( ); } + if (provider && site.orgId && site.orgId !== provider.orgId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Site must belong to the AI provider organization" + ) + ); + } + const { internalPort, targetIps } = await pickPort(site.siteId!, db); if (!internalPort) { @@ -221,8 +277,13 @@ export async function updateTarget( } const pathMatchTypeRemoved = parsedBody.data.pathMatchType === null; - const nextMode = - parsedBody.data.mode === null ? undefined : parsedBody.data.mode; + const nextMode = provider + ? parsedBody.data.mode !== undefined + ? "http" + : undefined + : parsedBody.data.mode === null + ? undefined + : parsedBody.data.mode; let updatedTarget: any; let updatedHc: any; @@ -233,7 +294,10 @@ export async function updateTarget( siteId: parsedBody.data.siteId, ip: parsedBody.data.ip, mode: nextMode, - method: parsedBody.data.method, + method: + provider && parsedBody.data.method + ? parsedBody.data.method.toLowerCase() + : parsedBody.data.method, port: parsedBody.data.port, internalPort, enabled: parsedBody.data.enabled, @@ -368,15 +432,25 @@ export async function updateTarget( .where(eq(newts.siteId, site.siteId)) .limit(1); - if (["http", "tcp", "udp"].includes(updatedTarget.mode)) { + if ( + provider || + ["http", "tcp", "udp"].includes(updatedTarget.mode) + ) { await addTargets( newt.newtId, [updatedTarget], [updatedHc], - resource.mode === "udp" ? "udp" : "tcp", + provider + ? "tcp" + : (resource!.mode as string) === "udp" + ? "udp" + : "tcp", newt.version ); - } else if (["ssh", "rdp", "vnc"].includes(updatedTarget.mode)) { + } else if ( + !provider && + ["ssh", "rdp", "vnc"].includes(updatedTarget.mode) + ) { await sendBrowserGatewayTargets( newt.newtId, [updatedTarget], diff --git a/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx b/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx index b8042b2f3..0db4cb156 100644 --- a/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx +++ b/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx @@ -596,6 +596,7 @@ export function ProxyResourceTargetsForm({ priority: 100, enabled: true, resourceId: resource?.resourceId ?? 0, + providerId: null, hcEnabled: false, hcPath: null, hcMethod: null, From 1073011a2a8bc98102ffc8ef8e8e5f1cbaec5f5f Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 2 Aug 2026 10:16:36 -0400 Subject: [PATCH 018/296] first pass of traefik -> basic gateway --- server/db/pg/schema/schema.ts | 42 ++- server/db/sqlite/schema/schema.ts | 42 ++- server/lib/traefik/TraefikConfigManager.ts | 6 +- server/lib/traefik/getTraefikConfig.ts | 121 +++++++- .../private/lib/traefik/getTraefikConfig.ts | 269 +++++++++++++++++- server/private/routers/hybrid.ts | 4 +- server/routers/aiGateway/chatCompletions.ts | 256 +++++++++++++++++ server/routers/aiGateway/index.ts | 1 + server/routers/internal.ts | 8 + server/routers/resource/createResource.ts | 21 +- server/routers/resource/updateResource.ts | 9 + .../siteResource/createSiteResource.ts | 16 +- .../siteResource/updateSiteResource.ts | 15 +- .../routers/traefik/traefikConfigProvider.ts | 6 +- 14 files changed, 795 insertions(+), 21 deletions(-) create mode 100644 server/routers/aiGateway/chatCompletions.ts create mode 100644 server/routers/aiGateway/index.ts diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 0f59b1c26..e253cd6d5 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -210,7 +210,11 @@ export const resources = pgTable( authDaemonPort: integer("authDaemonPort").default(22123), status: varchar("status") .$type<"pending" | "approved">() - .default("approved") + .default("approved"), + aiProviderId: integer("aiProviderId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ) }, (t) => [ index("idx_resources_fulldomain") @@ -221,6 +225,19 @@ export const resources = pgTable( ] ); +export const resourceAiModels = pgTable( + "resourceAiModels", + { + resourceId: integer("resourceId") + .notNull() + .references(() => resources.resourceId, { onDelete: "cascade" }), + modelId: integer("modelId") + .notNull() + .references(() => aiModels.modelId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.resourceId, t.modelId] })] +); + export const labels = pgTable("labels", { labelId: serial("labelId").primaryKey(), name: varchar("name").notNull(), @@ -475,11 +492,30 @@ export const siteResources = pgTable( fullDomain: varchar("fullDomain"), status: varchar("status") .$type<"pending" | "approved">() - .default("approved") + .default("approved"), + aiProviderId: integer("aiProviderId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ) }, (t) => [index("idx_siteresources_orgid_niceid").on(t.orgId, t.niceId)] ); +export const siteResourceAiModels = pgTable( + "siteResourceAiModels", + { + siteResourceId: integer("siteResourceId") + .notNull() + .references(() => siteResources.siteResourceId, { + onDelete: "cascade" + }), + modelId: integer("modelId") + .notNull() + .references(() => aiModels.modelId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.siteResourceId, t.modelId] })] +); + export const networks = pgTable( "networks", { @@ -1698,3 +1734,5 @@ export type UserPolicy = InferSelectModel; export type ResourcePolicyRule = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; +export type ResourceAiModel = InferSelectModel; +export type SiteResourceAiModel = InferSelectModel; diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 73e73f04a..3d3ff2ded 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -215,9 +215,26 @@ export const resources = sqliteTable("resources", { .$type<"site" | "remote" | "native">() .default("site"), authDaemonPort: integer("authDaemonPort").default(22123), - status: text("status").$type<"pending" | "approved">().default("approved") + status: text("status").$type<"pending" | "approved">().default("approved"), + aiProviderId: integer("aiProviderId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ) }); +export const resourceAiModels = sqliteTable( + "resourceAiModels", + { + resourceId: integer("resourceId") + .notNull() + .references(() => resources.resourceId, { onDelete: "cascade" }), + modelId: integer("modelId") + .notNull() + .references(() => aiModels.modelId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.resourceId, t.modelId] })] +); + export const labels = sqliteTable("labels", { labelId: integer("labelId").primaryKey({ autoIncrement: true }), name: text("name").notNull(), @@ -462,9 +479,28 @@ export const siteResources = sqliteTable("siteResources", { }), subdomain: text("subdomain"), fullDomain: text("fullDomain"), - status: text("status").$type<"pending" | "approved">().default("approved") + status: text("status").$type<"pending" | "approved">().default("approved"), + aiProviderId: integer("aiProviderId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ) }); +export const siteResourceAiModels = sqliteTable( + "siteResourceAiModels", + { + siteResourceId: integer("siteResourceId") + .notNull() + .references(() => siteResources.siteResourceId, { + onDelete: "cascade" + }), + modelId: integer("modelId") + .notNull() + .references(() => aiModels.modelId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.siteResourceId, t.modelId] })] +); + export const networks = sqliteTable("networks", { networkId: integer("networkId").primaryKey({ autoIncrement: true }), niceId: text("niceId"), @@ -1680,3 +1716,5 @@ export type RolePolicy = InferSelectModel; export type UserPolicy = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; +export type ResourceAiModel = InferSelectModel; +export type SiteResourceAiModel = InferSelectModel; diff --git a/server/lib/traefik/TraefikConfigManager.ts b/server/lib/traefik/TraefikConfigManager.ts index cc7299ff7..73d46ee64 100644 --- a/server/lib/traefik/TraefikConfigManager.ts +++ b/server/lib/traefik/TraefikConfigManager.ts @@ -516,6 +516,9 @@ export class TraefikConfigManager { const maintenanceHost = config.getRawConfig().server.internal_hostname; const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`; + const aiGatewayUrl = `http://${maintenanceHost}:${ + config.getRawConfig().server.internal_port + }/api/v1/ai-gateway`; // logger.debug(`Fetching traefik config for exit node: ${currentExitNode}`); traefikConfig = await getTraefikConfig( @@ -528,7 +531,8 @@ export class TraefikConfigManager { ? false : config.getRawConfig().traefik.allow_raw_resources, // dont allow raw resources on saas otherwise use config pangolinUIUrl, // generate maintenance pages on cloud and hybrid - pangolinUIUrl // generate browser gateway targets on cloud and hybrid + pangolinUIUrl, // generate browser gateway targets on cloud and hybrid + aiGatewayUrl ); const domains = new Set(); diff --git a/server/lib/traefik/getTraefikConfig.ts b/server/lib/traefik/getTraefikConfig.ts index 3e84cd570..4f5dd3860 100644 --- a/server/lib/traefik/getTraefikConfig.ts +++ b/server/lib/traefik/getTraefikConfig.ts @@ -1,4 +1,4 @@ -import { db, targetHealthCheck, domains } from "@server/db"; +import { db, targetHealthCheck, domains, aiProviders } from "@server/db"; import { and, eq, @@ -45,7 +45,8 @@ export async function getTraefikConfig( generateLoginPageRouters = false, // UNUSED BUT USED IN PRIVATE allowRawResources = true, maintenancePageUiUrl: string | null = null, // UNUSED BUT USED IN PRIVATE - browserGatewayUiUrl: string | null = null // UNUSED BUT USED IN PRIVATE + browserGatewayUiUrl: string | null = null, // UNUSED BUT USED IN PRIVATE + aiGatewayUrl: string | null = null ): Promise { // Get resources with their targets and sites in a single optimized query // Start from sites on this exit node, then join to targets and resources @@ -209,8 +210,37 @@ export async function getTraefikConfig( }); }); + // Inference-mode resources have no targets/sites (their "backend" is the + // central AI gateway), so they can't be reached via the targets->sites + // join above - query them separately and include them on every exit node. + const inferenceResources = await db + .select({ + resourceId: resources.resourceId, + resourceName: resources.name, + fullDomain: resources.fullDomain, + ssl: resources.ssl, + subdomain: resources.subdomain, + domainId: resources.domainId, + enabled: resources.enabled, + domainCertResolver: domains.certResolver, + preferWildcardCert: domains.preferWildcardCert + }) + .from(resources) + .innerJoin( + aiProviders, + eq(resources.aiProviderId, aiProviders.providerId) + ) + .leftJoin(domains, eq(domains.domainId, resources.domainId)) + .where( + and( + eq(resources.mode, "inference"), + eq(resources.enabled, true), + eq(aiProviders.enabled, true) + ) + ); + // make sure we have at least one resource - if (resourcesMap.size === 0) { + if (resourcesMap.size === 0 && inferenceResources.length === 0) { return {}; } @@ -673,5 +703,90 @@ export async function getTraefikConfig( }; } } + + if (aiGatewayUrl) { + for (const ir of inferenceResources) { + if (!ir.enabled) continue; + if (!ir.domainId || !ir.fullDomain) continue; + + if (!config_output.http.routers) config_output.http.routers = {}; + if (!config_output.http.services) config_output.http.services = {}; + + const fullDomain = ir.fullDomain; + const irKey = `inference-r${ir.resourceId}`; + const routerName = `${irKey}-router`; + const serviceName = `${irKey}-service`; + const rule = `Host(\`${fullDomain}\`)`; + + const domainParts = fullDomain.split("."); + let wildCard; + if (domainParts.length <= 2) { + wildCard = `*.${domainParts.join(".")}`; + } else { + wildCard = `*.${domainParts.slice(1).join(".")}`; + } + if (!ir.subdomain) { + wildCard = fullDomain; + } + + const globalDefaultResolver = + config.getRawConfig().traefik.cert_resolver; + const globalDefaultPreferWildcard = + config.getRawConfig().traefik.prefer_wildcard_cert; + const resolverName = ir.domainCertResolver + ? ir.domainCertResolver.trim() + : globalDefaultResolver; + const preferWildcard = + ir.preferWildcardCert !== undefined && + ir.preferWildcardCert !== null + ? ir.preferWildcardCert + : globalDefaultPreferWildcard; + + const tls = { + certResolver: resolverName, + ...(preferWildcard ? { domains: [{ main: wildCard }] } : {}) + }; + + const additionalMiddlewares = + config.getRawConfig().traefik.additional_middlewares || []; + const routerMiddlewares = [ + badgerMiddlewareName, + ...additionalMiddlewares + ]; + + config_output.http.routers[routerName] = { + entryPoints: [ + ir.ssl + ? config.getRawConfig().traefik.https_entrypoint + : config.getRawConfig().traefik.http_entrypoint + ], + middlewares: routerMiddlewares, + service: serviceName, + rule, + priority: 100, + ...(ir.ssl ? { tls } : {}) + }; + + if (ir.ssl) { + config_output.http.routers[routerName + "-redirect"] = { + entryPoints: [ + config.getRawConfig().traefik.http_entrypoint + ], + middlewares: [redirectHttpsMiddlewareName], + service: serviceName, + rule, + priority: 100 + }; + } + + config_output.http.services[serviceName] = { + loadBalancer: { + servers: [{ url: `${aiGatewayUrl}/chat/completions` }], + passHostHeader: true + } + }; + } + } + return config_output; } diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 0544cafe7..8e1b1eca2 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -41,7 +41,8 @@ import { siteNetworks, siteResources, Target, - targets + targets, + aiProviders } from "@server/db"; import { sanitize, @@ -88,7 +89,8 @@ export async function getTraefikConfig( generateLoginPageRouters = false, allowRawResources = true, maintenancePageUiUrl: string | null = null, - browserGatewayUiUrl: string | null = null + browserGatewayUiUrl: string | null = null, + aiGatewayUrl: string | null = null ): Promise { // Get resources with their targets and sites in a single optimized query // Start from sites on this exit node, then join to targets and resources @@ -395,6 +397,65 @@ export async function getTraefikConfig( ); } + // Inference-mode resources/siteResources have no targets/sites/network + // (their "backend" is the central AI gateway, not something on a site), + // so they can't be reached via the joins above - query them separately + // and include them on every exit node. + const inferenceResources = await db + .select({ + resourceId: resources.resourceId, + fullDomain: resources.fullDomain, + ssl: resources.ssl, + subdomain: resources.subdomain, + domainId: resources.domainId, + enabled: resources.enabled, + wildcard: resources.wildcard, + domainCertResolver: domains.certResolver, + preferWildcardCert: domains.preferWildcardCert + }) + .from(resources) + .innerJoin( + aiProviders, + eq(resources.aiProviderId, aiProviders.providerId) + ) + .leftJoin(domains, eq(domains.domainId, resources.domainId)) + .where( + and( + eq(resources.mode, "inference"), + eq(resources.enabled, true), + eq(aiProviders.enabled, true) + ) + ); + + let siteResourcesInference: { + siteResourceId: number; + alias: string | null; + ssl: boolean | null; + enabled: boolean | null; + }[] = []; + if (build == "enterprise") { + siteResourcesInference = await db + .select({ + siteResourceId: siteResources.siteResourceId, + alias: siteResources.alias, + ssl: siteResources.ssl, + enabled: siteResources.enabled + }) + .from(siteResources) + .innerJoin( + aiProviders, + eq(siteResources.aiProviderId, aiProviders.providerId) + ) + .where( + and( + eq(siteResources.mode, "inference"), + eq(siteResources.enabled, true), + eq(aiProviders.enabled, true), + isNotNull(siteResources.alias) + ) + ); + } + let validCerts: CertificateResult[] = []; if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { // create a list of all domains to get certs for @@ -416,6 +477,17 @@ export async function getTraefikConfig( domains.add(bgResource.fullDomain); } } + // Include inference resource/siteResource domains + for (const ir of inferenceResources) { + if (ir.enabled && ir.ssl && ir.fullDomain) { + domains.add(ir.fullDomain); + } + } + for (const sr of siteResourcesInference) { + if (sr.enabled && sr.ssl && sr.alias) { + domains.add(sr.alias); + } + } // get the valid certs for these domains validCerts = await getValidCertificatesForDomains(domains, true); // we are caching here because this is called often // logger.debug(`Valid certs for domains: ${JSON.stringify(validCerts)}`); @@ -1462,6 +1534,199 @@ export async function getTraefikConfig( } } + if (aiGatewayUrl) { + // Public inference resources: same TLS/cert-resolver handling as + // plain http-mode resources, but the service points at the AI + // gateway instead of any real backend targets. + for (const ir of inferenceResources) { + if (!ir.enabled) continue; + if (!ir.domainId || !ir.fullDomain) continue; + + if (!config_output.http.routers) config_output.http.routers = {}; + if (!config_output.http.services) config_output.http.services = {}; + + const fullDomain = ir.fullDomain; + const irKey = `inference-r${ir.resourceId}`; + const routerName = `${irKey}-router`; + const serviceName = `${irKey}-service`; + + let rule: string; + if (ir.wildcard && fullDomain.startsWith("*.")) { + const escaped = fullDomain.slice(2).replace(/\./g, "\\."); + rule = `HostRegexp(\`^[^.]+\\.${escaped}$\`)`; + } else { + rule = `Host(\`${fullDomain}\`)`; + } + + let tls: any = {}; + if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { + const domainParts = fullDomain.split("."); + let wildCard; + if (domainParts.length <= 2) { + wildCard = `*.${domainParts.join(".")}`; + } else { + wildCard = `*.${domainParts.slice(1).join(".")}`; + } + if (!ir.subdomain) { + wildCard = fullDomain; + } + + const globalDefaultResolver = + config.getRawConfig().traefik.cert_resolver; + const globalDefaultPreferWildcard = + config.getRawConfig().traefik.prefer_wildcard_cert; + const resolverName = ir.domainCertResolver + ? ir.domainCertResolver.trim() + : globalDefaultResolver; + const preferWildcard = + ir.preferWildcardCert !== undefined && + ir.preferWildcardCert !== null + ? ir.preferWildcardCert + : globalDefaultPreferWildcard; + + tls = { + certResolver: resolverName, + ...(preferWildcard + ? { domains: [{ main: wildCard }] } + : {}) + }; + } else { + const matchingCert = validCerts.find( + (cert) => cert.queriedDomain === fullDomain + ); + if (!matchingCert) { + logger.debug( + `No matching certificate found for inference resource domain: ${fullDomain}` + ); + continue; + } + } + + const additionalMiddlewares = + config.getRawConfig().traefik.additional_middlewares || []; + const routerMiddlewares = [ + badgerMiddlewareName, + ...additionalMiddlewares + ]; + + if (ir.ssl) { + config_output.http.routers[routerName + "-redirect"] = { + entryPoints: [ + config.getRawConfig().traefik.http_entrypoint + ], + middlewares: [redirectHttpsMiddlewareName], + service: serviceName, + rule, + priority: 100 + }; + } + + config_output.http.routers[routerName] = { + entryPoints: [ + ir.ssl + ? config.getRawConfig().traefik.https_entrypoint + : config.getRawConfig().traefik.http_entrypoint + ], + middlewares: routerMiddlewares, + service: serviceName, + rule, + priority: 100, + ...(ir.ssl ? { tls } : {}) + }; + + config_output.http.services[serviceName] = { + loadBalancer: { + servers: [{ url: `${aiGatewayUrl}/chat/completions` }], + passHostHeader: true + } + }; + } + + // Private (siteResource) inference resources: routed by their alias + // instead of a public fullDomain, and deliberately WITHOUT the + // badger middleware - no per-user auth/policy stack exists for + // siteResources today (see plan doc), so gating here is + // reachability-only for now. + for (const sr of siteResourcesInference) { + if (!sr.enabled || !sr.alias) continue; + + if (!config_output.http.routers) config_output.http.routers = {}; + if (!config_output.http.services) config_output.http.services = {}; + + const alias = sr.alias; + const srKey = `inference-sr${sr.siteResourceId}`; + const routerName = `${srKey}-router`; + const serviceName = `${srKey}-service`; + const rule = `Host(\`${alias}\`)`; + + let tls: any = {}; + if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { + const domainParts = alias.split("."); + const wildCard = + domainParts.length <= 2 + ? `*.${domainParts.join(".")}` + : `*.${domainParts.slice(1).join(".")}`; + + const globalDefaultResolver = + config.getRawConfig().traefik.cert_resolver; + const globalDefaultPreferWildcard = + config.getRawConfig().traefik.prefer_wildcard_cert; + + tls = { + certResolver: globalDefaultResolver, + ...(globalDefaultPreferWildcard + ? { domains: [{ main: wildCard }] } + : {}) + }; + } else { + const matchingCert = validCerts.find( + (cert) => cert.queriedDomain === alias + ); + if (!matchingCert) { + logger.debug( + `No matching certificate found for inference siteResource alias: ${alias}` + ); + continue; + } + } + + const additionalMiddlewares = + config.getRawConfig().traefik.additional_middlewares || []; + + if (sr.ssl) { + config_output.http.routers[routerName + "-redirect"] = { + entryPoints: [ + config.getRawConfig().traefik.http_entrypoint + ], + middlewares: [redirectHttpsMiddlewareName], + service: serviceName, + rule, + priority: 100 + }; + } + + config_output.http.routers[routerName] = { + entryPoints: [ + sr.ssl + ? config.getRawConfig().traefik.https_entrypoint + : config.getRawConfig().traefik.http_entrypoint + ], + middlewares: additionalMiddlewares, + service: serviceName, + rule, + priority: 100, + ...(sr.ssl ? { tls } : {}) + }; + + config_output.http.services[serviceName] = { + loadBalancer: { + servers: [{ url: `${aiGatewayUrl}/chat/completions` }], + passHostHeader: true + } + }; + } + } + if (generateLoginPageRouters) { const exitNodeLoginPages = await db .select({ diff --git a/server/private/routers/hybrid.ts b/server/private/routers/hybrid.ts index 06799bcd6..e8650e234 100644 --- a/server/private/routers/hybrid.ts +++ b/server/private/routers/hybrid.ts @@ -351,6 +351,7 @@ hybridRouter.get( } const pangolinUIUrl = config.getRawConfig().app.dashboard_url; // points to the dashboard to serve from there + const aiGatewayUrl = `${config.getRawConfig().app.dashboard_url}/api/v1/ai-gateway`; try { const traefikConfig = await getTraefikConfig( @@ -360,7 +361,8 @@ hybridRouter.get( false, // Dont include login pages, true, // allow raw resources pangolinUIUrl, // dont generate maintenance page - pangolinUIUrl // generate browser gateway targets + pangolinUIUrl, // generate browser gateway targets + aiGatewayUrl ); return response(res, { diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts new file mode 100644 index 000000000..cddcf8404 --- /dev/null +++ b/server/routers/aiGateway/chatCompletions.ts @@ -0,0 +1,256 @@ +import { Request, Response } from "express"; +import { and, eq } from "drizzle-orm"; +import { + AiProvider, + aiModels, + aiProviders, + db, + resourceAiModels, + resources, + siteResourceAiModels, + siteResources +} from "@server/db"; +import config from "@server/lib/config"; +import { decrypt } from "@server/lib/crypto"; +import { + AiProviderAuthType, + AiProviderRoutingMode, + AiProviderType, + resolveAiProviderConfig +} from "@server/lib/aiProviderDefaults"; +import logger from "@server/logger"; +import HttpCode from "@server/types/HttpCode"; + +type ResolvedTarget = { + provider: AiProvider; + // null = no restriction; every enabled model on the provider is allowed + allowedModelIds: number[] | null; +}; + +async function resolveTarget(host: string): Promise { + const [resourceRow] = await db + .select({ resourceId: resources.resourceId, provider: aiProviders }) + .from(resources) + .innerJoin( + aiProviders, + eq(resources.aiProviderId, aiProviders.providerId) + ) + .where( + and( + eq(resources.fullDomain, host), + eq(resources.mode, "inference"), + eq(resources.enabled, true), + eq(aiProviders.enabled, true) + ) + ) + .limit(1); + + if (resourceRow) { + const restrictions = await db + .select({ modelId: resourceAiModels.modelId }) + .from(resourceAiModels) + .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)); + + return { + provider: resourceRow.provider, + allowedModelIds: restrictions.length + ? restrictions.map((r) => r.modelId) + : null + }; + } + + const [siteResourceRow] = await db + .select({ + siteResourceId: siteResources.siteResourceId, + provider: aiProviders + }) + .from(siteResources) + .innerJoin( + aiProviders, + eq(siteResources.aiProviderId, aiProviders.providerId) + ) + .where( + and( + eq(siteResources.alias, host), + eq(siteResources.mode, "inference"), + eq(siteResources.enabled, true), + eq(aiProviders.enabled, true) + ) + ) + .limit(1); + + if (siteResourceRow) { + const restrictions = await db + .select({ modelId: siteResourceAiModels.modelId }) + .from(siteResourceAiModels) + .where( + eq( + siteResourceAiModels.siteResourceId, + siteResourceRow.siteResourceId + ) + ); + + return { + provider: siteResourceRow.provider, + allowedModelIds: restrictions.length + ? restrictions.map((r) => r.modelId) + : null + }; + } + + return null; +} + +// Generic OpenAI-wire-compatible passthrough. Anthropic's native API uses a +// different path/schema; everything else here is OpenAI-compatible today. +function getCompletionsPath(type: AiProviderType): string { + if (type === "anthropic") { + return "/v1/messages"; + } + return "/chat/completions"; +} + +export async function chatCompletions(req: Request, res: Response): Promise { + try { + const host = (req.headers.host || "").split(":")[0]; + if (!host) { + return res + .status(HttpCode.BAD_REQUEST) + .json({ error: { message: "Missing Host header" } }); + } + + const target = await resolveTarget(host); + if (!target) { + return res.status(HttpCode.NOT_FOUND).json({ + error: { + message: "No inference resource found for this host" + } + }); + } + + const { provider, allowedModelIds } = target; + const requestedModel = + typeof req.body?.model === "string" ? req.body.model : undefined; + + if (allowedModelIds) { + if (!requestedModel) { + return res.status(HttpCode.FORBIDDEN).json({ + error: { + message: + "This resource restricts access to specific models; a model must be specified" + } + }); + } + + const [matchedModel] = await db + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .where( + and( + eq(aiModels.providerId, provider.providerId), + eq(aiModels.modelKey, requestedModel) + ) + ) + .limit(1); + + if ( + !matchedModel || + !allowedModelIds.includes(matchedModel.modelId) + ) { + return res.status(HttpCode.FORBIDDEN).json({ + error: { + message: `Model "${requestedModel}" is not permitted on this resource` + } + }); + } + } + + if (!provider.apiKey) { + return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ + error: { message: "AI provider has no API key configured" } + }); + } + + const secret = config.getRawConfig().server.secret!; + const apiKey = decrypt(provider.apiKey, secret); + + const { upstreamUrl, authType } = resolveAiProviderConfig({ + type: provider.type as AiProviderType, + upstreamUrl: provider.upstreamUrl, + authType: provider.authType as AiProviderAuthType | null, + routingMode: provider.routingMode as AiProviderRoutingMode | null + }); + + if (!upstreamUrl) { + return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ + error: { + message: "AI provider has no upstream URL configured" + } + }); + } + + const targetUrl = `${upstreamUrl.replace(/\/$/, "")}${getCompletionsPath( + provider.type as AiProviderType + )}`; + + const headers: Record = { + "Content-Type": "application/json" + }; + if (authType === "bearer") { + headers["Authorization"] = `Bearer ${apiKey}`; + } + + // No dedicated per-request TLS agent is wired up (no extra deps for + // this v1 gateway) - toggle the process-wide Node TLS check instead. + // Known limitation: this is not safe under concurrent requests mixing + // skipTlsVerification providers with strict ones. + const restoreTlsReject = process.env.NODE_TLS_REJECT_UNAUTHORIZED; + if (provider.skipTlsVerification) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; + } + + let upstreamRes: globalThis.Response; + try { + upstreamRes = await fetch(targetUrl, { + method: "POST", + headers, + body: JSON.stringify(req.body) + }); + } finally { + if (provider.skipTlsVerification) { + if (restoreTlsReject === undefined) { + delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; + } else { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = restoreTlsReject; + } + } + } + + const contentType = upstreamRes.headers.get("content-type") || ""; + const isStream = + req.body?.stream === true || + contentType.includes("text/event-stream"); + + res.status(upstreamRes.status); + res.setHeader("Content-Type", contentType || "application/json"); + + if (isStream && upstreamRes.body) { + res.flushHeaders(); + const reader = upstreamRes.body.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + res.write(value); + } + return res.end(); + } + + const text = await upstreamRes.text(); + return res.send(text); + } catch (error) { + logger.error(error); + return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ + error: { message: "Failed to proxy inference request" } + }); + } +} diff --git a/server/routers/aiGateway/index.ts b/server/routers/aiGateway/index.ts new file mode 100644 index 000000000..691ebebe8 --- /dev/null +++ b/server/routers/aiGateway/index.ts @@ -0,0 +1 @@ +export * from "./chatCompletions"; diff --git a/server/routers/internal.ts b/server/routers/internal.ts index 2fa5239cc..af12d4429 100644 --- a/server/routers/internal.ts +++ b/server/routers/internal.ts @@ -3,6 +3,7 @@ import * as gerbil from "@server/routers/gerbil"; import * as traefik from "@server/routers/traefik"; import * as resource from "./resource"; import * as badger from "./badger"; +import * as aiGateway from "@server/routers/aiGateway"; import * as auth from "@server/routers/auth"; import * as supporterKey from "@server/routers/supporterKey"; import * as idp from "@server/routers/idp"; @@ -63,3 +64,10 @@ internalRouter.use("/badger", badgerRouter); badgerRouter.post("/verify-session", badger.verifyResourceSession); badgerRouter.post("/exchange-session", badger.exchangeSession); + +// AI inference gateway - minimal chat-completions proxy for inference-mode +// resources/siteResources +internalRouter.post( + "/ai-gateway/chat/completions", + aiGateway.chatCompletions +); diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index 2ff7b9ae3..e56896e55 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -90,11 +90,22 @@ const createHttpResourceSchema = z domainId: z.string(), stickySession: z.boolean().optional(), postAuthPath: z.string().nullable().optional(), - mode: z.enum(["http", "ssh", "rdp", "vnc", "tcp", "udp"]).optional(), + mode: z + .enum(["http", "ssh", "rdp", "vnc", "tcp", "udp", "inference"]) + .optional(), // SSH Settings pamMode: z.enum(["passthrough", "push"]).optional(), authDaemonPort: z.int().positive().optional(), - authDaemonMode: z.enum(["site", "remote", "native"]).optional() + authDaemonMode: z.enum(["site", "remote", "native"]).optional(), + // Inference settings + aiProviderId: z + .number() + .int() + .positive() + .optional() + .describe( + "For inference-mode resources: the AI provider this resource proxies chat completions to." + ) }) .refine( (data) => { @@ -365,7 +376,8 @@ async function createHttpResource( mode, authDaemonPort, authDaemonMode, - pamMode + pamMode, + aiProviderId } = parsedBody.data; const subdomain = parsedBody.data.subdomain; const stickySession = parsedBody.data.stickySession; @@ -552,7 +564,8 @@ async function createHttpResource( postAuthPath: postAuthPath, wildcard, health: "unknown", - defaultResourcePolicyId: defaultPolicy.resourcePolicyId + defaultResourcePolicyId: defaultPolicy.resourcePolicyId, + aiProviderId: aiProviderId ?? null }) .returning(); diff --git a/server/routers/resource/updateResource.ts b/server/routers/resource/updateResource.ts index f3e0c3909..c67cb2795 100644 --- a/server/routers/resource/updateResource.ts +++ b/server/routers/resource/updateResource.ts @@ -120,6 +120,15 @@ const updateHttpResourceBodySchema = z .optional() .describe( "ID of the resource policy to apply to this resource. Set to null to remove the resource policy and fall back to the inline policy settings." + ), + aiProviderId: z + .number() + .int() + .positive() + .nullable() + .optional() + .describe( + "For inference-mode resources: the AI provider this resource proxies chat completions to. Set to null to unlink." ) }) .refine((data) => Object.keys(data).length > 0, { diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 1a4cd9a4d..9a6f91ad7 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -78,7 +78,15 @@ const createSiteResourceSchema = z authDaemonMode: z.enum(["site", "remote", "native"]).optional(), pamMode: z.enum(["passthrough", "push"]).optional(), domainId: z.string().optional(), // only used for http mode, we need this to verify the alias is unique within the org - subdomain: z.string().optional() // only used for http mode, we need this to verify the alias is unique within the org + subdomain: z.string().optional(), // only used for http mode, we need this to verify the alias is unique within the org + aiProviderId: z + .number() + .int() + .positive() + .optional() + .describe( + "For inference-mode site resources: the AI provider this resource proxies chat completions to." + ) }) .strict() .refine( @@ -322,7 +330,8 @@ export async function createSiteResource( authDaemonMode, pamMode, domainId, - subdomain + subdomain, + aiProviderId } = parsedBody.data; // Backward compatibility: merge deprecated siteId into siteIds array @@ -594,7 +603,8 @@ export async function createSiteResource( domainId, subdomain: finalSubdomain, fullDomain, - requiresExitNodeConnection: mode === "inference" // in the future we might want to have different modes that do this + requiresExitNodeConnection: mode === "inference", // in the future we might want to have different modes that do this + aiProviderId: aiProviderId ?? null }; if (isLicensedSshPam) { if (authDaemonPort !== undefined) diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index 0ab928d04..0ae57c087 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -78,7 +78,16 @@ const updateSiteResourceSchema = z authDaemonMode: z.enum(["site", "remote", "native"]).optional(), pamMode: z.enum(["passthrough", "push"]).optional(), domainId: z.string().optional(), - subdomain: z.string().optional() + subdomain: z.string().optional(), + aiProviderId: z + .number() + .int() + .positive() + .nullable() + .optional() + .describe( + "For inference-mode site resources: the AI provider this resource proxies chat completions to. Set to null to unlink." + ) }) .strict() .refine( @@ -329,7 +338,8 @@ export async function updateSiteResource( authDaemonMode, pamMode, domainId, - subdomain + subdomain, + aiProviderId } = parsedBody.data; // Backward compatibility: merge deprecated siteId into siteIds array @@ -594,6 +604,7 @@ export async function updateSiteResource( networkId: mode === "inference" ? null : undefined, requiresExitNodeConnection: mode !== undefined ? mode === "inference" : undefined, + aiProviderId: aiProviderId, ...sshPamSet }) .where(and(eq(siteResources.siteResourceId, siteResourceId))) diff --git a/server/routers/traefik/traefikConfigProvider.ts b/server/routers/traefik/traefikConfigProvider.ts index 04cb30530..898ac42ee 100644 --- a/server/routers/traefik/traefikConfigProvider.ts +++ b/server/routers/traefik/traefikConfigProvider.ts @@ -20,6 +20,9 @@ export async function traefikConfigProvider( const maintenancePort = config.getRawConfig().server.next_port; const maintenanceHost = config.getRawConfig().server.internal_hostname; const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`; + const aiGatewayUrl = `http://${maintenanceHost}:${ + config.getRawConfig().server.internal_port + }/api/v1/ai-gateway`; const traefikConfig = await getTraefikConfig( currentExitNodeId, @@ -28,7 +31,8 @@ export async function traefikConfigProvider( build != "oss", // generate the login pages on the cloud and and enterprise, config.getRawConfig().traefik.allow_raw_resources, pangolinUIUrl, - pangolinUIUrl + pangolinUIUrl, + aiGatewayUrl ); if (traefikConfig?.http?.middlewares) { From f7689b7a5a812361c38bb7fc08525aa5082bf867 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 2 Aug 2026 10:50:50 -0400 Subject: [PATCH 019/296] Endpoints to update models on the resource --- server/auth/actions.ts | 2 + server/routers/external.ts | 30 ++++ server/routers/integration.ts | 86 +++++++++ .../routers/resource/addAiModelToResource.ts | 161 +++++++++++++++++ server/routers/resource/index.ts | 4 + .../routers/resource/listResourceAiModels.ts | 107 +++++++++++ .../resource/removeAiModelFromResource.ts | 141 +++++++++++++++ .../routers/resource/setResourceAiModels.ts | 155 ++++++++++++++++ .../siteResource/addAiModelToSiteResource.ts | 170 ++++++++++++++++++ server/routers/siteResource/index.ts | 4 + .../siteResource/listSiteResourceAiModels.ts | 115 ++++++++++++ .../removeAiModelFromSiteResource.ts | 140 +++++++++++++++ .../siteResource/setSiteResourceAiModels.ts | 167 +++++++++++++++++ 13 files changed, 1282 insertions(+) create mode 100644 server/routers/resource/addAiModelToResource.ts create mode 100644 server/routers/resource/listResourceAiModels.ts create mode 100644 server/routers/resource/removeAiModelFromResource.ts create mode 100644 server/routers/resource/setResourceAiModels.ts create mode 100644 server/routers/siteResource/addAiModelToSiteResource.ts create mode 100644 server/routers/siteResource/listSiteResourceAiModels.ts create mode 100644 server/routers/siteResource/removeAiModelFromSiteResource.ts create mode 100644 server/routers/siteResource/setSiteResourceAiModels.ts diff --git a/server/auth/actions.ts b/server/auth/actions.ts index 944adbff4..1b4a33e89 100644 --- a/server/auth/actions.ts +++ b/server/auth/actions.ts @@ -50,6 +50,8 @@ export enum ActionsEnum { setResourceUsers = "setResourceUsers", setResourceRoles = "setResourceRoles", listResourceUsers = "listResourceUsers", + listResourceAiModels = "listResourceAiModels", + setResourceAiModels = "setResourceAiModels", // removeRoleSite = "removeRoleSite", // addRoleAction = "addRoleAction", // removeRoleAction = "removeRoleAction", diff --git a/server/routers/external.ts b/server/routers/external.ts index 99bca3a8e..4ebf58cae 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -407,6 +407,13 @@ authenticated.get( siteResource.listSiteResourceClients ); +authenticated.get( + "/site-resource/:siteResourceId/ai-models", + verifySiteResourceAccess, + verifyUserHasAction(ActionsEnum.listResourceAiModels), + siteResource.listSiteResourceAiModels +); + authenticated.post( "/site-resource/:siteResourceId/roles", verifySiteResourceAccess, @@ -417,6 +424,14 @@ authenticated.post( siteResource.setSiteResourceRoles ); +authenticated.post( + "/site-resource/:siteResourceId/ai-models", + verifySiteResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.setSiteResourceAiModels +); + authenticated.post( "/site-resource/:siteResourceId/users", verifySiteResourceAccess, @@ -651,6 +666,13 @@ authenticated.get( resource.listResourceUsers ); +authenticated.get( + "/resource/:resourceId/ai-models", + verifyResourceAccess, + verifyUserHasAction(ActionsEnum.listResourceAiModels), + resource.listResourceAiModels +); + authenticated.get( "/resource/:resourceId", verifyResourceAccess, @@ -854,6 +876,14 @@ authenticated.post( resource.setResourceUsers ); +authenticated.post( + "/resource/:resourceId/ai-models", + verifyResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.setResourceAiModels +); + authenticated.put( "/resource-policy/:resourcePolicyId/access-control", verifyResourcePolicyAccess, diff --git a/server/routers/integration.ts b/server/routers/integration.ts index 384214e4c..41757e7a0 100644 --- a/server/routers/integration.ts +++ b/server/routers/integration.ts @@ -246,6 +246,16 @@ authenticated.get( siteResource.listSiteResourceClients ); +authenticated.get( + [ + "/site-resource/:siteResourceId/ai-models", + "/private-resource/:siteResourceId/ai-models" + ], + verifyApiKeySiteResourceAccess, + verifyApiKeyHasAction(ActionsEnum.listResourceAiModels), + siteResource.listSiteResourceAiModels +); + authenticated.post( [ "/site-resource/:siteResourceId/roles", @@ -298,6 +308,39 @@ authenticated.post( siteResource.removeRoleFromSiteResource ); +authenticated.post( + [ + "/site-resource/:siteResourceId/ai-models", + "/private-resource/:siteResourceId/ai-models" + ], + verifyApiKeySiteResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.setSiteResourceAiModels +); + +authenticated.post( + [ + "/site-resource/:siteResourceId/ai-models/add", + "/private-resource/:siteResourceId/ai-models/add" + ], + verifyApiKeySiteResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.addAiModelToSiteResource +); + +authenticated.post( + [ + "/site-resource/:siteResourceId/ai-models/remove", + "/private-resource/:siteResourceId/ai-models/remove" + ], + verifyApiKeySiteResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.removeAiModelFromSiteResource +); + authenticated.post( [ "/site-resource/:siteResourceId/users/add", @@ -510,6 +553,16 @@ authenticated.get( resource.listResourceUsers ); +authenticated.get( + [ + "/resource/:resourceId/ai-models", + "/public-resource/:resourceId/ai-models" + ], + verifyApiKeyResourceAccess, + verifyApiKeyHasAction(ActionsEnum.listResourceAiModels), + resource.listResourceAiModels +); + authenticated.get( ["/resource/:resourceId", "/public-resource/:resourceId"], verifyApiKeyResourceAccess, @@ -711,6 +764,17 @@ authenticated.post( resource.setResourceRoles ); +authenticated.post( + [ + "/resource/:resourceId/ai-models", + "/public-resource/:resourceId/ai-models" + ], + verifyApiKeyResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.setResourceAiModels +); + authenticated.post( ["/resource/:resourceId/users", "/public-resource/:resourceId/users"], verifyApiKeyResourceAccess, @@ -903,6 +967,28 @@ authenticated.post( resource.removeRoleFromResource ); +authenticated.post( + [ + "/resource/:resourceId/ai-models/add", + "/public-resource/:resourceId/ai-models/add" + ], + verifyApiKeyResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.addAiModelToResource +); + +authenticated.post( + [ + "/resource/:resourceId/ai-models/remove", + "/public-resource/:resourceId/ai-models/remove" + ], + verifyApiKeyResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.removeAiModelFromResource +); + authenticated.post( [ "/resource/:resourceId/users/add", diff --git a/server/routers/resource/addAiModelToResource.ts b/server/routers/resource/addAiModelToResource.ts new file mode 100644 index 000000000..fd33a3e91 --- /dev/null +++ b/server/routers/resource/addAiModelToResource.ts @@ -0,0 +1,161 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, resources, resourceAiModels, aiModels } from "@server/db"; +import { eq, and } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; + +const addAiModelToResourceBodySchema = z.strictObject({ + modelId: z.int().positive() +}); + +const addAiModelToResourceParamsSchema = z.strictObject({ + resourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/resource/{resourceId}/ai-models/add", + description: + "Add a single AI model to a resource's model restriction allow-list.", + tags: [OpenAPITags.PublicResource], + request: { + params: addAiModelToResourceParamsSchema, + body: { + content: { + "application/json": { + schema: addAiModelToResourceBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function addAiModelToResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = addAiModelToResourceBodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { modelId } = parsedBody.data; + + const parsedParams = addAiModelToResourceParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { resourceId } = parsedParams.data; + + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Resource not found") + ); + } + + if (!resource.aiProviderId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Resource has no AI provider linked" + ) + ); + } + + const [model] = await db + .select() + .from(aiModels) + .where( + and( + eq(aiModels.modelId, modelId), + eq(aiModels.providerId, resource.aiProviderId) + ) + ) + .limit(1); + + if (!model) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + "Model not found or does not belong to this resource's AI provider" + ) + ); + } + + const existingEntry = await db + .select() + .from(resourceAiModels) + .where( + and( + eq(resourceAiModels.resourceId, resourceId), + eq(resourceAiModels.modelId, modelId) + ) + ); + + if (existingEntry.length > 0) { + return next( + createHttpError( + HttpCode.CONFLICT, + "Model already assigned to resource" + ) + ); + } + + await db.insert(resourceAiModels).values({ resourceId, modelId }); + + return response(res, { + data: {}, + success: true, + error: false, + message: "Model added to resource successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/resource/index.ts b/server/routers/resource/index.ts index 709bf7340..32992d364 100644 --- a/server/routers/resource/index.ts +++ b/server/routers/resource/index.ts @@ -35,3 +35,7 @@ export * from "./removeEmailFromResourceWhitelist"; export * from "./getStatusHistory"; export * from "./getBatchedStatusHistory"; export * from "./getResourcePolicies"; +export * from "./listResourceAiModels"; +export * from "./setResourceAiModels"; +export * from "./addAiModelToResource"; +export * from "./removeAiModelFromResource"; diff --git a/server/routers/resource/listResourceAiModels.ts b/server/routers/resource/listResourceAiModels.ts new file mode 100644 index 000000000..f6875faff --- /dev/null +++ b/server/routers/resource/listResourceAiModels.ts @@ -0,0 +1,107 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, resources, resourceAiModels, aiModels } from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; + +const listResourceAiModelsParamsSchema = z.strictObject({ + resourceId: z.coerce.number().int().positive() +}); + +async function query(resourceId: number) { + return await db + .select({ + modelId: aiModels.modelId, + modelKey: aiModels.modelKey, + name: aiModels.name, + enabled: aiModels.enabled + }) + .from(resourceAiModels) + .innerJoin(aiModels, eq(resourceAiModels.modelId, aiModels.modelId)) + .where(eq(resourceAiModels.resourceId, resourceId)); +} + +export type ListResourceAiModelsResponse = { + models: NonNullable>>; +}; + +registry.registerPath({ + method: "get", + path: "/resource/{resourceId}/ai-models", + description: + "List the AI models a resource is restricted to. An empty list means the resource is not restricted and every enabled model on its linked AI provider is allowed.", + tags: [OpenAPITags.PublicResource], + request: { + params: listResourceAiModelsParamsSchema + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function listResourceAiModels( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = listResourceAiModelsParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { resourceId } = parsedParams.data; + + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Resource not found") + ); + } + + const models = await query(resourceId); + + return response(res, { + data: { models }, + success: true, + error: false, + message: "Resource AI models retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/resource/removeAiModelFromResource.ts b/server/routers/resource/removeAiModelFromResource.ts new file mode 100644 index 000000000..47445bbf5 --- /dev/null +++ b/server/routers/resource/removeAiModelFromResource.ts @@ -0,0 +1,141 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, resources, resourceAiModels } from "@server/db"; +import { eq, and } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; + +const removeAiModelFromResourceBodySchema = z.strictObject({ + modelId: z.int().positive() +}); + +const removeAiModelFromResourceParamsSchema = z.strictObject({ + resourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/resource/{resourceId}/ai-models/remove", + description: + "Remove a single AI model from a resource's model restriction allow-list.", + tags: [OpenAPITags.PublicResource], + request: { + params: removeAiModelFromResourceParamsSchema, + body: { + content: { + "application/json": { + schema: removeAiModelFromResourceBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function removeAiModelFromResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = removeAiModelFromResourceBodySchema.safeParse( + req.body + ); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { modelId } = parsedBody.data; + + const parsedParams = removeAiModelFromResourceParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { resourceId } = parsedParams.data; + + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Resource not found") + ); + } + + const existingEntry = await db + .select() + .from(resourceAiModels) + .where( + and( + eq(resourceAiModels.resourceId, resourceId), + eq(resourceAiModels.modelId, modelId) + ) + ); + + if (existingEntry.length === 0) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + "Model not found in resource's restriction list" + ) + ); + } + + await db + .delete(resourceAiModels) + .where( + and( + eq(resourceAiModels.resourceId, resourceId), + eq(resourceAiModels.modelId, modelId) + ) + ); + + return response(res, { + data: {}, + success: true, + error: false, + message: "Model removed from resource successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/resource/setResourceAiModels.ts b/server/routers/resource/setResourceAiModels.ts new file mode 100644 index 000000000..2ef56f4fc --- /dev/null +++ b/server/routers/resource/setResourceAiModels.ts @@ -0,0 +1,155 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, resources, resourceAiModels, aiModels } from "@server/db"; +import { eq, and, inArray } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; + +const setResourceAiModelsBodySchema = z.strictObject({ + modelIds: z.array(z.int().positive()) +}); + +const setResourceAiModelsParamsSchema = z.strictObject({ + resourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/resource/{resourceId}/ai-models", + description: + "Set the AI models a resource is restricted to. This replaces all existing restrictions. Pass an empty array to remove the restriction (allow every enabled model on the linked provider).", + tags: [OpenAPITags.PublicResource], + request: { + params: setResourceAiModelsParamsSchema, + body: { + content: { + "application/json": { + schema: setResourceAiModelsBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function setResourceAiModels( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = setResourceAiModelsBodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { modelIds } = parsedBody.data; + + const parsedParams = setResourceAiModelsParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { resourceId } = parsedParams.data; + + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Resource not found") + ); + } + + if (modelIds.length > 0) { + if (!resource.aiProviderId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Resource has no AI provider linked" + ) + ); + } + + const validModels = await db + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .where( + and( + inArray(aiModels.modelId, modelIds), + eq(aiModels.providerId, resource.aiProviderId) + ) + ); + + if (validModels.length !== new Set(modelIds).size) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "One or more model IDs do not exist or do not belong to this resource's AI provider" + ) + ); + } + } + + await db.transaction(async (trx) => { + await trx + .delete(resourceAiModels) + .where(eq(resourceAiModels.resourceId, resourceId)); + + if (modelIds.length > 0) { + await trx + .insert(resourceAiModels) + .values( + modelIds.map((modelId) => ({ resourceId, modelId })) + ); + } + }); + + return response(res, { + data: {}, + success: true, + error: false, + message: "AI models set for resource successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/siteResource/addAiModelToSiteResource.ts b/server/routers/siteResource/addAiModelToSiteResource.ts new file mode 100644 index 000000000..2e55f100b --- /dev/null +++ b/server/routers/siteResource/addAiModelToSiteResource.ts @@ -0,0 +1,170 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { + db, + siteResources, + siteResourceAiModels, + aiModels +} from "@server/db"; +import { eq, and } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; + +const addAiModelToSiteResourceBodySchema = z.strictObject({ + modelId: z.int().positive() +}); + +const addAiModelToSiteResourceParamsSchema = z.strictObject({ + siteResourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/site-resource/{siteResourceId}/ai-models/add", + description: + "Add a single AI model to a site resource's model restriction allow-list.", + tags: [OpenAPITags.PrivateResource], + request: { + params: addAiModelToSiteResourceParamsSchema, + body: { + content: { + "application/json": { + schema: addAiModelToSiteResourceBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function addAiModelToSiteResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = addAiModelToSiteResourceBodySchema.safeParse( + req.body + ); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { modelId } = parsedBody.data; + + const parsedParams = addAiModelToSiteResourceParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteResourceId } = parsedParams.data; + + const [siteResource] = await db + .select() + .from(siteResources) + .where(eq(siteResources.siteResourceId, siteResourceId)) + .limit(1); + + if (!siteResource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Site resource not found") + ); + } + + if (!siteResource.aiProviderId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Site resource has no AI provider linked" + ) + ); + } + + const [model] = await db + .select() + .from(aiModels) + .where( + and( + eq(aiModels.modelId, modelId), + eq(aiModels.providerId, siteResource.aiProviderId) + ) + ) + .limit(1); + + if (!model) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + "Model not found or does not belong to this site resource's AI provider" + ) + ); + } + + const existingEntry = await db + .select() + .from(siteResourceAiModels) + .where( + and( + eq(siteResourceAiModels.siteResourceId, siteResourceId), + eq(siteResourceAiModels.modelId, modelId) + ) + ); + + if (existingEntry.length > 0) { + return next( + createHttpError( + HttpCode.CONFLICT, + "Model already assigned to site resource" + ) + ); + } + + await db + .insert(siteResourceAiModels) + .values({ siteResourceId, modelId }); + + return response(res, { + data: {}, + success: true, + error: false, + message: "Model added to site resource successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/siteResource/index.ts b/server/routers/siteResource/index.ts index 5c09d3883..2acaf1e33 100644 --- a/server/routers/siteResource/index.ts +++ b/server/routers/siteResource/index.ts @@ -17,3 +17,7 @@ export * from "./setSiteResourceClients"; export * from "./addClientToSiteResource"; export * from "./batchAddClientToSiteResources"; export * from "./removeClientFromSiteResource"; +export * from "./listSiteResourceAiModels"; +export * from "./setSiteResourceAiModels"; +export * from "./addAiModelToSiteResource"; +export * from "./removeAiModelFromSiteResource"; diff --git a/server/routers/siteResource/listSiteResourceAiModels.ts b/server/routers/siteResource/listSiteResourceAiModels.ts new file mode 100644 index 000000000..26dea7cd0 --- /dev/null +++ b/server/routers/siteResource/listSiteResourceAiModels.ts @@ -0,0 +1,115 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { + db, + siteResources, + siteResourceAiModels, + aiModels +} from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; + +const listSiteResourceAiModelsParamsSchema = z.strictObject({ + siteResourceId: z.coerce.number().int().positive() +}); + +async function query(siteResourceId: number) { + return await db + .select({ + modelId: aiModels.modelId, + modelKey: aiModels.modelKey, + name: aiModels.name, + enabled: aiModels.enabled + }) + .from(siteResourceAiModels) + .innerJoin( + aiModels, + eq(siteResourceAiModels.modelId, aiModels.modelId) + ) + .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); +} + +export type ListSiteResourceAiModelsResponse = { + models: NonNullable>>; +}; + +registry.registerPath({ + method: "get", + path: "/site-resource/{siteResourceId}/ai-models", + description: + "List the AI models a site resource is restricted to. An empty list means the site resource is not restricted and every enabled model on its linked AI provider is allowed.", + tags: [OpenAPITags.PrivateResource], + request: { + params: listSiteResourceAiModelsParamsSchema + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function listSiteResourceAiModels( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = listSiteResourceAiModelsParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteResourceId } = parsedParams.data; + + const [siteResource] = await db + .select() + .from(siteResources) + .where(eq(siteResources.siteResourceId, siteResourceId)) + .limit(1); + + if (!siteResource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Site resource not found") + ); + } + + const models = await query(siteResourceId); + + return response(res, { + data: { models }, + success: true, + error: false, + message: "Site resource AI models retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/siteResource/removeAiModelFromSiteResource.ts b/server/routers/siteResource/removeAiModelFromSiteResource.ts new file mode 100644 index 000000000..5af2f4d87 --- /dev/null +++ b/server/routers/siteResource/removeAiModelFromSiteResource.ts @@ -0,0 +1,140 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, siteResources, siteResourceAiModels } from "@server/db"; +import { eq, and } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; + +const removeAiModelFromSiteResourceBodySchema = z.strictObject({ + modelId: z.int().positive() +}); + +const removeAiModelFromSiteResourceParamsSchema = z.strictObject({ + siteResourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/site-resource/{siteResourceId}/ai-models/remove", + description: + "Remove a single AI model from a site resource's model restriction allow-list.", + tags: [OpenAPITags.PrivateResource], + request: { + params: removeAiModelFromSiteResourceParamsSchema, + body: { + content: { + "application/json": { + schema: removeAiModelFromSiteResourceBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function removeAiModelFromSiteResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = removeAiModelFromSiteResourceBodySchema.safeParse( + req.body + ); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { modelId } = parsedBody.data; + + const parsedParams = + removeAiModelFromSiteResourceParamsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteResourceId } = parsedParams.data; + + const [siteResource] = await db + .select() + .from(siteResources) + .where(eq(siteResources.siteResourceId, siteResourceId)) + .limit(1); + + if (!siteResource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Site resource not found") + ); + } + + const existingEntry = await db + .select() + .from(siteResourceAiModels) + .where( + and( + eq(siteResourceAiModels.siteResourceId, siteResourceId), + eq(siteResourceAiModels.modelId, modelId) + ) + ); + + if (existingEntry.length === 0) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + "Model not found in site resource's restriction list" + ) + ); + } + + await db + .delete(siteResourceAiModels) + .where( + and( + eq(siteResourceAiModels.siteResourceId, siteResourceId), + eq(siteResourceAiModels.modelId, modelId) + ) + ); + + return response(res, { + data: {}, + success: true, + error: false, + message: "Model removed from site resource successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/siteResource/setSiteResourceAiModels.ts b/server/routers/siteResource/setSiteResourceAiModels.ts new file mode 100644 index 000000000..1366187af --- /dev/null +++ b/server/routers/siteResource/setSiteResourceAiModels.ts @@ -0,0 +1,167 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { + db, + siteResources, + siteResourceAiModels, + aiModels +} from "@server/db"; +import { eq, and, inArray } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; + +const setSiteResourceAiModelsBodySchema = z.strictObject({ + modelIds: z.array(z.int().positive()) +}); + +const setSiteResourceAiModelsParamsSchema = z.strictObject({ + siteResourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/site-resource/{siteResourceId}/ai-models", + description: + "Set the AI models a site resource is restricted to. This replaces all existing restrictions. Pass an empty array to remove the restriction (allow every enabled model on the linked provider).", + tags: [OpenAPITags.PrivateResource], + request: { + params: setSiteResourceAiModelsParamsSchema, + body: { + content: { + "application/json": { + schema: setSiteResourceAiModelsBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function setSiteResourceAiModels( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = setSiteResourceAiModelsBodySchema.safeParse( + req.body + ); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { modelIds } = parsedBody.data; + + const parsedParams = setSiteResourceAiModelsParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteResourceId } = parsedParams.data; + + const [siteResource] = await db + .select() + .from(siteResources) + .where(eq(siteResources.siteResourceId, siteResourceId)) + .limit(1); + + if (!siteResource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Site resource not found") + ); + } + + if (modelIds.length > 0) { + if (!siteResource.aiProviderId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Site resource has no AI provider linked" + ) + ); + } + + const validModels = await db + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .where( + and( + inArray(aiModels.modelId, modelIds), + eq(aiModels.providerId, siteResource.aiProviderId) + ) + ); + + if (validModels.length !== new Set(modelIds).size) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "One or more model IDs do not exist or do not belong to this site resource's AI provider" + ) + ); + } + } + + await db.transaction(async (trx) => { + await trx + .delete(siteResourceAiModels) + .where( + eq(siteResourceAiModels.siteResourceId, siteResourceId) + ); + + if (modelIds.length > 0) { + await trx + .insert(siteResourceAiModels) + .values( + modelIds.map((modelId) => ({ + siteResourceId, + modelId + })) + ); + } + }); + + return response(res, { + data: {}, + success: true, + error: false, + message: "AI models set for site resource successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} From 1a644b131df2dd7ca8335a1dbcc0a521698db007 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 2 Aug 2026 12:12:48 -0400 Subject: [PATCH 020/296] Pull the session cookie properly --- server/routers/aiGateway/chatCompletions.ts | 129 +++++++++++++++++++- server/routers/badger/verifySession.ts | 4 +- 2 files changed, 129 insertions(+), 4 deletions(-) diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index cddcf8404..7a4ba3a84 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -8,7 +8,8 @@ import { resourceAiModels, resources, siteResourceAiModels, - siteResources + siteResources, + users } from "@server/db"; import config from "@server/lib/config"; import { decrypt } from "@server/lib/crypto"; @@ -18,18 +19,121 @@ import { AiProviderType, resolveAiProviderConfig } from "@server/lib/aiProviderDefaults"; +import { verifyResourceAccessToken } from "@server/auth/verifyResourceAccessToken"; +import { + SESSION_COOKIE_NAME, + validateSessionToken +} from "@server/auth/sessions/app"; +import { getUserOrgRoles } from "@server/lib/userOrgRoles"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; type ResolvedTarget = { + resourceId: number | null; + orgId: string | null; provider: AiProvider; // null = no restriction; every enabled model on the provider is allowed allowedModelIds: number[] | null; }; +// Fallback for clients that hit the endpoint directly (e.g. an AI tool's +// "API key" field) instead of going through a browser session - badger +// forwards whatever Authorization header the client sent untouched in that +// case. This prefix lets us tell "this bearer value is a Pangolin resource +// access token" apart from an arbitrary/opaque API key a user might paste +// in, without guessing based on format alone. +const USER_TOKEN_PREFIX = "pu_"; + +export type RequestUser = { + userId: string; + username: string; + email: string | null; + name: string | null; + role: string | null; +}; + +async function buildRequestUser( + userId: string, + orgId: string | null +): Promise { + const [user] = await db + .select() + .from(users) + .where(eq(users.userId, userId)) + .limit(1); + + if (!user) { + return null; + } + + const orgRoles = orgId ? await getUserOrgRoles(user.userId, orgId) : []; + + return { + userId: user.userId, + username: user.username, + email: user.email, + name: user.name, + role: orgRoles.map((r) => r.roleName).join(", ") || null + }; +} + +async function resolveRequestUser( + req: Request, + resourceId: number | null, + orgId: string | null +): Promise { + // Public resources behind badger: badger passes the resource session + // cookie through to the backend (same mechanism the browser gateway, + // e.g. the SSH page, relies on), so we can validate it exactly like + // verifySessionUserMiddleware does for the dashboard. + const sessionToken = req.cookies?.[SESSION_COOKIE_NAME]; + if (sessionToken) { + const { session, user } = await validateSessionToken(sessionToken); + if (session && user) { + return buildRequestUser(user.userId, orgId); + } + } + + // User devices hitting the endpoint directly (no browser session to + // forward) fall back to a Pangolin resource access token passed as the + // client's "API key". + const authHeader = req.headers["authorization"]; + if (typeof authHeader !== "string" || !resourceId) { + return null; + } + + const bearer = authHeader.match(/^Bearer\s+(.+)$/i)?.[1]; + if (!bearer || !bearer.startsWith(USER_TOKEN_PREFIX)) { + return null; + } + + const [accessTokenId, accessToken] = bearer + .slice(USER_TOKEN_PREFIX.length) + .split("."); + if (!accessTokenId || !accessToken) { + return null; + } + + const { valid, tokenItem } = await verifyResourceAccessToken({ + accessToken, + accessTokenId, + resourceId + }); + + if (!valid || !tokenItem?.userId) { + return null; + } + + return buildRequestUser(tokenItem.userId, tokenItem.orgId); +} + async function resolveTarget(host: string): Promise { const [resourceRow] = await db - .select({ resourceId: resources.resourceId, provider: aiProviders }) + .select({ + resourceId: resources.resourceId, + orgId: resources.orgId, + provider: aiProviders + }) .from(resources) .innerJoin( aiProviders, @@ -52,6 +156,8 @@ async function resolveTarget(host: string): Promise { .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)); return { + resourceId: resourceRow.resourceId, + orgId: resourceRow.orgId, provider: resourceRow.provider, allowedModelIds: restrictions.length ? restrictions.map((r) => r.modelId) @@ -62,6 +168,7 @@ async function resolveTarget(host: string): Promise { const [siteResourceRow] = await db .select({ siteResourceId: siteResources.siteResourceId, + orgId: siteResources.orgId, provider: aiProviders }) .from(siteResources) @@ -91,6 +198,11 @@ async function resolveTarget(host: string): Promise { ); return { + // siteResources have no per-user auth/policy stack today (see + // the routing comment in getTraefikConfig.ts), so there's no + // resource access token scope to validate a user token against. + resourceId: null, + orgId: siteResourceRow.orgId, provider: siteResourceRow.provider, allowedModelIds: restrictions.length ? restrictions.map((r) => r.modelId) @@ -128,7 +240,18 @@ export async function chatCompletions(req: Request, res: Response): Promise }); } - const { provider, allowedModelIds } = target; + const { provider, allowedModelIds, resourceId, orgId } = target; + + // Best-effort identity resolution - not yet enforced, but lets us + // start making per-user access decisions (e.g. model/role-based + // restrictions) without another round of plumbing later. + const requestUser = await resolveRequestUser(req, resourceId, orgId); + if (requestUser) { + logger.debug( + `AI gateway request from user ${requestUser.userId} (${requestUser.username})` + ); + } + const requestedModel = typeof req.body?.model === "string" ? req.body.model : undefined; diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index 99e0d9812..b0aed6e30 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -222,7 +222,9 @@ export async function verifyResourceSession( } const { blockAccess, mode } = resource; - const dontStripSession = ["ssh", "rdp", "vnc"].includes(mode); + const dontStripSession = ["ssh", "rdp", "vnc", "inference"].includes( + mode + ); if (blockAccess) { logger.debug("Resource blocked", host); From b0edc6302e0d0fb0751d9a0e53abc9c13f606a62 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Mon, 3 Aug 2026 10:36:42 -0400 Subject: [PATCH 021/296] providers table, create, and edit first pass --- messages/en-US.json | 77 +++ .../[providerId]/configuration/page.tsx | 426 +++++++++++++++ .../[providerId]/general/page.tsx | 180 ++++++ .../ai-providers/[providerId]/layout.tsx | 90 +++ .../ai-providers/[providerId]/page.tsx | 10 + .../settings/ai-providers/create/layout.tsx | 13 + .../settings/ai-providers/create/page.tsx | 514 ++++++++++++++++++ .../[orgId]/settings/ai-providers/page.tsx | 71 +++ src/app/navigation.tsx | 21 + src/components/AiProviderTypeSelect.tsx | 147 +++++ src/components/AiProvidersTable.tsx | 317 +++++++++++ src/components/HorizontalTabs.tsx | 1 + src/contexts/aiProviderContext.ts | 13 + src/hooks/useAiProviderContext.ts | 12 + src/lib/aiProviderFormSchema.ts | 202 +++++++ src/providers/AiProviderProvider.tsx | 47 ++ 16 files changed, 2141 insertions(+) create mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx create mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx create mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx create mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/page.tsx create mode 100644 src/app/[orgId]/settings/ai-providers/create/layout.tsx create mode 100644 src/app/[orgId]/settings/ai-providers/create/page.tsx create mode 100644 src/app/[orgId]/settings/ai-providers/page.tsx create mode 100644 src/components/AiProviderTypeSelect.tsx create mode 100644 src/components/AiProvidersTable.tsx create mode 100644 src/contexts/aiProviderContext.ts create mode 100644 src/hooks/useAiProviderContext.ts create mode 100644 src/lib/aiProviderFormSchema.ts create mode 100644 src/providers/AiProviderProvider.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 94dd49f96..4febc876a 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1632,6 +1632,83 @@ "sidebarInvitations": "Invitations", "sidebarRoles": "Roles", "sidebarShareableLinks": "Shareable Links", + "sidebarAi": "AI", + "sidebarAiProviders": "Providers", + "commandAiProviders": "AI Providers", + "aiProvidersTitle": "AI Providers", + "aiProvidersDescription": "Connect model providers for AI workloads in this organization", + "aiProvidersAdd": "Add Provider", + "aiProvidersSearch": "Search providers...", + "aiProvidersEmpty": "No AI providers yet", + "aiProviderCreate": "Create AI Provider", + "aiProviderCreateDescription": "Add a model provider for this organization", + "aiProviderSeeAll": "See All Providers", + "aiProviderSetting": "Provider Settings for {providerName}", + "aiProviderSettingDescription": "Configure this AI provider", + "aiProviderGeneral": "General", + "aiProviderGeneralDescription": "Basic settings for this provider", + "aiProviderConfiguration": "Configuration", + "aiProviderConfigurationDescription": "Upstream URL, routing, authentication, and TLS settings", + "aiProviderType": "Provider Type", + "aiProviderTypeSearch": "Search providers...", + "aiProviderTypeNotFound": "No provider type found", + "aiProviderTypeOpenai": "OpenAI", + "aiProviderTypeAnthropic": "Anthropic", + "aiProviderTypeGoogleGemini": "Google Gemini", + "aiProviderTypeVertexAi": "Vertex AI", + "aiProviderTypeBedrock": "Amazon Bedrock", + "aiProviderTypeMicrosoftFoundry": "Microsoft Foundry", + "aiProviderTypeOpenRouter": "OpenRouter", + "aiProviderTypeVercelAiGateway": "Vercel AI Gateway", + "aiProviderTypeCustom": "Custom", + "aiProviderTypeOpenaiDescription": "OpenAI API with default upstream URL", + "aiProviderTypeAnthropicDescription": "Anthropic API with default upstream URL", + "aiProviderTypeGoogleGeminiDescription": "Google Gemini OpenAI-compatible endpoint", + "aiProviderTypeVertexAiDescription": "Google Vertex AI; upstream URL required", + "aiProviderTypeBedrockDescription": "Amazon Bedrock Runtime", + "aiProviderTypeMicrosoftFoundryDescription": "Microsoft Foundry; upstream URL required", + "aiProviderTypeOpenRouterDescription": "OpenRouter API", + "aiProviderTypeVercelAiGatewayDescription": "Vercel AI Gateway", + "aiProviderTypeCustomDescription": "Bring your own OpenAI-compatible endpoint or route via Pangolin targets", + "aiProviderUpstreamUrl": "Upstream URL", + "aiProviderUpstreamUrlDescription": "Base URL for the provider API", + "aiProviderUpstreamUrlOptionalDescription": "Leave blank to use the default upstream URL for this provider", + "aiProviderEffectiveUpstreamUrl": "Effective Upstream URL", + "aiProviderApiKey": "API Key", + "aiProviderApiKeyDescription": "Stored encrypted. Leave blank on edit to keep the existing key.", + "aiProviderApiKeyLastChars": "API Key", + "aiProviderAuthType": "Auth Type", + "aiProviderAuthTypeBearer": "Bearer", + "aiProviderAuthTypeDescription": "How the upstream API authenticates requests", + "aiProviderRoutingMode": "Routing Mode", + "aiProviderRoutingModeDescription": "Send traffic to an upstream URL or to Pangolin HTTPS targets", + "aiProviderRoutingModeUrl": "Upstream URL", + "aiProviderRoutingModeUrlDescription": "Call a public or private API base URL", + "aiProviderRoutingModeTarget": "Pangolin Targets", + "aiProviderRoutingModeTargetDescription": "Route through HTTPS targets on your sites", + "aiProviderRoutingModeTargetNote": "Target configuration will be available in a later update. You can still create this provider now.", + "aiProviderSkipTlsVerification": "Skip TLS Verification", + "aiProviderSkipTlsVerificationDescription": "Disable TLS certificate verification for the upstream connection", + "aiProviderBudget": "Budget", + "aiProviderBudgetDescription": "Optional spending or token budget for this provider", + "aiProviderBudgetAmount": "Budget Amount", + "aiProviderBudgetUnit": "Budget Unit", + "aiProviderBudgetUnitUsd": "USD", + "aiProviderBudgetUnitTokens": "Tokens", + "aiProviderEnabled": "Enabled", + "aiProviderEnabledDescription": "Disable to stop using this provider without deleting it", + "aiProviderErrorCreate": "Failed to create AI provider", + "aiProviderErrorUpdate": "Failed to update AI provider", + "aiProviderErrorDelete": "Failed to delete AI provider", + "aiProviderErrorLoad": "Failed to load AI provider", + "aiProviderCreated": "AI provider created", + "aiProviderUpdated": "AI provider updated", + "aiProviderDeleted": "AI provider deleted", + "aiProviderDelete": "Delete Provider", + "aiProviderDeleteConfirm": "Delete Provider", + "aiProviderQuestionRemove": "Are you sure you want to delete this AI provider?", + "aiProviderMessageRemove": "This will permanently delete the provider and its models and targets. This cannot be undone.", + "aiProviderErrorNoUpdate": "AI provider is not available to update", "sidebarApiKeys": "API Keys", "sidebarProvisioning": "Provisioning", "sidebarSettings": "Settings", diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx new file mode 100644 index 000000000..03a9ab48f --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx @@ -0,0 +1,426 @@ +"use client"; + +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { StrategySelect } from "@app/components/StrategySelect"; +import { SwitchInput } from "@app/components/SwitchInput"; +import { Alert, AlertDescription, AlertTitle } from "@app/components/ui/alert"; +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { Input } from "@app/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue +} from "@app/components/ui/select"; +import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { + aiProviderFormSchema, + showsUpstreamUrlField, + toAiProviderConfigurationPayload, + upstreamUrlRequired, + type AiProviderFormValues +} from "@app/lib/aiProviderFormSchema"; +import { zodResolver } from "@hookform/resolvers/zod"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; +import type { AxiosResponse } from "axios"; +import { InfoIcon } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; +import { useForm } from "react-hook-form"; + +export default function AiProviderConfigurationPage() { + const { provider, updateProvider } = useAiProviderContext(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const router = useRouter(); + const t = useTranslations(); + const [saveLoading, setSaveLoading] = useState(false); + + const form = useForm({ + resolver: zodResolver(aiProviderFormSchema), + defaultValues: { + name: provider.name, + type: provider.type as AiProviderType, + upstreamUrl: provider.upstreamUrl ?? "", + apiKey: "", + authType: (provider.authType as "bearer" | null) ?? "bearer", + routingMode: (provider.routingMode as "url" | "target") ?? "url", + skipTlsVerification: provider.skipTlsVerification, + budgetAmount: provider.budgetAmount, + budgetUnit: provider.budgetUnit as "usd" | "tokens" | null, + enabled: provider.enabled + } + }); + + const providerType = form.watch("type"); + const routingMode = form.watch("routingMode"); + const showUpstream = showsUpstreamUrlField(providerType, routingMode); + const requireUpstream = upstreamUrlRequired(providerType, routingMode); + const showRoutingMode = providerType === "custom"; + const showAuthType = + providerType === "custom" && (routingMode ?? "url") === "url"; + const showTargetNote = + providerType === "custom" && routingMode === "target"; + + async function onSubmit(values: AiProviderFormValues) { + setSaveLoading(true); + try { + const res = await api.post< + AxiosResponse + >( + `/ai-provider/${provider.providerId}`, + toAiProviderConfigurationPayload({ + ...values, + type: provider.type as AiProviderType + }) + ); + const updated = res.data.data.provider; + updateProvider(updated); + form.reset({ + name: updated.name, + type: updated.type as AiProviderType, + upstreamUrl: updated.upstreamUrl ?? "", + apiKey: "", + authType: (updated.authType as "bearer" | null) ?? "bearer", + routingMode: (updated.routingMode as "url" | "target") ?? "url", + skipTlsVerification: updated.skipTlsVerification, + budgetAmount: updated.budgetAmount, + budgetUnit: updated.budgetUnit as "usd" | "tokens" | null, + enabled: updated.enabled + }); + toast({ + title: t("success"), + description: t("aiProviderUpdated") + }); + router.refresh(); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiProviderErrorUpdate"), + description: formatAxiosError(e, t("aiProviderErrorUpdate")) + }); + } finally { + setSaveLoading(false); + } + } + + return ( + + + + + {t("aiProviderConfiguration")} + + + {t("aiProviderConfigurationDescription")} + + + + + +
+ + + {showRoutingMode && ( + + ( + + + {t( + "aiProviderRoutingMode" + )} + + + { + field.onChange( + value + ); + if ( + value === + "target" + ) { + form.setValue( + "upstreamUrl", + "" + ); + } + }} + /> + + + {t( + "aiProviderRoutingModeDescription" + )} + + + + )} + /> + + )} + + {showTargetNote && ( + + + + + {t( + "aiProviderRoutingModeTarget" + )} + + + {t( + "aiProviderRoutingModeTargetNote" + )} + + + + )} + + {showUpstream && ( + + ( + + + {t( + "aiProviderUpstreamUrl" + )} + {requireUpstream + ? "" + : " (optional)"} + + + + + + {requireUpstream + ? t( + "aiProviderUpstreamUrlDescription" + ) + : t( + "aiProviderUpstreamUrlOptionalDescription" + )} + + {provider.effectiveUpstreamUrl && ( + + {t( + "aiProviderEffectiveUpstreamUrl" + )} + {": "} + + { + provider.effectiveUpstreamUrl + } + + + )} + + + )} + /> + + )} + + {showAuthType && ( + + ( + + + {t( + "aiProviderAuthType" + )} + + + + {t( + "aiProviderAuthTypeDescription" + )} + + + + )} + /> + + )} + + + ( + + + {t("aiProviderApiKey")} + + + + + + {provider.apiKeyLastChars + ? `••••${provider.apiKeyLastChars}. ${t("aiProviderApiKeyDescription")}` + : t( + "aiProviderApiKeyDescription" + )} + + + + )} + /> + + + + ( + + + + + + + )} + /> + + +
+ + + + + + + + + ); +} diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx new file mode 100644 index 000000000..99db2eda8 --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx @@ -0,0 +1,180 @@ +"use client"; + +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { SwitchInput } from "@app/components/SwitchInput"; +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { Input } from "@app/components/ui/input"; +import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { zodResolver } from "@hookform/resolvers/zod"; +import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; +import type { AxiosResponse } from "axios"; +import { useTranslations } from "next-intl"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; + +const generalSchema = z.object({ + name: z.string().trim().min(1), + enabled: z.boolean() +}); + +type GeneralFormValues = z.infer; + +export default function AiProviderGeneralPage() { + const { provider, updateProvider } = useAiProviderContext(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const router = useRouter(); + const t = useTranslations(); + const [saveLoading, setSaveLoading] = useState(false); + + const form = useForm({ + resolver: zodResolver(generalSchema), + defaultValues: { + name: provider.name, + enabled: provider.enabled + } + }); + + async function onSubmit(values: GeneralFormValues) { + setSaveLoading(true); + try { + const res = await api.post< + AxiosResponse + >(`/ai-provider/${provider.providerId}`, { + name: values.name.trim(), + enabled: values.enabled + }); + const updated = res.data.data.provider; + updateProvider(updated); + form.reset({ + name: updated.name, + enabled: updated.enabled + }); + toast({ + title: t("success"), + description: t("aiProviderUpdated") + }); + router.refresh(); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiProviderErrorUpdate"), + description: formatAxiosError(e, t("aiProviderErrorUpdate")) + }); + } finally { + setSaveLoading(false); + } + } + + return ( + + + + + {t("aiProviderGeneral")} + + + {t("aiProviderGeneralDescription")} + + + + + +
+ + + + ( + + + + + + + )} + /> + + + + ( + + + {t("name")} + + + + + + + )} + /> + + +
+ + + + + + + + + ); +} diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx new file mode 100644 index 000000000..b734f6084 --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx @@ -0,0 +1,90 @@ +import { HorizontalTabs } from "@app/components/HorizontalTabs"; +import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; +import { internal } from "@app/lib/api"; +import { authCookieHeader } from "@app/lib/api/cookies"; +import OrgProvider from "@app/providers/OrgProvider"; +import AiProviderProvider from "@app/providers/AiProviderProvider"; +import type { GetOrgResponse } from "@server/routers/org"; +import type { GetAiProviderResponse } from "@server/routers/aiProvider/types"; +import type { AxiosResponse } from "axios"; +import type { Metadata } from "next"; +import { getTranslations } from "next-intl/server"; +import { redirect } from "next/navigation"; +import { cache } from "react"; + +export const metadata: Metadata = { + title: "AI Provider" +}; + +export const dynamic = "force-dynamic"; + +type Props = { + children: React.ReactNode; + params: Promise<{ orgId: string; providerId: string }>; +}; + +export default async function AiProviderLayout({ children, params }: Props) { + const { orgId, providerId } = await params; + const t = await getTranslations(); + + let provider = null; + try { + const res = await internal.get>( + `/ai-provider/${providerId}`, + await authCookieHeader() + ); + provider = res.data.data.provider; + } catch { + redirect(`/${orgId}/settings/ai-providers`); + } + + if (!provider || provider.orgId !== orgId) { + redirect(`/${orgId}/settings/ai-providers`); + } + + let org = null; + try { + const getOrg = cache(async () => + internal.get>( + `/org/${orgId}`, + await authCookieHeader() + ) + ); + const res = await getOrg(); + org = res.data.data; + } catch { + redirect(`/${orgId}/settings/ai-providers`); + } + + if (!org) { + redirect(`/${orgId}/settings/ai-providers`); + } + + const navItems = [ + { + title: t("general"), + href: "/{orgId}/settings/ai-providers/{providerId}/general" + }, + { + title: t("aiProviderConfiguration"), + href: "/{orgId}/settings/ai-providers/{providerId}/configuration" + } + ]; + + return ( + <> + + + + + {children} + + + + ); +} diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/page.tsx new file mode 100644 index 000000000..2daca1227 --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/page.tsx @@ -0,0 +1,10 @@ +import { redirect } from "next/navigation"; + +type Props = { + params: Promise<{ orgId: string; providerId: string }>; +}; + +export default async function AiProviderPage({ params }: Props) { + const { orgId, providerId } = await params; + redirect(`/${orgId}/settings/ai-providers/${providerId}/general`); +} diff --git a/src/app/[orgId]/settings/ai-providers/create/layout.tsx b/src/app/[orgId]/settings/ai-providers/create/layout.tsx new file mode 100644 index 000000000..f33dcd80d --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/create/layout.tsx @@ -0,0 +1,13 @@ +import type { Metadata } from "next"; + +export const metadata: Metadata = { + title: "Create AI Provider" +}; + +export default function CreateAiProviderLayout({ + children +}: { + children: React.ReactNode; +}) { + return children; +} diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx new file mode 100644 index 000000000..9a9258d8d --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -0,0 +1,514 @@ +"use client"; + +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import HeaderTitle from "@app/components/SettingsSectionTitle"; +import { AiProviderTypeSelect } from "@app/components/AiProviderTypeSelect"; +import { StrategySelect } from "@app/components/StrategySelect"; +import { SwitchInput } from "@app/components/SwitchInput"; +import { Alert, AlertDescription, AlertTitle } from "@app/components/ui/alert"; +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { Input } from "@app/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue +} from "@app/components/ui/select"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { + aiProviderFormSchema, + emptyUpstreamForType, + showsUpstreamUrlField, + toAiProviderCreatePayload, + upstreamUrlRequired, + type AiProviderFormValues +} from "@app/lib/aiProviderFormSchema"; +import { zodResolver } from "@hookform/resolvers/zod"; +import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; +import type { AxiosResponse } from "axios"; +import { InfoIcon } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useParams, useRouter } from "next/navigation"; +import { useState } from "react"; +import { useForm } from "react-hook-form"; + +export default function CreateAiProviderPage() { + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const params = useParams(); + const orgId = params.orgId as string; + const router = useRouter(); + const t = useTranslations(); + const [loading, setLoading] = useState(false); + + const form = useForm({ + resolver: zodResolver(aiProviderFormSchema), + defaultValues: { + name: "", + type: "openai", + upstreamUrl: emptyUpstreamForType("openai"), + apiKey: "", + authType: "bearer", + routingMode: "url", + skipTlsVerification: false, + budgetAmount: null, + budgetUnit: null, + enabled: true + } + }); + + const providerType = form.watch("type"); + const routingMode = form.watch("routingMode"); + + const showUpstream = showsUpstreamUrlField(providerType, routingMode); + const requireUpstream = upstreamUrlRequired(providerType, routingMode); + const showRoutingMode = providerType === "custom"; + const showAuthType = + providerType === "custom" && (routingMode ?? "url") === "url"; + const showTargetNote = + providerType === "custom" && routingMode === "target"; + + async function onSubmit(values: AiProviderFormValues) { + setLoading(true); + try { + const res = await api.put< + AxiosResponse + >(`/org/${orgId}/ai-provider`, toAiProviderCreatePayload(values)); + + toast({ + title: t("success"), + description: t("aiProviderCreated") + }); + + router.push( + `/${orgId}/settings/ai-providers/${res.data.data.provider.providerId}` + ); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiProviderErrorCreate"), + description: formatAxiosError(e, t("aiProviderErrorCreate")) + }); + } finally { + setLoading(false); + } + } + + return ( + <> +
+ + +
+ + + + + + {t("aiProviderGeneral")} + + + {t("aiProviderGeneralDescription")} + + + + + +
+ + + + ( + + + + + + + )} + /> + + + + ( + + + {t("name")} + + + + + + + )} + /> + + + + ( + + + {t( + "aiProviderType" + )} + + + { + field.onChange( + value + ); + form.setValue( + "upstreamUrl", + emptyUpstreamForType( + value + ) + ); + if ( + value !== + "custom" + ) { + form.setValue( + "routingMode", + "url" + ); + } + }} + /> + + + + )} + /> + + + {showRoutingMode && ( + + ( + + + {t( + "aiProviderRoutingMode" + )} + + + { + field.onChange( + value + ); + if ( + value === + "target" + ) { + form.setValue( + "upstreamUrl", + "" + ); + } + }} + /> + + + {t( + "aiProviderRoutingModeDescription" + )} + + + + )} + /> + + )} + + {showTargetNote && ( + + + + + {t( + "aiProviderRoutingModeTarget" + )} + + + {t( + "aiProviderRoutingModeTargetNote" + )} + + + + )} + + {showUpstream && ( + + ( + + + {t( + "aiProviderUpstreamUrl" + )} + {requireUpstream + ? "" + : " (optional)"} + + + + + + {requireUpstream + ? t( + "aiProviderUpstreamUrlDescription" + ) + : t( + "aiProviderUpstreamUrlOptionalDescription" + )} + + + + )} + /> + + )} + + {showAuthType && ( + + ( + + + {t( + "aiProviderAuthType" + )} + + + + {t( + "aiProviderAuthTypeDescription" + )} + + + + )} + /> + + )} + + + ( + + + {t( + "aiProviderApiKey" + )} + + + + + + {t( + "aiProviderApiKeyDescription" + )} + + + + )} + /> + + + + ( + + + + + + + )} + /> + + +
+ + + + + + + + + + ); +} diff --git a/src/app/[orgId]/settings/ai-providers/page.tsx b/src/app/[orgId]/settings/ai-providers/page.tsx new file mode 100644 index 000000000..9858641e2 --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/page.tsx @@ -0,0 +1,71 @@ +import AiProvidersTable from "@app/components/AiProvidersTable"; +import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; +import { internal } from "@app/lib/api"; +import { authCookieHeader } from "@app/lib/api/cookies"; +import type { ListAiProvidersResponse } from "@server/routers/aiProvider/types"; +import type { AxiosResponse } from "axios"; +import type { Metadata } from "next"; +import { getTranslations } from "next-intl/server"; + +export const metadata: Metadata = { + title: "AI Providers" +}; + +export const dynamic = "force-dynamic"; + +type Props = { + params: Promise<{ orgId: string }>; + searchParams: Promise>; +}; + +export default async function AiProvidersPage({ params, searchParams }: Props) { + const { orgId } = await params; + const searchParamsObj = new URLSearchParams(await searchParams); + const t = await getTranslations(); + + let providers: ListAiProvidersResponse["providers"] = []; + let pagination: ListAiProvidersResponse["pagination"] = { + total: 0, + page: 1, + pageSize: 20 + }; + + try { + const res = await internal.get>( + `/org/${orgId}/ai-providers?${searchParamsObj.toString()}`, + await authCookieHeader() + ); + const responseData = res.data.data; + providers = responseData.providers; + pagination = responseData.pagination; + } catch { + // empty list on error + } + + return ( + <> + + + ({ + providerId: provider.providerId, + name: provider.name, + type: provider.type, + routingMode: provider.routingMode, + enabled: provider.enabled, + effectiveUpstreamUrl: provider.effectiveUpstreamUrl, + apiKeyLastChars: provider.apiKeyLastChars + }))} + rowCount={pagination.total} + pagination={{ + pageIndex: pagination.page - 1, + pageSize: pagination.pageSize + }} + /> + + ); +} diff --git a/src/app/navigation.tsx b/src/app/navigation.tsx index b0650ce05..cb91bd734 100644 --- a/src/app/navigation.tsx +++ b/src/app/navigation.tsx @@ -25,6 +25,7 @@ import { Server, Settings, ShieldIcon, + Sparkles, SquareMousePointer, TagIcon, TicketCheck, @@ -186,6 +187,16 @@ export const orgNavSections = ( } ] }, + { + heading: "sidebarAi", + items: [ + { + title: "sidebarAiProviders", + href: "/{orgId}/settings/ai-providers", + icon: + } + ] + }, { heading: "sidebarOrganization", items: [ @@ -471,6 +482,16 @@ export const commandBarNavSections = ( } ] }, + { + heading: "sidebarAi", + items: [ + { + title: "commandAiProviders", + href: "/{orgId}/settings/ai-providers", + icon: + } + ] + }, { heading: "commandLogsAndAnalytics", items: [ diff --git a/src/components/AiProviderTypeSelect.tsx b/src/components/AiProviderTypeSelect.tsx new file mode 100644 index 000000000..72e076f53 --- /dev/null +++ b/src/components/AiProviderTypeSelect.tsx @@ -0,0 +1,147 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList +} from "@app/components/ui/command"; +import { + Popover, + PopoverContent, + PopoverTrigger +} from "@app/components/ui/popover"; +import { cn } from "@app/lib/cn"; +import { aiProviderTypeValues } from "@app/lib/aiProviderFormSchema"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import { CheckIcon, ChevronsUpDown } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useMemo, useState } from "react"; + +const typeLabelMap = { + openai: "aiProviderTypeOpenai", + anthropic: "aiProviderTypeAnthropic", + googleGemini: "aiProviderTypeGoogleGemini", + vertexAi: "aiProviderTypeVertexAi", + bedrock: "aiProviderTypeBedrock", + microsoftFoundry: "aiProviderTypeMicrosoftFoundry", + openRouter: "aiProviderTypeOpenRouter", + vercelAiGateway: "aiProviderTypeVercelAiGateway", + custom: "aiProviderTypeCustom" +} as const; + +const typeDescriptionMap = { + openai: "aiProviderTypeOpenaiDescription", + anthropic: "aiProviderTypeAnthropicDescription", + googleGemini: "aiProviderTypeGoogleGeminiDescription", + vertexAi: "aiProviderTypeVertexAiDescription", + bedrock: "aiProviderTypeBedrockDescription", + microsoftFoundry: "aiProviderTypeMicrosoftFoundryDescription", + openRouter: "aiProviderTypeOpenRouterDescription", + vercelAiGateway: "aiProviderTypeVercelAiGatewayDescription", + custom: "aiProviderTypeCustomDescription" +} as const; + +type AiProviderTypeSelectProps = { + value: AiProviderType; + onChange: (value: AiProviderType) => void; + disabled?: boolean; + className?: string; +}; + +export function AiProviderTypeSelect({ + value, + onChange, + disabled, + className +}: AiProviderTypeSelectProps) { + const t = useTranslations(); + const [open, setOpen] = useState(false); + + const options = useMemo( + () => + aiProviderTypeValues.map((type) => ({ + type, + title: t(typeLabelMap[type]), + description: t(typeDescriptionMap[type]) + })), + [t] + ); + + const selected = options.find((option) => option.type === value); + + return ( + + + + + + + + + + {t("aiProviderTypeNotFound")} + + + {options.map((option) => ( + { + onChange(option.type); + setOpen(false); + }} + > + +
+ + {option.title} + + + {option.description} + +
+ + ))} + + + + + + ); +} diff --git a/src/components/AiProvidersTable.tsx b/src/components/AiProvidersTable.tsx new file mode 100644 index 000000000..ba7fbb00c --- /dev/null +++ b/src/components/AiProvidersTable.tsx @@ -0,0 +1,317 @@ +"use client"; + +import ConfirmDeleteDialog from "@app/components/ConfirmDeleteDialog"; +import { Button } from "@app/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger +} from "@app/components/ui/dropdown-menu"; +import { Switch } from "@app/components/ui/switch"; +import { + ControlledDataTable, + type ExtendedColumnDef +} from "@app/components/ui/controlled-data-table"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { useNavigationContext } from "@app/hooks/useNavigationContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import type { PaginationState } from "@tanstack/react-table"; +import { ArrowRight, MoreHorizontal } from "lucide-react"; +import { useTranslations } from "next-intl"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useEffect, useMemo, useState, useTransition } from "react"; +import { useDebouncedCallback } from "use-debounce"; + +export type AiProviderRow = { + providerId: number; + name: string; + type: string; + routingMode: string; + enabled: boolean; + effectiveUpstreamUrl: string | null; + apiKeyLastChars: string | null; +}; + +type AiProvidersTableProps = { + providers: AiProviderRow[]; + orgId: string; + pagination: PaginationState; + rowCount: number; +}; + +export default function AiProvidersTable({ + providers, + orgId, + pagination, + rowCount +}: AiProvidersTableProps) { + const router = useRouter(); + const t = useTranslations(); + const api = createApiClient(useEnvContext()); + const { + navigate: filter, + isNavigating: isFiltering, + searchParams + } = useNavigationContext(); + + const [rows, setRows] = useState(providers); + const [selected, setSelected] = useState(null); + const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const [isRefreshing, startTransition] = useTransition(); + + useEffect(() => { + setRows(providers); + }, [providers]); + + function refreshData() { + startTransition(() => { + try { + router.refresh(); + } catch { + toast({ + title: t("error"), + description: t("refreshError"), + variant: "destructive" + }); + } + }); + } + + const handlePaginationChange = (newPage: PaginationState) => { + searchParams.set("page", (newPage.pageIndex + 1).toString()); + searchParams.set("pageSize", newPage.pageSize.toString()); + filter({ searchParams }); + }; + + const handleSearchChange = useDebouncedCallback((query: string) => { + searchParams.set("query", query); + searchParams.delete("page"); + filter({ searchParams }); + }, 300); + + function typeLabel(type: string) { + const key = `aiProviderType${type.charAt(0).toUpperCase()}${type.slice(1)}`; + const map: Record = { + openai: "aiProviderTypeOpenai", + anthropic: "aiProviderTypeAnthropic", + googleGemini: "aiProviderTypeGoogleGemini", + vertexAi: "aiProviderTypeVertexAi", + bedrock: "aiProviderTypeBedrock", + microsoftFoundry: "aiProviderTypeMicrosoftFoundry", + openRouter: "aiProviderTypeOpenRouter", + vercelAiGateway: "aiProviderTypeVercelAiGateway", + custom: "aiProviderTypeCustom" + }; + return t(map[type] ?? key); + } + + function routingLabel(mode: string) { + return mode === "target" + ? t("aiProviderRoutingModeTarget") + : t("aiProviderRoutingModeUrl"); + } + + async function toggleEnabled(row: AiProviderRow, enabled: boolean) { + setRows((prev) => + prev.map((r) => + r.providerId === row.providerId ? { ...r, enabled } : r + ) + ); + + try { + await api.post(`/ai-provider/${row.providerId}`, { enabled }); + toast({ + title: t("success"), + description: t("aiProviderUpdated") + }); + router.refresh(); + } catch (e) { + setRows((prev) => + prev.map((r) => + r.providerId === row.providerId + ? { ...r, enabled: row.enabled } + : r + ) + ); + toast({ + variant: "destructive", + title: t("aiProviderErrorUpdate"), + description: formatAxiosError(e, t("aiProviderErrorUpdate")) + }); + } + } + + function deleteProvider(row: AiProviderRow) { + startTransition(async () => { + try { + await api.delete(`/ai-provider/${row.providerId}`); + setRows((prev) => + prev.filter((r) => r.providerId !== row.providerId) + ); + setIsDeleteModalOpen(false); + setSelected(null); + toast({ + title: t("success"), + description: t("aiProviderDeleted") + }); + router.refresh(); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiProviderErrorDelete"), + description: formatAxiosError(e, t("aiProviderErrorDelete")) + }); + } + }); + } + + const columns = useMemo[]>( + () => [ + { + accessorKey: "name", + enableHiding: false, + header: () => {t("name")}, + cell: ({ row }) => ( + + {row.original.name} + + ) + }, + { + accessorKey: "type", + header: () => ( + {t("aiProviderType")} + ), + cell: ({ row }) => typeLabel(row.original.type) + }, + { + accessorKey: "routingMode", + header: () => ( + {t("aiProviderRoutingMode")} + ), + cell: ({ row }) => routingLabel(row.original.routingMode) + }, + { + accessorKey: "effectiveUpstreamUrl", + header: () => ( + {t("aiProviderUpstreamUrl")} + ), + cell: ({ row }) => ( + {row.original.effectiveUpstreamUrl ?? "-"} + ) + }, + { + accessorKey: "enabled", + header: () => ( + {t("aiProviderEnabled")} + ), + cell: ({ row }) => ( + + toggleEnabled(row.original, checked) + } + /> + ) + }, + { + id: "actions", + enableHiding: false, + header: () => , + cell: ({ row }) => ( +
+ + + + + + + + {t("edit")} + + + { + setSelected(row.original); + setIsDeleteModalOpen(true); + }} + > + + {t("delete")} + + + + + + + +
+ ) + } + ], + [orgId, t] + ); + + return ( + <> + {selected && ( + { + setIsDeleteModalOpen(val); + if (!val) { + setSelected(null); + } + }} + dialog={ +
+

{t("aiProviderQuestionRemove")}

+

{t("aiProviderMessageRemove")}

+
+ } + buttonText={t("aiProviderDeleteConfirm")} + onConfirm={async () => deleteProvider(selected)} + string={selected.name} + title={t("aiProviderDelete")} + /> + )} + + + router.push(`/${orgId}/settings/ai-providers/create`) + } + tableId="ai-providers-table" + searchPlaceholder={t("aiProvidersSearch")} + pagination={pagination} + onPaginationChange={handlePaginationChange} + searchQuery={searchParams.get("query")?.toString()} + onSearch={handleSearchChange} + onRefresh={refreshData} + isRefreshing={isRefreshing || isFiltering} + rowCount={rowCount} + stickyRightColumn="actions" + /> + + ); +} diff --git a/src/components/HorizontalTabs.tsx b/src/components/HorizontalTabs.tsx index bffaadeba..6c91dbfef 100644 --- a/src/components/HorizontalTabs.tsx +++ b/src/components/HorizontalTabs.tsx @@ -47,6 +47,7 @@ export function HorizontalTabs({ .replace("{userId}", params.userId as string) .replace("{clientId}", params.clientId as string) .replace("{apiKeyId}", params.apiKeyId as string) + .replace("{providerId}", params.providerId as string) .replace("{remoteExitNodeId}", params.remoteExitNodeId as string); } diff --git a/src/contexts/aiProviderContext.ts b/src/contexts/aiProviderContext.ts new file mode 100644 index 000000000..bb72eef1a --- /dev/null +++ b/src/contexts/aiProviderContext.ts @@ -0,0 +1,13 @@ +import { createContext } from "react"; +import type { AiProviderPublic } from "@server/routers/aiProvider/types"; + +export type AiProviderContextType = { + provider: AiProviderPublic; + updateProvider: (updated: Partial) => void; +}; + +const AiProviderContext = createContext( + undefined +); + +export default AiProviderContext; diff --git a/src/hooks/useAiProviderContext.ts b/src/hooks/useAiProviderContext.ts new file mode 100644 index 000000000..9ab1182ca --- /dev/null +++ b/src/hooks/useAiProviderContext.ts @@ -0,0 +1,12 @@ +import AiProviderContext from "@app/contexts/aiProviderContext"; +import { useContext } from "react"; + +export function useAiProviderContext() { + const context = useContext(AiProviderContext); + if (context === undefined) { + throw new Error( + "useAiProviderContext must be used within an AiProviderProvider" + ); + } + return context; +} diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts new file mode 100644 index 000000000..184be14c9 --- /dev/null +++ b/src/lib/aiProviderFormSchema.ts @@ -0,0 +1,202 @@ +import { z } from "zod"; +import { + AI_PROVIDER_DEFAULTS, + providerRequiresUpstreamUrl, + type AiProviderType +} from "@server/lib/aiProviderDefaults"; + +export const aiProviderTypeValues = [ + "openai", + "anthropic", + "googleGemini", + "vertexAi", + "bedrock", + "microsoftFoundry", + "openRouter", + "vercelAiGateway", + "custom" +] as const satisfies readonly AiProviderType[]; + +export const aiProviderFormSchema = z + .object({ + name: z.string().trim().min(1), + type: z.enum(aiProviderTypeValues), + upstreamUrl: z.string().optional().nullable(), + apiKey: z.string().optional(), + authType: z.enum(["bearer"]).optional().nullable(), + routingMode: z.enum(["url", "target"]).optional(), + skipTlsVerification: z.boolean().optional(), + budgetAmount: z.number().positive().nullable().optional(), + budgetUnit: z.enum(["usd", "tokens"]).optional().nullable(), + enabled: z.boolean().optional() + }) + .superRefine((data, ctx) => { + const routingMode = + data.type === "custom" ? (data.routingMode ?? "url") : "url"; + + if (data.type !== "custom" && data.routingMode === "target") { + ctx.addIssue({ + code: "custom", + message: + "routingMode target is only allowed for custom providers", + path: ["routingMode"] + }); + } + + const upstreamUrl = + data.upstreamUrl && data.upstreamUrl.trim().length > 0 + ? data.upstreamUrl.trim() + : null; + + if (upstreamUrl) { + try { + new URL(upstreamUrl); + } catch { + ctx.addIssue({ + code: "custom", + message: "Invalid URL", + path: ["upstreamUrl"] + }); + } + } + + if ( + providerRequiresUpstreamUrl(data.type, routingMode) && + !upstreamUrl + ) { + ctx.addIssue({ + code: "custom", + message: `upstreamUrl is required for ${data.type} providers`, + path: ["upstreamUrl"] + }); + } + + if (data.type === "custom" && routingMode === "url" && !data.authType) { + ctx.addIssue({ + code: "custom", + message: "authType is required for custom providers", + path: ["authType"] + }); + } + + const hasAmount = + data.budgetAmount !== undefined && data.budgetAmount !== null; + const hasUnit = + data.budgetUnit !== undefined && data.budgetUnit !== null; + + if (hasAmount !== hasUnit) { + ctx.addIssue({ + code: "custom", + message: + "budgetAmount and budgetUnit must both be set or both omitted", + path: hasAmount ? ["budgetUnit"] : ["budgetAmount"] + }); + } + }); + +export type AiProviderFormValues = z.infer; + +export function emptyUpstreamForType(type: AiProviderType): string { + if (type === "custom") { + return ""; + } + return AI_PROVIDER_DEFAULTS[type].upstreamUrl ?? ""; +} + +export function showsUpstreamUrlField( + type: AiProviderType, + routingMode: "url" | "target" | undefined +): boolean { + const mode = type === "custom" ? (routingMode ?? "url") : "url"; + if (mode === "target") { + return false; + } + return true; +} + +export function upstreamUrlRequired( + type: AiProviderType, + routingMode: "url" | "target" | undefined +): boolean { + const mode = type === "custom" ? (routingMode ?? "url") : "url"; + return providerRequiresUpstreamUrl(type, mode); +} + +export function toAiProviderCreatePayload(values: AiProviderFormValues) { + const routingMode = + values.type === "custom" ? (values.routingMode ?? "url") : "url"; + const upstreamRaw = values.upstreamUrl?.trim() ?? ""; + const upstreamUrl = + routingMode === "target" + ? null + : upstreamRaw.length > 0 + ? upstreamRaw + : null; + + const hasBudget = + values.budgetAmount !== undefined && + values.budgetAmount !== null && + values.budgetUnit; + + return { + name: values.name.trim(), + type: values.type, + routingMode: values.type === "custom" ? routingMode : undefined, + upstreamUrl, + apiKey: values.apiKey?.trim() ? values.apiKey.trim() : undefined, + authType: + values.type === "custom" && routingMode === "url" + ? (values.authType ?? "bearer") + : (values.authType ?? undefined), + skipTlsVerification: values.skipTlsVerification, + budgetAmount: hasBudget ? values.budgetAmount : null, + budgetUnit: hasBudget ? values.budgetUnit : null, + enabled: values.enabled ?? true + }; +} + +export function toAiProviderUpdatePayload(values: AiProviderFormValues) { + const routingMode = + values.type === "custom" ? (values.routingMode ?? "url") : "url"; + const upstreamRaw = values.upstreamUrl?.trim() ?? ""; + const upstreamUrl = + routingMode === "target" + ? null + : upstreamRaw.length > 0 + ? upstreamRaw + : null; + + const hasBudget = + values.budgetAmount !== undefined && + values.budgetAmount !== null && + values.budgetUnit; + + const payload: Record = { + name: values.name.trim(), + routingMode: values.type === "custom" ? routingMode : "url", + upstreamUrl, + authType: + values.type === "custom" && routingMode === "url" + ? (values.authType ?? "bearer") + : (values.authType ?? null), + skipTlsVerification: values.skipTlsVerification ?? false, + budgetAmount: hasBudget ? values.budgetAmount : null, + budgetUnit: hasBudget ? values.budgetUnit : null, + enabled: values.enabled ?? true + }; + + if (values.apiKey?.trim()) { + payload.apiKey = values.apiKey.trim(); + } + + return payload; +} + +export function toAiProviderConfigurationPayload(values: AiProviderFormValues) { + const { + name: _name, + enabled: _enabled, + ...payload + } = toAiProviderUpdatePayload(values); + return payload; +} diff --git a/src/providers/AiProviderProvider.tsx b/src/providers/AiProviderProvider.tsx new file mode 100644 index 000000000..072f75608 --- /dev/null +++ b/src/providers/AiProviderProvider.tsx @@ -0,0 +1,47 @@ +"use client"; + +import AiProviderContext from "@app/contexts/aiProviderContext"; +import type { AiProviderPublic } from "@server/routers/aiProvider/types"; +import { useTranslations } from "next-intl"; +import { useEffect, useState } from "react"; + +type AiProviderProviderProps = { + children: React.ReactNode; + provider: AiProviderPublic; +}; + +export function AiProviderProvider({ + children, + provider: serverProvider +}: AiProviderProviderProps) { + const [provider, setProvider] = useState(serverProvider); + const t = useTranslations(); + + useEffect(() => { + setProvider(serverProvider); + }, [serverProvider]); + + const updateProvider = (updated: Partial) => { + if (!provider) { + throw new Error(t("aiProviderErrorNoUpdate")); + } + + setProvider((prev) => { + if (!prev) { + return prev; + } + return { + ...prev, + ...updated + }; + }); + }; + + return ( + + {children} + + ); +} + +export default AiProviderProvider; From 33b775e2633c6fbbf08ca9f6466fd9a2ae54a69b Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Mon, 3 Aug 2026 16:06:28 -0400 Subject: [PATCH 022/296] clean up providers ui --- messages/en-US.json | 17 +- server/routers/aiProvider/createAiProvider.ts | 4 +- server/routers/aiProvider/getAiProvider.ts | 4 +- server/routers/aiProvider/types.ts | 24 +- server/routers/aiProvider/updateAiProvider.ts | 16 +- server/routers/aiProvider/validation.ts | 2 +- .../[providerId]/authentication/page.tsx | 235 ++++++ .../[providerId]/configuration/page.tsx | 432 +--------- .../ai-providers/[providerId]/layout.tsx | 8 +- .../[providerId]/network/page.tsx | 362 +++++++++ .../settings/ai-providers/create/page.tsx | 751 ++++++++++-------- .../public/ProxyResourceTargetsForm.tsx | 431 +++++----- src/app/navigation.tsx | 7 +- src/components/AiProviderTypeSelect.tsx | 16 +- .../resource-target-address-item.tsx | 16 +- src/lib/aiProviderFormSchema.ts | 35 +- src/lib/queries.ts | 14 + 17 files changed, 1412 insertions(+), 962 deletions(-) create mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx create mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 4febc876a..d6759f822 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1632,7 +1632,7 @@ "sidebarInvitations": "Invitations", "sidebarRoles": "Roles", "sidebarShareableLinks": "Shareable Links", - "sidebarAi": "AI", + "sidebarAiGateway": "AI Gateway", "sidebarAiProviders": "Providers", "commandAiProviders": "AI Providers", "aiProvidersTitle": "AI Providers", @@ -1648,7 +1648,11 @@ "aiProviderGeneral": "General", "aiProviderGeneralDescription": "Basic settings for this provider", "aiProviderConfiguration": "Configuration", - "aiProviderConfigurationDescription": "Upstream URL, routing, authentication, and TLS settings", + "aiProviderConfigurationDescription": "Network routing and authentication for this provider", + "aiProviderNetworkSettings": "Network Settings", + "aiProviderNetworkSettingsDescription": "Choose how traffic reaches this provider", + "aiProviderAuthSettings": "Authentication", + "aiProviderAuthSettingsDescription": "Credentials used for both upstream URL and Pangolin target routing", "aiProviderType": "Provider Type", "aiProviderTypeSearch": "Search providers...", "aiProviderTypeNotFound": "No provider type found", @@ -1675,18 +1679,19 @@ "aiProviderUpstreamUrlOptionalDescription": "Leave blank to use the default upstream URL for this provider", "aiProviderEffectiveUpstreamUrl": "Effective Upstream URL", "aiProviderApiKey": "API Key", - "aiProviderApiKeyDescription": "Stored encrypted. Leave blank on edit to keep the existing key.", + "aiProviderApiKeyDescription": "API key used to authenticate requests to this provider", "aiProviderApiKeyLastChars": "API Key", "aiProviderAuthType": "Auth Type", "aiProviderAuthTypeBearer": "Bearer", "aiProviderAuthTypeDescription": "How the upstream API authenticates requests", "aiProviderRoutingMode": "Routing Mode", - "aiProviderRoutingModeDescription": "Send traffic to an upstream URL or to Pangolin HTTPS targets", + "aiProviderRoutingModeDescription": "Send traffic to an upstream URL or to HTTP targets on your sites", "aiProviderRoutingModeUrl": "Upstream URL", "aiProviderRoutingModeUrlDescription": "Call a public or private API base URL", - "aiProviderRoutingModeTarget": "Pangolin Targets", + "aiProviderRoutingModeTarget": "Site Targets", "aiProviderRoutingModeTargetDescription": "Route through HTTPS targets on your sites", - "aiProviderRoutingModeTargetNote": "Target configuration will be available in a later update. You can still create this provider now.", + "aiProviderRoutingModeTargetNote": "After creating this provider, configure site targets on the Network Settings tab.", + "aiProviderTargetNoOne": "This provider doesn't have any targets. Add a target to route requests through your sites.", "aiProviderSkipTlsVerification": "Skip TLS Verification", "aiProviderSkipTlsVerificationDescription": "Disable TLS certificate verification for the upstream connection", "aiProviderBudget": "Budget", diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index 947810ee1..1d58344cb 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -135,7 +135,9 @@ export async function createAiProvider( .returning(); return response(res, { - data: { provider: toPublicAiProvider(provider) }, + data: { + provider: toPublicAiProvider(provider, { includeApiKey: true }) + }, success: true, error: false, message: "AI provider created successfully", diff --git a/server/routers/aiProvider/getAiProvider.ts b/server/routers/aiProvider/getAiProvider.ts index 32a337c85..28a0152e1 100644 --- a/server/routers/aiProvider/getAiProvider.ts +++ b/server/routers/aiProvider/getAiProvider.ts @@ -67,7 +67,9 @@ export async function getAiProvider( } return response(res, { - data: { provider: toPublicAiProvider(provider) }, + data: { + provider: toPublicAiProvider(provider, { includeApiKey: true }) + }, success: true, error: false, message: "AI provider retrieved successfully", diff --git a/server/routers/aiProvider/types.ts b/server/routers/aiProvider/types.ts index 2a6047e23..3fa966785 100644 --- a/server/routers/aiProvider/types.ts +++ b/server/routers/aiProvider/types.ts @@ -6,8 +6,12 @@ import { type AiProviderRoutingMode, type AiProviderType } from "@server/lib/aiProviderDefaults"; +import { decrypt } from "@server/lib/crypto"; +import config from "@server/lib/config"; export type AiProviderPublic = Omit & { + /** Decrypted API key. Only included on get/create/update of a single provider. */ + apiKey?: string | null; effectiveUpstreamUrl: string | null; effectiveAuthType: AiProviderAuthType | null; }; @@ -36,8 +40,11 @@ export type CreateOrEditAiModelResponse = { model: AiModel; }; -export function toPublicAiProvider(provider: AiProvider): AiProviderPublic { - const { apiKey: _apiKey, ...rest } = provider; +export function toPublicAiProvider( + provider: AiProvider, + options?: { includeApiKey?: boolean } +): AiProviderPublic { + const { apiKey: encryptedApiKey, ...rest } = provider; const resolved = resolveAiProviderConfig({ type: provider.type as AiProviderType, upstreamUrl: provider.upstreamUrl, @@ -45,8 +52,21 @@ export function toPublicAiProvider(provider: AiProvider): AiProviderPublic { routingMode: provider.routingMode as AiProviderRoutingMode | null }); + let apiKey: string | null | undefined; + if (options?.includeApiKey) { + if (encryptedApiKey) { + apiKey = decrypt( + encryptedApiKey, + config.getRawConfig().server.secret! + ); + } else { + apiKey = null; + } + } + return { ...rest, + ...(options?.includeApiKey ? { apiKey } : {}), effectiveUpstreamUrl: resolved.upstreamUrl, effectiveAuthType: resolved.authType }; diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts index 494e459f6..39787fb93 100644 --- a/server/routers/aiProvider/updateAiProvider.ts +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -125,7 +125,10 @@ export async function updateAiProvider( ? body.upstreamUrl : existing.upstreamUrl; const nextAuthType = - body.authType !== undefined ? body.authType : existing.authType; + body.authType !== undefined + ? body.authType + : (existing.authType ?? + (providerType === "custom" ? "bearer" : null)); const validation = z .object({ @@ -178,6 +181,13 @@ export async function updateAiProvider( } if (body.authType !== undefined) { updateData.authType = body.authType; + } else if ( + providerType === "custom" && + !existing.authType && + nextAuthType + ) { + // Backfill required authType for custom providers created without one + updateData.authType = nextAuthType; } if (body.apiKey !== undefined) { @@ -193,7 +203,9 @@ export async function updateAiProvider( .returning(); return response(res, { - data: { provider: toPublicAiProvider(provider) }, + data: { + provider: toPublicAiProvider(provider, { includeApiKey: true }) + }, success: true, error: false, message: "AI provider updated successfully", diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts index a0b0aa684..c830c08de 100644 --- a/server/routers/aiProvider/validation.ts +++ b/server/routers/aiProvider/validation.ts @@ -75,7 +75,7 @@ export function refineProviderUpstreamFields( }); } - if (data.type === "custom" && routingMode === "url" && !data.authType) { + if (data.type === "custom" && !data.authType) { ctx.addIssue({ code: "custom", message: "authType is required for custom providers", diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx new file mode 100644 index 000000000..f054c7901 --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx @@ -0,0 +1,235 @@ +"use client"; + +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { Input } from "@app/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue +} from "@app/components/ui/select"; +import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { + aiProviderFormSchema, + toAiProviderAuthPayload, + type AiProviderFormValues +} from "@app/lib/aiProviderFormSchema"; +import { zodResolver } from "@hookform/resolvers/zod"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; +import type { AxiosResponse } from "axios"; +import { useTranslations } from "next-intl"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; +import { useForm } from "react-hook-form"; + +export default function AiProviderAuthenticationPage() { + const { provider, updateProvider } = useAiProviderContext(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const router = useRouter(); + const t = useTranslations(); + const [saveLoading, setSaveLoading] = useState(false); + + const form = useForm({ + resolver: zodResolver(aiProviderFormSchema), + defaultValues: { + name: provider.name, + type: provider.type as AiProviderType, + upstreamUrl: provider.upstreamUrl ?? "", + apiKey: provider.apiKey ?? "", + authType: (provider.authType as "bearer" | null) ?? "bearer", + routingMode: (provider.routingMode as "url" | "target") ?? "url", + skipTlsVerification: provider.skipTlsVerification, + budgetAmount: provider.budgetAmount, + budgetUnit: provider.budgetUnit as "usd" | "tokens" | null, + enabled: provider.enabled + } + }); + + const showAuthType = provider.type === "custom"; + + async function onSubmit(values: AiProviderFormValues) { + setSaveLoading(true); + try { + const res = await api.post< + AxiosResponse + >( + `/ai-provider/${provider.providerId}`, + toAiProviderAuthPayload({ + ...values, + type: provider.type as AiProviderType + }) + ); + const updated = res.data.data.provider; + updateProvider(updated); + form.reset({ + name: updated.name, + type: updated.type as AiProviderType, + upstreamUrl: updated.upstreamUrl ?? "", + apiKey: updated.apiKey ?? "", + authType: (updated.authType as "bearer" | null) ?? "bearer", + routingMode: (updated.routingMode as "url" | "target") ?? "url", + skipTlsVerification: updated.skipTlsVerification, + budgetAmount: updated.budgetAmount, + budgetUnit: updated.budgetUnit as "usd" | "tokens" | null, + enabled: updated.enabled + }); + toast({ + title: t("success"), + description: t("aiProviderUpdated") + }); + router.refresh(); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiProviderErrorUpdate"), + description: formatAxiosError(e, t("aiProviderErrorUpdate")) + }); + } finally { + setSaveLoading(false); + } + } + + return ( + + + + + {t("aiProviderAuthSettings")} + + + {t("aiProviderAuthSettingsDescription")} + + + + + +
+ + + {showAuthType && ( + + ( + + + {t( + "aiProviderAuthType" + )} + + + + {t( + "aiProviderAuthTypeDescription" + )} + + + + )} + /> + + )} + + + ( + + + {t("aiProviderApiKey")} + + + + + + {t( + "aiProviderApiKeyDescription" + )} + + + + )} + /> + + +
+ + + + + + + + + ); +} diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx index 03a9ab48f..bf003b96e 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx @@ -1,426 +1,12 @@ -"use client"; +import { redirect } from "next/navigation"; -import { - SettingsContainer, - SettingsFormCell, - SettingsFormGrid, - SettingsSection, - SettingsSectionBody, - SettingsSectionDescription, - SettingsSectionFooter, - SettingsSectionForm, - SettingsSectionHeader, - SettingsSectionTitle -} from "@app/components/Settings"; -import { StrategySelect } from "@app/components/StrategySelect"; -import { SwitchInput } from "@app/components/SwitchInput"; -import { Alert, AlertDescription, AlertTitle } from "@app/components/ui/alert"; -import { Button } from "@app/components/ui/button"; -import { - Form, - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage -} from "@app/components/ui/form"; -import { Input } from "@app/components/ui/input"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue -} from "@app/components/ui/select"; -import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; -import { useEnvContext } from "@app/hooks/useEnvContext"; -import { toast } from "@app/hooks/useToast"; -import { createApiClient, formatAxiosError } from "@app/lib/api"; -import { - aiProviderFormSchema, - showsUpstreamUrlField, - toAiProviderConfigurationPayload, - upstreamUrlRequired, - type AiProviderFormValues -} from "@app/lib/aiProviderFormSchema"; -import { zodResolver } from "@hookform/resolvers/zod"; -import type { AiProviderType } from "@server/lib/aiProviderDefaults"; -import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; -import type { AxiosResponse } from "axios"; -import { InfoIcon } from "lucide-react"; -import { useTranslations } from "next-intl"; -import { useRouter } from "next/navigation"; -import { useState } from "react"; -import { useForm } from "react-hook-form"; +type Props = { + params: Promise<{ orgId: string; providerId: string }>; +}; -export default function AiProviderConfigurationPage() { - const { provider, updateProvider } = useAiProviderContext(); - const { env } = useEnvContext(); - const api = createApiClient({ env }); - const router = useRouter(); - const t = useTranslations(); - const [saveLoading, setSaveLoading] = useState(false); - - const form = useForm({ - resolver: zodResolver(aiProviderFormSchema), - defaultValues: { - name: provider.name, - type: provider.type as AiProviderType, - upstreamUrl: provider.upstreamUrl ?? "", - apiKey: "", - authType: (provider.authType as "bearer" | null) ?? "bearer", - routingMode: (provider.routingMode as "url" | "target") ?? "url", - skipTlsVerification: provider.skipTlsVerification, - budgetAmount: provider.budgetAmount, - budgetUnit: provider.budgetUnit as "usd" | "tokens" | null, - enabled: provider.enabled - } - }); - - const providerType = form.watch("type"); - const routingMode = form.watch("routingMode"); - const showUpstream = showsUpstreamUrlField(providerType, routingMode); - const requireUpstream = upstreamUrlRequired(providerType, routingMode); - const showRoutingMode = providerType === "custom"; - const showAuthType = - providerType === "custom" && (routingMode ?? "url") === "url"; - const showTargetNote = - providerType === "custom" && routingMode === "target"; - - async function onSubmit(values: AiProviderFormValues) { - setSaveLoading(true); - try { - const res = await api.post< - AxiosResponse - >( - `/ai-provider/${provider.providerId}`, - toAiProviderConfigurationPayload({ - ...values, - type: provider.type as AiProviderType - }) - ); - const updated = res.data.data.provider; - updateProvider(updated); - form.reset({ - name: updated.name, - type: updated.type as AiProviderType, - upstreamUrl: updated.upstreamUrl ?? "", - apiKey: "", - authType: (updated.authType as "bearer" | null) ?? "bearer", - routingMode: (updated.routingMode as "url" | "target") ?? "url", - skipTlsVerification: updated.skipTlsVerification, - budgetAmount: updated.budgetAmount, - budgetUnit: updated.budgetUnit as "usd" | "tokens" | null, - enabled: updated.enabled - }); - toast({ - title: t("success"), - description: t("aiProviderUpdated") - }); - router.refresh(); - } catch (e) { - toast({ - variant: "destructive", - title: t("aiProviderErrorUpdate"), - description: formatAxiosError(e, t("aiProviderErrorUpdate")) - }); - } finally { - setSaveLoading(false); - } - } - - return ( - - - - - {t("aiProviderConfiguration")} - - - {t("aiProviderConfigurationDescription")} - - - - - -
- - - {showRoutingMode && ( - - ( - - - {t( - "aiProviderRoutingMode" - )} - - - { - field.onChange( - value - ); - if ( - value === - "target" - ) { - form.setValue( - "upstreamUrl", - "" - ); - } - }} - /> - - - {t( - "aiProviderRoutingModeDescription" - )} - - - - )} - /> - - )} - - {showTargetNote && ( - - - - - {t( - "aiProviderRoutingModeTarget" - )} - - - {t( - "aiProviderRoutingModeTargetNote" - )} - - - - )} - - {showUpstream && ( - - ( - - - {t( - "aiProviderUpstreamUrl" - )} - {requireUpstream - ? "" - : " (optional)"} - - - - - - {requireUpstream - ? t( - "aiProviderUpstreamUrlDescription" - ) - : t( - "aiProviderUpstreamUrlOptionalDescription" - )} - - {provider.effectiveUpstreamUrl && ( - - {t( - "aiProviderEffectiveUpstreamUrl" - )} - {": "} - - { - provider.effectiveUpstreamUrl - } - - - )} - - - )} - /> - - )} - - {showAuthType && ( - - ( - - - {t( - "aiProviderAuthType" - )} - - - - {t( - "aiProviderAuthTypeDescription" - )} - - - - )} - /> - - )} - - - ( - - - {t("aiProviderApiKey")} - - - - - - {provider.apiKeyLastChars - ? `••••${provider.apiKeyLastChars}. ${t("aiProviderApiKeyDescription")}` - : t( - "aiProviderApiKeyDescription" - )} - - - - )} - /> - - - - ( - - - - - - - )} - /> - - -
- - - - - - - - - ); +export default async function AiProviderConfigurationRedirect({ + params +}: Props) { + const { orgId, providerId } = await params; + redirect(`/${orgId}/settings/ai-providers/${providerId}/network`); } diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx index b734f6084..5846ad86d 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx @@ -66,8 +66,12 @@ export default async function AiProviderLayout({ children, params }: Props) { href: "/{orgId}/settings/ai-providers/{providerId}/general" }, { - title: t("aiProviderConfiguration"), - href: "/{orgId}/settings/ai-providers/{providerId}/configuration" + title: t("aiProviderNetworkSettings"), + href: "/{orgId}/settings/ai-providers/{providerId}/network" + }, + { + title: t("aiProviderAuthSettings"), + href: "/{orgId}/settings/ai-providers/{providerId}/authentication" } ]; diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx new file mode 100644 index 000000000..90ef74d10 --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx @@ -0,0 +1,362 @@ +"use client"; + +import { + ProxyResourceTargetsForm, + type ProxyResourceTargetsFormHandle +} from "@app/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm"; +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle, + SettingsSubsectionDescription, + SettingsSubsectionHeader, + SettingsSubsectionTitle +} from "@app/components/Settings"; +import { StrategySelect } from "@app/components/StrategySelect"; +import { SwitchInput } from "@app/components/SwitchInput"; +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { Input } from "@app/components/ui/input"; +import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { + aiProviderFormSchema, + showsUpstreamUrlField, + toAiProviderNetworkPayload, + upstreamUrlRequired, + type AiProviderFormValues +} from "@app/lib/aiProviderFormSchema"; +import { aiProviderQueries } from "@app/lib/queries"; +import { zodResolver } from "@hookform/resolvers/zod"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; +import { useQuery } from "@tanstack/react-query"; +import type { AxiosResponse } from "axios"; +import { useTranslations } from "next-intl"; +import { useParams, useRouter } from "next/navigation"; +import { useRef, useState } from "react"; +import { useForm } from "react-hook-form"; + +export default function AiProviderNetworkPage() { + const { provider, updateProvider } = useAiProviderContext(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const params = useParams(); + const orgId = params.orgId as string; + const router = useRouter(); + const t = useTranslations(); + const [saveLoading, setSaveLoading] = useState(false); + const targetsFormRef = useRef(null); + + const form = useForm({ + resolver: zodResolver(aiProviderFormSchema), + defaultValues: { + name: provider.name, + type: provider.type as AiProviderType, + upstreamUrl: provider.upstreamUrl ?? "", + apiKey: "", + authType: (provider.authType as "bearer" | null) ?? "bearer", + routingMode: (provider.routingMode as "url" | "target") ?? "url", + skipTlsVerification: provider.skipTlsVerification, + budgetAmount: provider.budgetAmount, + budgetUnit: provider.budgetUnit as "usd" | "tokens" | null, + enabled: provider.enabled + } + }); + + const providerType = form.watch("type"); + const routingMode = form.watch("routingMode"); + const showUpstream = showsUpstreamUrlField(providerType, routingMode); + const requireUpstream = upstreamUrlRequired(providerType, routingMode); + const showRoutingMode = providerType === "custom"; + const isTargetModeSelected = routingMode === "target"; + const isTargetModeSaved = + provider.type === "custom" && provider.routingMode === "target"; + const showTargetsForm = showRoutingMode && isTargetModeSelected; + + const { data: remoteTargets = [], isLoading: isLoadingTargets } = useQuery({ + ...aiProviderQueries.providerTargets({ + providerId: provider.providerId + }), + enabled: isTargetModeSaved + }); + + async function onSubmit(values: AiProviderFormValues) { + setSaveLoading(true); + try { + const res = await api.post< + AxiosResponse + >( + `/ai-provider/${provider.providerId}`, + toAiProviderNetworkPayload({ + ...values, + type: provider.type as AiProviderType + }) + ); + const updated = res.data.data.provider; + updateProvider(updated); + form.reset({ + name: updated.name, + type: updated.type as AiProviderType, + upstreamUrl: updated.upstreamUrl ?? "", + apiKey: "", + authType: (updated.authType as "bearer" | null) ?? "bearer", + routingMode: (updated.routingMode as "url" | "target") ?? "url", + skipTlsVerification: updated.skipTlsVerification, + budgetAmount: updated.budgetAmount, + budgetUnit: updated.budgetUnit as "usd" | "tokens" | null, + enabled: updated.enabled + }); + + if (values.routingMode === "target" && targetsFormRef.current) { + const targetsSaved = await targetsFormRef.current.save({ + silent: true + }); + if (!targetsSaved) { + return; + } + } + + toast({ + title: t("success"), + description: t("aiProviderUpdated") + }); + router.refresh(); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiProviderErrorUpdate"), + description: formatAxiosError(e, t("aiProviderErrorUpdate")) + }); + } finally { + setSaveLoading(false); + } + } + + return ( + + + + + {t("aiProviderNetworkSettings")} + + + {t("aiProviderNetworkSettingsDescription")} + + + + + +
+ + + {showRoutingMode && ( + + ( + + + {t( + "aiProviderRoutingMode" + )} + + + { + field.onChange( + value + ); + if ( + value === + "target" + ) { + form.setValue( + "upstreamUrl", + "" + ); + } + }} + /> + + + {t( + "aiProviderRoutingModeDescription" + )} + + + + )} + /> + + )} + + {showUpstream && ( + + ( + + + {t( + "aiProviderUpstreamUrl" + )} + {requireUpstream + ? "" + : " (optional)"} + + + + + + {requireUpstream + ? t( + "aiProviderUpstreamUrlDescription" + ) + : t( + "aiProviderUpstreamUrlOptionalDescription" + )} + + + + )} + /> + + )} + + + ( + + + + + + + )} + /> + + +
+ + + + {showTargetsForm && + (!isTargetModeSaved || !isLoadingTargets) && ( +
+ + + {t("targets")} + + + {t("targetsDescription")} + + + +
+ )} + + + + + + + ); +} diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 9a9258d8d..ec6eee6c7 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -1,5 +1,9 @@ "use client"; +import { + ProxyResourceTargetsForm, + type LocalTarget +} from "@app/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm"; import { SettingsContainer, SettingsFormCell, @@ -7,16 +11,17 @@ import { SettingsSection, SettingsSectionBody, SettingsSectionDescription, - SettingsSectionFooter, SettingsSectionForm, SettingsSectionHeader, - SettingsSectionTitle + SettingsSectionTitle, + SettingsSubsectionDescription, + SettingsSubsectionHeader, + SettingsSubsectionTitle } from "@app/components/Settings"; import HeaderTitle from "@app/components/SettingsSectionTitle"; import { AiProviderTypeSelect } from "@app/components/AiProviderTypeSelect"; import { StrategySelect } from "@app/components/StrategySelect"; import { SwitchInput } from "@app/components/SwitchInput"; -import { Alert, AlertDescription, AlertTitle } from "@app/components/ui/alert"; import { Button } from "@app/components/ui/button"; import { Form, @@ -39,7 +44,7 @@ import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { - aiProviderFormSchema, + aiProviderCreateFormSchema, emptyUpstreamForType, showsUpstreamUrlField, toAiProviderCreatePayload, @@ -49,10 +54,9 @@ import { import { zodResolver } from "@hookform/resolvers/zod"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import type { AxiosResponse } from "axios"; -import { InfoIcon } from "lucide-react"; import { useTranslations } from "next-intl"; import { useParams, useRouter } from "next/navigation"; -import { useState } from "react"; +import { useRef, useState } from "react"; import { useForm } from "react-hook-form"; export default function CreateAiProviderPage() { @@ -63,9 +67,10 @@ export default function CreateAiProviderPage() { const router = useRouter(); const t = useTranslations(); const [loading, setLoading] = useState(false); + const targetsRef = useRef([]); const form = useForm({ - resolver: zodResolver(aiProviderFormSchema), + resolver: zodResolver(aiProviderCreateFormSchema), defaultValues: { name: "", type: "openai", @@ -86,26 +91,101 @@ export default function CreateAiProviderPage() { const showUpstream = showsUpstreamUrlField(providerType, routingMode); const requireUpstream = upstreamUrlRequired(providerType, routingMode); const showRoutingMode = providerType === "custom"; - const showAuthType = - providerType === "custom" && (routingMode ?? "url") === "url"; - const showTargetNote = - providerType === "custom" && routingMode === "target"; + const showAuthType = providerType === "custom"; + const showTargets = providerType === "custom" && routingMode === "target"; + + async function createTargets( + providerId: number, + localTargets: LocalTarget[] + ) { + for (const target of localTargets) { + const data = { + ip: target.ip, + port: target.port, + method: target.method, + enabled: target.enabled, + siteId: target.siteId, + hcEnabled: target.hcEnabled, + hcPath: target.hcPath || null, + hcMethod: target.hcMethod || null, + hcInterval: target.hcInterval || null, + hcTimeout: target.hcTimeout || null, + hcHeaders: target.hcHeaders || null, + hcScheme: target.hcScheme || null, + hcHostname: target.hcHostname || null, + hcPort: target.hcPort || null, + hcFollowRedirects: target.hcFollowRedirects || null, + hcStatus: target.hcStatus || null, + hcUnhealthyInterval: target.hcUnhealthyInterval || null, + hcMode: target.hcMode || null, + hcTlsServerName: target.hcTlsServerName, + hcHealthyThreshold: target.hcHealthyThreshold || null, + hcUnhealthyThreshold: target.hcUnhealthyThreshold || null, + path: target.path, + pathMatchType: target.pathMatchType, + rewritePath: target.rewritePath, + rewritePathType: target.rewritePathType, + priority: target.priority + }; + await api.put(`/ai-provider/${providerId}/target`, data); + } + } async function onSubmit(values: AiProviderFormValues) { + const targets = targetsRef.current; + + if (showTargets) { + const invalidTargets = targets.filter( + (target) => + !target.ip || + target.ip.trim() === "" || + !target.port || + target.port <= 0 || + isNaN(target.port) + ); + if (invalidTargets.length > 0) { + toast({ + variant: "destructive", + title: t("targetErrorInvalidIp"), + description: t("targetErrorInvalidIpDescription") + }); + return; + } + } + setLoading(true); try { const res = await api.put< AxiosResponse >(`/org/${orgId}/ai-provider`, toAiProviderCreatePayload(values)); + const providerId = res.data.data.provider.providerId; + + if (showTargets && targets.length > 0) { + try { + await createTargets(providerId, targets); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiProviderErrorCreate"), + description: formatAxiosError( + e, + t("aiProviderErrorCreate") + ) + }); + router.push( + `/${orgId}/settings/ai-providers/${providerId}/network` + ); + return; + } + } + toast({ title: t("success"), description: t("aiProviderCreated") }); - router.push( - `/${orgId}/settings/ai-providers/${res.data.data.provider.providerId}` - ); + router.push(`/${orgId}/settings/ai-providers/${providerId}`); } catch (e) { toast({ variant: "destructive", @@ -134,91 +214,143 @@ export default function CreateAiProviderPage() { - - - - - {t("aiProviderGeneral")} - - - {t("aiProviderGeneralDescription")} - - +
+ + + + + {t("aiProviderGeneral")} + + + {t("aiProviderGeneralDescription")} + + - - - - - + + + + + ( + + + {t("name")} + + + + + + + )} + /> + + + + ( + + + {t("aiProviderType")} + + + { + field.onChange( + value + ); + form.setValue( + "upstreamUrl", + emptyUpstreamForType( + value + ) + ); + if ( + value !== + "custom" + ) { + form.setValue( + "routingMode", + "url" + ); + targetsRef.current = + []; + } + }} + /> + + + + )} + /> + + + + + + + + + + {t("aiProviderNetworkSettings")} + + + {t("aiProviderNetworkSettingsDescription")} + + + + + + + {showRoutingMode && ( ( - - - - - - - )} - /> - - - - ( - - - {t("name")} - - - - - - - )} - /> - - - - ( {t( - "aiProviderType" + "aiProviderRoutingMode" )} - + + {t( + "aiProviderRoutingModeDescription" + )} + )} /> + )} - {showRoutingMode && ( - - ( - - - {t( - "aiProviderRoutingMode" - )} - - - { - field.onChange( - value - ); - if ( - value === - "target" - ) { - form.setValue( - "upstreamUrl", - "" - ); - } - }} - /> - - - {t( - "aiProviderRoutingModeDescription" - )} - - - - )} - /> - - )} - - {showTargetNote && ( - - - - - {t( - "aiProviderRoutingModeTarget" - )} - - - {t( - "aiProviderRoutingModeTargetNote" - )} - - - - )} - - {showUpstream && ( - - ( - - - {t( - "aiProviderUpstreamUrl" - )} - {requireUpstream - ? "" - : " (optional)"} - - - - - - {requireUpstream - ? t( - "aiProviderUpstreamUrlDescription" - ) - : t( - "aiProviderUpstreamUrlOptionalDescription" - )} - - - - )} - /> - - )} - - {showAuthType && ( - - ( - - - {t( - "aiProviderAuthType" - )} - - - - {t( - "aiProviderAuthTypeDescription" - )} - - - - )} - /> - - )} - + {showUpstream && ( ( {t( - "aiProviderApiKey" + "aiProviderUpstreamUrl" )} + {requireUpstream + ? "" + : " (optional)"} + {requireUpstream + ? t( + "aiProviderUpstreamUrlDescription" + ) + : t( + "aiProviderUpstreamUrlOptionalDescription" + )} + + + + )} + /> + + )} + + + ( + + + + + + + )} + /> + + + + + {showTargets && ( +
+ + + {t("targets")} + + + {t("targetsDescription")} + + + { + targetsRef.current = nextTargets; + }} + allowedMethods={["http", "https"]} + emptyMessage={t( + "aiProviderTargetNoOne" + )} + embedded + hideSaveButton + /> +
+ )} + + + + + + + {t("aiProviderAuthSettings")} + + + {t("aiProviderAuthSettingsDescription")} + + + + + + + {showAuthType && ( + + ( + + {t( - "aiProviderApiKeyDescription" + "aiProviderAuthType" + )} + + + + {t( + "aiProviderAuthTypeDescription" )} @@ -462,53 +546,68 @@ export default function CreateAiProviderPage() { )} /> + )} - - ( - - - - - - - )} - /> - - - - - - - - - - -
+ + ( + + + {t("aiProviderApiKey")} + + + + + + {t( + "aiProviderApiKeyDescription" + )} + + + + )} + /> + + + + + + + +
+ + +
+ ); } diff --git a/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx b/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx index 0db4cb156..db3bec315 100644 --- a/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx +++ b/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx @@ -42,7 +42,7 @@ import { toast } from "@app/hooks/useToast"; import { createApiClient } from "@app/lib/api"; import { formatAxiosError } from "@app/lib/api/formatAxiosError"; import { DockerManager, DockerState } from "@app/lib/docker"; -import { orgQueries, resourceQueries } from "@app/lib/queries"; +import { orgQueries, resourceQueries, aiProviderQueries } from "@app/lib/queries"; import { build } from "@server/build"; import { type GetResourceResponse } from "@server/routers/resource"; import { CreateTargetResponse } from "@server/routers/target"; @@ -63,9 +63,11 @@ import { ExternalLink, Info, Plus } from "lucide-react"; import { useTranslations } from "next-intl"; import { useRouter } from "next/navigation"; import { + forwardRef, useActionState, useCallback, useEffect, + useImperativeHandle, useMemo, useState } from "react"; @@ -80,27 +82,52 @@ export type LocalTarget = Omit< "protocol" >; -interface ProxyResourceTargetsFormProps { +export type ProxyResourceTargetsFormHandle = { + save: (options?: { silent?: boolean }) => Promise; +}; + +type ProxyResourceTargetsFormProps = { orgId: string; isHttp: boolean; initialTargets?: LocalTarget[]; - /** Edit mode: when provided, shows a save button and polls for health status */ + /** Edit mode for a public resource: save button + health polling */ resource?: GetResourceResponse; + /** Edit mode for an AI provider: save button + health polling */ + providerId?: number; updateResource?: ResourceContextType["updateResource"]; /** Create mode: called whenever the targets list changes */ onChange?: (targets: LocalTarget[]) => void; -} + /** HTTP method options for address selector. Defaults to http/https/h2c. */ + allowedMethods?: ("http" | "https" | "h2c")[]; + emptyMessage?: string; + /** Render table without its own SettingsSection wrapper */ + embedded?: boolean; + /** Hide the built-in save button (use ref.save from parent) */ + hideSaveButton?: boolean; +}; -export function ProxyResourceTargetsForm({ - orgId, - isHttp, - initialTargets = [], - resource, - updateResource, - onChange -}: ProxyResourceTargetsFormProps) { +export const ProxyResourceTargetsForm = forwardRef< + ProxyResourceTargetsFormHandle, + ProxyResourceTargetsFormProps +>(function ProxyResourceTargetsForm( + { + orgId, + isHttp, + initialTargets = [], + resource, + providerId, + updateResource, + onChange, + allowedMethods = ["http", "https", "h2c"], + emptyMessage, + embedded = false, + hideSaveButton = false + }, + ref +) { const t = useTranslations(); const api = createApiClient(useEnvContext()); + const isEditMode = !!resource || !!providerId; const [targets, setTargets] = useState(initialTargets); const [targetsToRemove, setTargetsToRemove] = useState([]); @@ -111,7 +138,7 @@ export function ProxyResourceTargetsForm({ }, [targets]); // Poll health status only in edit mode - const { data: polledTargets } = useQuery({ + const { data: polledResourceTargets } = useQuery({ ...resourceQueries.resourceTargets({ resourceId: resource?.resourceId ?? 0 }), @@ -119,6 +146,18 @@ export function ProxyResourceTargetsForm({ enabled: !!resource }); + const { data: polledProviderTargets } = useQuery({ + ...aiProviderQueries.providerTargets({ + providerId: providerId ?? 0 + }), + refetchInterval: 10_000, + enabled: !!providerId + }); + + const polledTargets = providerId + ? polledProviderTargets + : polledResourceTargets; + useEffect(() => { if (!polledTargets) return; setTargets((prev) => @@ -427,6 +466,7 @@ export function ProxyResourceTargetsForm({ isHttp={isHttp} proxyTarget={row.original} updateTarget={updateTarget} + allowedMethods={allowedMethods} /> ); }, @@ -576,16 +616,22 @@ export function ProxyResourceTargetsForm({ refreshContainersForSite, openHealthCheckDialog, removeTarget, + allowedMethods, t ]); function addNewTarget() { + const defaultMethod = providerId + ? (allowedMethods[0] ?? "https") + : isHttp + ? "http" + : null; const newTarget: LocalTarget = { targetId: -Date.now(), ip: "", mode: ((resource?.mode as LocalTarget["mode"]) ?? (isHttp ? "http" : "tcp")) as LocalTarget["mode"], - method: isHttp ? "http" : null, + method: defaultMethod, port: 0, siteId: sites.length > 0 ? sites[0].siteId : 0, siteName: sites.length > 0 ? sites[0].name : "", @@ -595,8 +641,8 @@ export function ProxyResourceTargetsForm({ rewritePathType: null, priority: 100, enabled: true, - resourceId: resource?.resourceId ?? 0, - providerId: null, + resourceId: resource?.resourceId ?? null, + providerId: providerId ?? null, hcEnabled: false, hcPath: null, hcMethod: null, @@ -671,10 +717,16 @@ export function ProxyResourceTargetsForm({ } }, [isAdvancedMode]); - const [, formAction, isSubmitting] = useActionState(saveTargets, null); + const [, formAction, isSubmitting] = useActionState( + async () => { + await saveTargets(); + return null; + }, + null + ); const addTargetButton = ( - @@ -682,8 +734,8 @@ export function ProxyResourceTargetsForm({ const hasTargets = targets.length > 0; - async function saveTargets() { - if (!resource) return; + async function saveTargets(options?: { silent?: boolean }) { + if (!isEditMode) return true; const targetsWithInvalidFields = targets.filter( (target) => @@ -699,7 +751,7 @@ export function ProxyResourceTargetsForm({ title: t("targetErrorInvalidIp"), description: t("targetErrorInvalidIpDescription") }); - return; + return false; } try { @@ -743,9 +795,12 @@ export function ProxyResourceTargetsForm({ } if (target.new) { + const createPath = providerId + ? `/ai-provider/${providerId}/target` + : `/resource/${resource!.resourceId}/target`; const res = await api.put< AxiosResponse - >(`/resource/${resource.resourceId}/target`, data); + >(createPath, data); target.targetId = res.data.data.targetId; target.new = false; } else if (target.updated) { @@ -754,24 +809,33 @@ export function ProxyResourceTargetsForm({ } } - toast({ - title: - targets.length === 0 - ? t("targetTargetsCleared") - : t("settingsUpdated"), - description: - targets.length === 0 - ? t("targetTargetsClearedDescription") - : t("settingsUpdatedDescription") - }); + if (!options?.silent) { + toast({ + title: + targets.length === 0 + ? t("targetTargetsCleared") + : t("settingsUpdated"), + description: + targets.length === 0 + ? t("targetTargetsClearedDescription") + : t("settingsUpdatedDescription") + }); + } setTargetsToRemove([]); router.refresh(); - await queryClient.invalidateQueries( - resourceQueries.resourceTargets({ - resourceId: resource.resourceId - }) - ); + if (providerId) { + await queryClient.invalidateQueries( + aiProviderQueries.providerTargets({ providerId }) + ); + } else if (resource) { + await queryClient.invalidateQueries( + resourceQueries.resourceTargets({ + resourceId: resource.resourceId + }) + ); + } + return true; } catch (err) { console.error(err); toast({ @@ -782,151 +846,164 @@ export function ProxyResourceTargetsForm({ t("settingsErrorUpdateDescription") ) }); + return false; } } + useImperativeHandle(ref, () => ({ + save: saveTargets + })); + + const advancedModeToggleId = providerId + ? `advanced-mode-toggle-provider-${providerId}` + : resource + ? `advanced-mode-toggle-resource-${resource.resourceId}` + : "advanced-mode-toggle"; + + const targetsTable = ( + <> +
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + const isActionsColumn = + header.column.id === "actions"; + const isSiteColumn = + header.column.id === "site"; + return ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef + .header, + header.getContext() + )} + + ); + })} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => { + const isActionsColumn = + cell.column.id === "actions"; + const isSiteColumn = + cell.column.id === "site"; + return ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext() + )} + + ); + })} + + )) + ) : ( + + )} + +
+
+ {hasTargets && ( +
+
+ {addTargetButton} +
+ + +
+
+
+ )} + {build === "saas" && + targets.length > 1 && + new Set(targets.map((t) => t.siteId)).size > 1 && ( +

+ {t("proxyMultiSiteRoundRobinNodeHelp")}{" "} + + {t("learnMore")} + + + . +

+ )} + + ); + return ( <> - - - {t("targets")} - - {t("targetsDescription")} - - - -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - const isActionsColumn = - header.column.id === "actions"; - const isSiteColumn = - header.column.id === "site"; - return ( - - {header.isPlaceholder - ? null - : flexRender( - header.column - .columnDef - .header, - header.getContext() - )} - - ); - })} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row - .getVisibleCells() - .map((cell) => { - const isActionsColumn = - cell.column.id === - "actions"; - const isSiteColumn = - cell.column.id === - "site"; - return ( - - {flexRender( - cell.column - .columnDef - .cell, - cell.getContext() - )} - - ); - })} - - )) - ) : ( - - )} - -
-
- {hasTargets && ( -
-
- {addTargetButton} -
- - -
-
-
- )} - {build === "saas" && - targets.length > 1 && - new Set(targets.map((t) => t.siteId)).size > 1 && ( -

- {t("proxyMultiSiteRoundRobinNodeHelp")}{" "} - - {t("learnMore")} - - - . -

- )} - + {embedded ? ( +
{targetsTable}
+ ) : ( + + + + {t("targets")} + + + {t("targetsDescription")} + + + {targetsTable} - {/* Save button — only shown in edit mode */} - {resource && ( -
- -
- )} - + {isEditMode && !hideSaveButton && ( +
+ +
+ )} + + )} {selectedTargetForHealthCheck && ( ); -} +}); diff --git a/src/app/navigation.tsx b/src/app/navigation.tsx index cb91bd734..e6971bc94 100644 --- a/src/app/navigation.tsx +++ b/src/app/navigation.tsx @@ -31,6 +31,7 @@ import { TicketCheck, Unplug, User, + UserCheck, UserCog, Users, Waypoints @@ -176,7 +177,7 @@ export const orgNavSections = ( { title: "sidebarApprovals", href: "/{orgId}/settings/access/approvals", - icon: + icon: } ] : []), @@ -188,7 +189,7 @@ export const orgNavSections = ( ] }, { - heading: "sidebarAi", + heading: "sidebarAiGateway", items: [ { title: "sidebarAiProviders", @@ -483,7 +484,7 @@ export const commandBarNavSections = ( ] }, { - heading: "sidebarAi", + heading: "sidebarAiGateway", items: [ { title: "commandAiProviders", diff --git a/src/components/AiProviderTypeSelect.tsx b/src/components/AiProviderTypeSelect.tsx index 72e076f53..ed0565f53 100644 --- a/src/components/AiProviderTypeSelect.tsx +++ b/src/components/AiProviderTypeSelect.tsx @@ -83,20 +83,14 @@ export function AiProviderTypeSelect({ aria-expanded={open} disabled={disabled} className={cn( - "w-full justify-between font-normal h-auto min-h-10 py-2", + "w-full justify-between", + !selected && "text-muted-foreground", className )} > -
- - {selected?.title ?? t("noneSelected")} - - {selected?.description && ( - - {selected.description} - - )} -
+ + {selected?.title ?? t("noneSelected")} + diff --git a/src/components/resource-target-address-item.tsx b/src/components/resource-target-address-item.tsx index 68acec7f1..58ef3b33b 100644 --- a/src/components/resource-target-address-item.tsx +++ b/src/components/resource-target-address-item.tsx @@ -133,12 +133,14 @@ export type ResourceTargetAddressItemProps = { updateTarget: (targetId: number, data: Partial) => void; proxyTarget: LocalTarget; isHttp: boolean; + allowedMethods?: ("http" | "https" | "h2c")[]; }; export function ResourceTargetAddressItem({ updateTarget, proxyTarget, - isHttp + isHttp, + allowedMethods = ["http", "https", "h2c"] }: ResourceTargetAddressItemProps) { return (
@@ -157,9 +159,15 @@ export function ResourceTargetAddressItem({ {proxyTarget.method || "http"} - http - https - h2c + {allowedMethods.includes("http") && ( + http + )} + {allowedMethods.includes("https") && ( + https + )} + {allowedMethods.includes("h2c") && ( + h2c + )} )} diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index 184be14c9..8861dcdbf 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -71,7 +71,7 @@ export const aiProviderFormSchema = z }); } - if (data.type === "custom" && routingMode === "url" && !data.authType) { + if (data.type === "custom" && !data.authType) { ctx.addIssue({ code: "custom", message: "authType is required for custom providers", @@ -96,6 +96,18 @@ export const aiProviderFormSchema = z export type AiProviderFormValues = z.infer; +export const aiProviderCreateFormSchema = aiProviderFormSchema.superRefine( + (data, ctx) => { + if (!data.apiKey?.trim()) { + ctx.addIssue({ + code: "custom", + message: "API key is required", + path: ["apiKey"] + }); + } + } +); + export function emptyUpstreamForType(type: AiProviderType): string { if (type === "custom") { return ""; @@ -145,7 +157,7 @@ export function toAiProviderCreatePayload(values: AiProviderFormValues) { upstreamUrl, apiKey: values.apiKey?.trim() ? values.apiKey.trim() : undefined, authType: - values.type === "custom" && routingMode === "url" + values.type === "custom" ? (values.authType ?? "bearer") : (values.authType ?? undefined), skipTlsVerification: values.skipTlsVerification, @@ -176,7 +188,7 @@ export function toAiProviderUpdatePayload(values: AiProviderFormValues) { routingMode: values.type === "custom" ? routingMode : "url", upstreamUrl, authType: - values.type === "custom" && routingMode === "url" + values.type === "custom" ? (values.authType ?? "bearer") : (values.authType ?? null), skipTlsVerification: values.skipTlsVerification ?? false, @@ -192,6 +204,23 @@ export function toAiProviderUpdatePayload(values: AiProviderFormValues) { return payload; } +export function toAiProviderNetworkPayload(values: AiProviderFormValues) { + const full = toAiProviderUpdatePayload(values); + return { + routingMode: full.routingMode, + upstreamUrl: full.upstreamUrl, + skipTlsVerification: full.skipTlsVerification + }; +} + +export function toAiProviderAuthPayload(values: AiProviderFormValues) { + const full = toAiProviderUpdatePayload(values); + return { + authType: full.authType, + ...(values.apiKey !== undefined ? { apiKey: values.apiKey.trim() } : {}) + }; +} + export function toAiProviderConfigurationPayload(values: AiProviderFormValues) { const { name: _name, diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 96024c3a0..bdc2f413a 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -1163,6 +1163,20 @@ export const logQueries = { }) }; +export const aiProviderQueries = { + providerTargets: ({ providerId }: { providerId: number }) => + queryOptions({ + queryKey: ["AI_PROVIDERS", providerId, "TARGETS"] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/ai-provider/${providerId}/targets`, { signal }); + + return res.data.data.targets; + } + }) +}; + export const resourceQueries = { resourceUsers: ({ resourceId }: { resourceId: number }) => queryOptions({ From 9b25ebd6cdca48efbb228a30f115c6af4d67d2d4 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 10:19:22 -0400 Subject: [PATCH 023/296] remove token fallback --- server/routers/aiGateway/chatCompletions.ts | 45 +++------------------ 1 file changed, 5 insertions(+), 40 deletions(-) diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index 7a4ba3a84..d8ec7f37c 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -36,14 +36,6 @@ type ResolvedTarget = { allowedModelIds: number[] | null; }; -// Fallback for clients that hit the endpoint directly (e.g. an AI tool's -// "API key" field) instead of going through a browser session - badger -// forwards whatever Authorization header the client sent untouched in that -// case. This prefix lets us tell "this bearer value is a Pangolin resource -// access token" apart from an arbitrary/opaque API key a user might paste -// in, without guessing based on format alone. -const USER_TOKEN_PREFIX = "pu_"; - export type RequestUser = { userId: string; username: string; @@ -94,37 +86,7 @@ async function resolveRequestUser( } } - // User devices hitting the endpoint directly (no browser session to - // forward) fall back to a Pangolin resource access token passed as the - // client's "API key". - const authHeader = req.headers["authorization"]; - if (typeof authHeader !== "string" || !resourceId) { - return null; - } - - const bearer = authHeader.match(/^Bearer\s+(.+)$/i)?.[1]; - if (!bearer || !bearer.startsWith(USER_TOKEN_PREFIX)) { - return null; - } - - const [accessTokenId, accessToken] = bearer - .slice(USER_TOKEN_PREFIX.length) - .split("."); - if (!accessTokenId || !accessToken) { - return null; - } - - const { valid, tokenItem } = await verifyResourceAccessToken({ - accessToken, - accessTokenId, - resourceId - }); - - if (!valid || !tokenItem?.userId) { - return null; - } - - return buildRequestUser(tokenItem.userId, tokenItem.orgId); + return null; } async function resolveTarget(host: string): Promise { @@ -222,7 +184,10 @@ function getCompletionsPath(type: AiProviderType): string { return "/chat/completions"; } -export async function chatCompletions(req: Request, res: Response): Promise { +export async function chatCompletions( + req: Request, + res: Response +): Promise { try { const host = (req.headers.host || "").split(":")[0]; if (!host) { From a7e44944fb1bb4f74375e545e264e9b0f0b3723c Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 10:43:00 -0400 Subject: [PATCH 024/296] Use ip to find user --- server/db/pg/schema/schema.ts | 4 +- server/routers/aiGateway/chatCompletions.ts | 95 ++++++++++++++++++++- 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index e253cd6d5..51552233c 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -99,7 +99,7 @@ export const sites = pgTable( name: varchar("name").notNull(), pubKey: varchar("pubKey"), subnet: varchar("subnet"), - exitNodeSubnet: text("exitNodeSubnet"), // this is the subnet when connecting to an exit node + exitNodeSubnet: text("exitNodeSubnet"), // this is the subnet when connecting to an exit node and INCLUDES THE CIDR megabytesIn: real("bytesIn").default(0), megabytesOut: real("bytesOut").default(0), lastBandwidthUpdate: varchar("lastBandwidthUpdate"), @@ -1234,7 +1234,7 @@ export const clients = pgTable( olmId: text("olmId"), // to lock it to a specific olm optionally name: varchar("name").notNull(), pubKey: varchar("pubKey"), - exitNodeSubnet: varchar("exitNodeSubnet").notNull(), + exitNodeSubnet: varchar("exitNodeSubnet").notNull(), // INCLUDES THE CIDR megabytesIn: real("bytesIn"), megabytesOut: real("bytesOut"), lastBandwidthUpdate: varchar("lastBandwidthUpdate"), diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index d8ec7f37c..c9c32c2ba 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -4,7 +4,9 @@ import { AiProvider, aiModels, aiProviders, + clients, db, + exitNodes, resourceAiModels, resources, siteResourceAiModels, @@ -25,9 +27,64 @@ import { validateSessionToken } from "@server/auth/sessions/app"; import { getUserOrgRoles } from "@server/lib/userOrgRoles"; +import { isIpInCidr } from "@server/lib/ip"; +import { localCache } from "@server/lib/cache"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; +// Short-lived local caches so a burst of requests from the same IP/user +// doesn't hit the database on every single request. None of this is +// security-critical to cache aggressively (identity is re-derived from the +// session cookie or from a client's exit-node-scoped subnet each time), so +// a small TTL is just an efficiency win, not a trust boundary. +const EXIT_NODE_RANGES_CACHE_KEY = "aiGateway:exitNodeRanges"; +const EXIT_NODE_RANGES_TTL_SEC = 6000; +const CLIENT_BY_IP_TTL_SEC = 30; +const REQUEST_USER_TTL_SEC = 30; + +type CachedClient = { clientId: number; userId: string | null } | null; + +// The set of CIDRs an exit node manages; client exitNodeSubnets are always +// /32s carved out of one of these ranges. Checking against this small, +// cacheable list lets us skip the (much more frequent) per-IP client lookup +// entirely for traffic that could never match a client anyway. +async function getExitNodeRanges(): Promise { + const cached = localCache.get(EXIT_NODE_RANGES_CACHE_KEY); + if (cached) { + return cached; + } + + const rows = await db + .select({ address: exitNodes.address }) + .from(exitNodes); + const ranges = rows.map((r) => r.address); + + localCache.set( + EXIT_NODE_RANGES_CACHE_KEY, + ranges, + EXIT_NODE_RANGES_TTL_SEC + ); + return ranges; +} + +async function findClientByIp(ip: string): Promise { + const cacheKey = `aiGateway:clientByIp:${ip}`; + const cached = localCache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + + const [client] = await db + .select({ clientId: clients.clientId, userId: clients.userId }) + .from(clients) + .where(eq(clients.exitNodeSubnet, `${ip}/32`)) + .limit(1); + + const result: CachedClient = client || null; + localCache.set(cacheKey, result, CLIENT_BY_IP_TTL_SEC); + return result; +} + type ResolvedTarget = { resourceId: number | null; orgId: string | null; @@ -48,6 +105,12 @@ async function buildRequestUser( userId: string, orgId: string | null ): Promise { + const cacheKey = `aiGateway:requestUser:${userId}:${orgId || ""}`; + const cached = localCache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + const [user] = await db .select() .from(users) @@ -55,18 +118,22 @@ async function buildRequestUser( .limit(1); if (!user) { + localCache.set(cacheKey, null, REQUEST_USER_TTL_SEC); return null; } const orgRoles = orgId ? await getUserOrgRoles(user.userId, orgId) : []; - return { + const requestUser: RequestUser = { userId: user.userId, username: user.username, email: user.email, name: user.name, role: orgRoles.map((r) => r.roleName).join(", ") || null }; + + localCache.set(cacheKey, requestUser, REQUEST_USER_TTL_SEC); + return requestUser; } async function resolveRequestUser( @@ -86,7 +153,31 @@ async function resolveRequestUser( } } - return null; + // TODO: MAKE SURE THIS CAN NOT BE SPOOFED AND CAN BE TRUSTED AS AN INTERNAL ADDRESS FROM A NODE + + // No session cookie - fall back to identifying the caller by source IP. + // A client's exitNodeSubnet is a /32 handed out from one of our exit + // node's address ranges, so an IP that isn't inside any of those ranges + // can never belong to a client and we can skip the DB entirely. + const ip = req.ip; + if (!ip) { + return null; + } + + const exitNodeRanges = await getExitNodeRanges(); + const inExitNodeRange = exitNodeRanges.some((range) => + isIpInCidr(ip, range) + ); + if (!inExitNodeRange) { + return null; + } + + const client = await findClientByIp(ip); + if (!client || !client.userId) { + return null; + } + + return buildRequestUser(client.userId, orgId); } async function resolveTarget(host: string): Promise { From 0b30cfc341984b8de49012b7ec743dff3d5070e2 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 11:48:36 -0400 Subject: [PATCH 025/296] Working on inference resource management --- server/lib/rebuildClientAssociations.ts | 19 +++++--- server/routers/olm/sync.ts | 43 +++++++++++++++---- .../siteResource/createSiteResource.ts | 8 ++-- .../siteResource/updateSiteResource.ts | 24 ++++++----- .../private/[niceId]/inference/page.tsx | 2 - src/lib/privateResourceForm.ts | 20 ++++++++- 6 files changed, 87 insertions(+), 29 deletions(-) diff --git a/server/lib/rebuildClientAssociations.ts b/server/lib/rebuildClientAssociations.ts index 194bfe446..63a71382d 100644 --- a/server/lib/rebuildClientAssociations.ts +++ b/server/lib/rebuildClientAssociations.ts @@ -1177,12 +1177,12 @@ async function syncClientExitNodeConnections( const connectPayloads: { clientId: string; message: { type: string; data: any }; - options: { compress: boolean }; + options: { compress: boolean; incrementConfigVersion: boolean }; }[] = []; const disconnectPayloads: { clientId: string; message: { type: string; data: any }; - options: { compress: boolean }; + options: { compress: boolean; incrementConfigVersion: boolean }; }[] = []; for (const client of clientsData) { @@ -1218,7 +1218,10 @@ async function syncClientExitNodeConnections( tunnelIP: client.exitNodeSubnet.split("/")[0] } }, - options: { compress: canCompress(olm.version, "olm") } + options: { + compress: canCompress(olm.version, "olm"), + incrementConfigVersion: true + } }); } else { disconnectPayloads.push({ @@ -1227,7 +1230,10 @@ async function syncClientExitNodeConnections( type: "olm/wg/exitnode/disconnect", data: {} }, - options: { compress: canCompress(olm.version, "olm") } + options: { + compress: canCompress(olm.version, "olm"), + incrementConfigVersion: true + } }); } } @@ -1293,7 +1299,10 @@ async function syncClientExitNodeAliasUpdate( newAliases } }, - options: { compress: canCompress(olm.version, "olm") } + options: { + compress: canCompress(olm.version, "olm"), + incrementConfigVersion: true // this is important information we would need to sync + } })); if (updatePayloads.length > 0) { diff --git a/server/routers/olm/sync.ts b/server/routers/olm/sync.ts index 46e1fbd88..19d759769 100644 --- a/server/routers/olm/sync.ts +++ b/server/routers/olm/sync.ts @@ -1,6 +1,7 @@ import { Client, db, + ExitNode, exitNodes, Olm, sites, @@ -48,12 +49,26 @@ export async function sendOlmSyncMessage(olm: Olm, client: Client) { } // NOTE: WE ARE HARDCODING THE RELAY PARAMETER TO FALSE HERE BUT IN THE REGISTER MESSAGE ITS DEFINED BY THE CLIENT - const siteConfigurations = await buildSiteConfigurationForOlmClient( - client, - client.pubKey, - false, - jitMode - ); + const { siteConfigurations, exitNodeAliases } = + await buildSiteConfigurationForOlmClient( + client, + client.pubKey, + false, + jitMode + ); + + // The exit node the client itself is assigned to (for site resources hosted + // on it, e.g. inference), same as what's sent in the initial olm/wg/connect + // message. This is separate from exitNodesData below, which is only the set + // of exit nodes used for hole punching to reach site peers. + let clientExitNode: ExitNode | null = null; + if (client.exitNodeId) { + [clientExitNode] = await db + .select() + .from(exitNodes) + .where(eq(exitNodes.exitNodeId, client.exitNodeId)) + .limit(1); + } // Get all exit nodes from sites where the client has peers const clientSites = await db @@ -113,11 +128,23 @@ export async function sendOlmSyncMessage(olm: Olm, client: Client) { type: "olm/sync", data: { sites: siteConfigurations, - exitNodes: exitNodesData + exitNodes: exitNodesData, // this is for the holepunch information + // this is for the backhaul connection to the exit node + exitNode: + clientExitNode && client.exitNodeSubnet + ? { + aliases: exitNodeAliases, + connect: exitNodeAliases.length > 0, // we do not need to connect to the exit node if we do not have inference resources and right now all site resources on the exit node have an alias + endpoint: `${clientExitNode.endpoint}:${clientExitNode.listenPort}`, + publicKey: clientExitNode.publicKey, + serverIP: clientExitNode.address.split("/")[0], + tunnelIP: client.exitNodeSubnet.split("/")[0] + } + : undefined } }, { - compress: canCompress(olm.version, "olm") + compress: canCompress(olm.version, "olm") // we dont increment the version here or we could get into a loop! } ).catch((error) => { logger.warn(`Error sending olm sync message:`, error); diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 9a6f91ad7..b80775b49 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -571,7 +571,7 @@ export async function createSiteResource( } let tcpPortRangeStringAdjusted = tcpPortRangeString; - if (mode === "http") { + if (mode === "http" || mode === "inference") { tcpPortRangeStringAdjusted = "443,80"; } else if (mode === "ssh") { tcpPortRangeStringAdjusted = destinationPort @@ -594,12 +594,14 @@ export async function createSiteResource( aliasAddress, tcpPortRangeString: tcpPortRangeStringAdjusted, udpPortRangeString: - mode == "http" || mode == "ssh" + mode == "http" || mode == "ssh" || mode == "inference" ? "" : udpPortRangeString, disableIcmp: disableIcmp || - (mode == "http" || mode == "ssh" ? true : false), // default to true for http resources, otherwise false + (mode == "http" || mode == "ssh" || mode == "inference" + ? true + : false), // default to true for http resources, otherwise false domainId, subdomain: finalSubdomain, fullDomain, diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index 0ae57c087..caf524804 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -166,8 +166,11 @@ const updateSiteResourceSchema = z if (data.mode === undefined && data.destination === undefined) { return true; } - // destination is only optional for ssh mode with native authDaemonMode - if (data.mode === "ssh" && data.authDaemonMode === "native") { + // destination is only optional for ssh mode with native authDaemonMode or inference + if ( + (data.mode === "ssh" && data.authDaemonMode === "native") || + data.mode == "inference" + ) { return true; } return ( @@ -558,8 +561,9 @@ export async function updateSiteResource( }) } : {}; + let tcpPortRangeStringAdjusted = tcpPortRangeString; - if (mode === "http") { + if (mode === "http" || mode == "inference") { tcpPortRangeStringAdjusted = "443,80"; } else if (mode === "ssh") { tcpPortRangeStringAdjusted = destinationPort @@ -583,20 +587,20 @@ export async function updateSiteResource( ? alias ? alias.trim() : null - : mode !== undefined && - mode !== "host" && - mode !== "ssh" - ? null - : undefined, + : undefined, tcpPortRangeString: tcpPortRangeStringAdjusted, udpPortRangeString: - mode == "http" || mode == "ssh" + mode == "http" || mode == "ssh" || mode == "inference" ? "" : udpPortRangeString, disableIcmp: mode !== undefined ? disableIcmp || - (mode == "http" || mode == "ssh" ? true : false) + (mode == "http" || + mode == "ssh" || + mode == "inference" + ? true + : false) : disableIcmp, domainId, subdomain: finalSubdomain, diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx index 3f27f1172..24aab5067 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx @@ -20,9 +20,7 @@ import { useTranslations } from "next-intl"; import { useActionState, useMemo, useState } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod"; -import { PrivateResourceSitesField } from "@app/components/PrivateResourceSitesField"; import { PrivateResourceInferenceDestinationFields } from "@app/components/PrivateResourceDestinationFields"; -import { PrivateResourcePortRanges } from "@app/components/PrivateResourcePortRanges"; import { useSaveSiteResource } from "@app/hooks/useSaveSiteResource"; import { asAnyControl, diff --git a/src/lib/privateResourceForm.ts b/src/lib/privateResourceForm.ts index f2ff103f6..2f11b6e7b 100644 --- a/src/lib/privateResourceForm.ts +++ b/src/lib/privateResourceForm.ts @@ -211,6 +211,14 @@ export function buildCreateSiteResourcePayload( authDaemonPort: data.authDaemonPort }) }), + ...(data.mode === "inference" && { + alias: + data.alias && + typeof data.alias === "string" && + data.alias.trim() + ? data.alias + : undefined + }), ...((data.mode === "host" || data.mode === "cidr") && { tcpPortRangeString: data.tcpPortRangeString, udpPortRangeString: data.udpPortRangeString, @@ -237,7 +245,9 @@ export function buildUpdateSiteResourcePayload( enabled: data.enabled, ...(isNativeSsh ? { destination: null, destinationPort: null } - : { destination: data.destination ?? undefined }), + : data.mode !== "inference" + ? { destination: data.destination ?? undefined } + : {}), ...(data.mode === "http" && { scheme: data.scheme, ssl: data.ssl ?? false, @@ -281,6 +291,14 @@ export function buildUpdateSiteResourcePayload( authDaemonPort: data.authDaemonPort || null }) }), + ...(data.mode === "inference" && { + alias: + data.alias && + typeof data.alias === "string" && + data.alias.trim() + ? data.alias + : null + }), ...((data.mode === "host" || data.mode === "cidr") && { tcpPortRangeString: data.tcpPortRangeString, udpPortRangeString: data.udpPortRangeString, From 973925b35d8c3b17bb162498749880ae7c5e7192 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 14:22:36 -0400 Subject: [PATCH 026/296] allow creating basic inference resource --- messages/en-US.json | 1 + .../siteResource/createSiteResource.ts | 5 +++- .../resources/private/create/page.tsx | 26 ++++++++++++++++- src/lib/privateResourceForm.ts | 28 ++++++++++++++----- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index d6759f822..81a60f0a1 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -2554,6 +2554,7 @@ "createInternalResourceDialogModeHttp": "HTTP", "createInternalResourceDialogModeHttps": "HTTPS", "createInternalResourceDialogModeSsh": "SSH", + "createInternalResourceDialogModeInference": "Inference", "scheme": "Scheme", "createInternalResourceDialogScheme": "Scheme", "createInternalResourceDialogEnableSsl": "Enable TLS", diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index b80775b49..3244668f5 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -164,7 +164,10 @@ const createSiteResourceSchema = z .refine( (data) => { // destination is only optional for ssh mode with native authDaemonMode - if (data.mode === "ssh" && data.authDaemonMode === "native") { + if ( + (data.mode === "ssh" && data.authDaemonMode === "native") || + data.mode == "inference" + ) { return true; } return ( diff --git a/src/app/[orgId]/settings/resources/private/create/page.tsx b/src/app/[orgId]/settings/resources/private/create/page.tsx index 7e6144172..936409d40 100644 --- a/src/app/[orgId]/settings/resources/private/create/page.tsx +++ b/src/app/[orgId]/settings/resources/private/create/page.tsx @@ -153,7 +153,11 @@ export default function CreatePrivateResourcePage() { label: t("createInternalResourceDialogModeSsh") } ] - : []) + : []), + { + value: "inference" as const, + label: t("createInternalResourceDialogModeInference") + } ]; const submitDisabled = @@ -313,6 +317,25 @@ export default function CreatePrivateResourcePage() { "destinationPort", 443 ); + } else if ( + newMode === + "inference" + ) { + form.setValue( + "siteIds", + [] + ); + setSelectedSites( + [] + ); + form.setValue( + "destination", + null + ); + form.setValue( + "destinationPort", + null + ); } else { form.setValue( "destinationPort", @@ -377,6 +400,7 @@ export default function CreatePrivateResourcePage() { )} {(mode === "host" || + mode === "inference" || (mode === "ssh" && !isNativeSsh)) && ( Date: Tue, 4 Aug 2026 14:23:25 -0400 Subject: [PATCH 027/296] send the aliases when connecting for the first time --- server/lib/rebuildClientAssociations.ts | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/server/lib/rebuildClientAssociations.ts b/server/lib/rebuildClientAssociations.ts index 63a71382d..fe1cdea6f 100644 --- a/server/lib/rebuildClientAssociations.ts +++ b/server/lib/rebuildClientAssociations.ts @@ -1116,7 +1116,8 @@ async function syncClientExitNodeConnections( const requiresExitNodeRows = await trx .select({ - clientId: clientSiteResourcesAssociationsCache.clientId + clientId: clientSiteResourcesAssociationsCache.clientId, + alias: siteResources.alias }) .from(clientSiteResourcesAssociationsCache) .innerJoin( @@ -1141,6 +1142,20 @@ async function syncClientExitNodeConnections( requiresExitNodeRows.map((r) => r.clientId) ); + // Aliases for every exit-node-backed resource this client can reach, so + // the live connect push carries the same alias list the register/reconnect + // path (buildSiteConfigurationForOlmClient) would compute. + const exitNodeAliasesByClientId = new Map(); + for (const row of requiresExitNodeRows) { + if (row.alias == null) continue; + const existing = exitNodeAliasesByClientId.get(row.clientId); + if (existing) { + existing.push(row.alias); + } else { + exitNodeAliasesByClientId.set(row.clientId, [row.alias]); + } + } + const exitNodeIds = Array.from( new Set( clientsData @@ -1215,7 +1230,10 @@ async function syncClientExitNodeConnections( relayPort, publicKey: exitNode.publicKey, serverIP: exitNode.address.split("/")[0], - tunnelIP: client.exitNodeSubnet.split("/")[0] + tunnelIP: client.exitNodeSubnet.split("/")[0], + aliases: exitNodeAliasesByClientId.get( + client.clientId + ) ?? [] } }, options: { From ed8545f8a252a15ce3a82577b1235567204f62c0 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 4 Aug 2026 14:49:41 -0400 Subject: [PATCH 028/296] update schema to support models and provider access on inferences resources and add budgets --- server/db/pg/schema/schema.ts | 84 +++++++++++++++++++++++++++--- server/db/sqlite/schema/schema.ts | 85 ++++++++++++++++++++++++++++--- 2 files changed, 156 insertions(+), 13 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 51552233c..6a24064a6 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -3,6 +3,7 @@ import { InferSelectModel, sql } from "drizzle-orm"; import { bigint, boolean, + check, index, integer, pgTable, @@ -214,7 +215,10 @@ export const resources = pgTable( aiProviderId: integer("aiProviderId").references( () => aiProviders.providerId, { onDelete: "set null" } - ) + ), + modelAccessMode: varchar("modelAccessMode").$type< + "passthrough" | "catalog" | "allowlist" + >() }, (t) => [ index("idx_resources_fulldomain") @@ -496,7 +500,10 @@ export const siteResources = pgTable( aiProviderId: integer("aiProviderId").references( () => aiProviders.providerId, { onDelete: "set null" } - ) + ), + modelAccessMode: varchar("modelAccessMode").$type< + "passthrough" | "catalog" | "allowlist" + >() }, (t) => [index("idx_siteresources_orgid_niceid").on(t.orgId, t.niceId)] ); @@ -1623,8 +1630,6 @@ export const aiProviders = pgTable("aiProviders", { skipTlsVerification: boolean("skipTlsVerification") .notNull() .default(false), - budgetAmount: real("budgetAmount"), - budgetUnit: varchar("budgetUnit").$type<"usd" | "tokens">(), enabled: boolean("enabled").notNull().default(true), createdAt: bigint("createdAt", { mode: "number" }).notNull(), updatedAt: bigint("updatedAt", { mode: "number" }).notNull() @@ -1639,8 +1644,6 @@ export const aiModels = pgTable( .references(() => aiProviders.providerId, { onDelete: "cascade" }), modelKey: varchar("modelKey").notNull(), name: varchar("name").notNull(), - budgetAmount: real("budgetAmount"), - budgetUnit: varchar("budgetUnit").$type<"usd" | "tokens">(), enabled: boolean("enabled").notNull().default(true), createdAt: bigint("createdAt", { mode: "number" }).notNull(), updatedAt: bigint("updatedAt", { mode: "number" }).notNull() @@ -1648,6 +1651,73 @@ export const aiModels = pgTable( (t) => [unique("ai_model_provider_key_uniq").on(t.providerId, t.modelKey)] ); +export const aiBudgets = pgTable( + "aiBudgets", + { + budgetId: serial("budgetId").primaryKey(), + orgId: varchar("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + providerId: integer("providerId").references( + () => aiProviders.providerId, + { onDelete: "cascade" } + ), + modelId: integer("modelId").references(() => aiModels.modelId, { + onDelete: "cascade" + }), + resourceId: integer("resourceId").references( + () => resources.resourceId, + { onDelete: "cascade" } + ), + siteResourceId: integer("siteResourceId").references( + () => siteResources.siteResourceId, + { onDelete: "cascade" } + ), + amount: real("amount").notNull(), + unit: varchar("unit").$type<"usd" | "tokens">().notNull(), + period: varchar("period") + .$type<"monthly">() + .notNull() + .default("monthly"), + enforcement: varchar("enforcement") + .$type<"hard" | "soft">() + .notNull() + .default("hard"), + enabled: boolean("enabled").notNull().default(true), + createdAt: bigint("createdAt", { mode: "number" }).notNull(), + updatedAt: bigint("updatedAt", { mode: "number" }).notNull() + }, + (t) => [ + check( + "ai_budget_one_scope", + sql`( + (CASE WHEN ${t.providerId} IS NOT NULL THEN 1 ELSE 0 END) + + (CASE WHEN ${t.modelId} IS NOT NULL THEN 1 ELSE 0 END) + + (CASE WHEN ${t.resourceId} IS NOT NULL THEN 1 ELSE 0 END) + + (CASE WHEN ${t.siteResourceId} IS NOT NULL THEN 1 ELSE 0 END) + ) = 1` + ), + unique("ai_budget_provider_uniq").on(t.providerId), + unique("ai_budget_model_uniq").on(t.modelId), + unique("ai_budget_resource_uniq").on(t.resourceId), + unique("ai_budget_site_resource_uniq").on(t.siteResourceId) + ] +); + +export const aiBudgetPeriods = pgTable( + "aiBudgetPeriods", + { + periodId: serial("periodId").primaryKey(), + budgetId: integer("budgetId") + .notNull() + .references(() => aiBudgets.budgetId, { onDelete: "cascade" }), + periodStart: bigint("periodStart", { mode: "number" }).notNull(), + periodEnd: bigint("periodEnd", { mode: "number" }).notNull(), + usedAmount: real("usedAmount").notNull().default(0) + }, + (t) => [unique("ai_budget_period_start_uniq").on(t.budgetId, t.periodStart)] +); + export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1734,5 +1804,7 @@ export type UserPolicy = InferSelectModel; export type ResourcePolicyRule = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; +export type AiBudget = InferSelectModel; +export type AiBudgetPeriod = InferSelectModel; export type ResourceAiModel = InferSelectModel; export type SiteResourceAiModel = InferSelectModel; diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 3d3ff2ded..f453d36f7 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1,6 +1,7 @@ import { randomUUID } from "crypto"; -import { InferSelectModel } from "drizzle-orm"; +import { InferSelectModel, sql } from "drizzle-orm"; import { + check, index, integer, primaryKey, @@ -219,7 +220,10 @@ export const resources = sqliteTable("resources", { aiProviderId: integer("aiProviderId").references( () => aiProviders.providerId, { onDelete: "set null" } - ) + ), + modelAccessMode: text("modelAccessMode").$type< + "passthrough" | "catalog" | "allowlist" + >() }); export const resourceAiModels = sqliteTable( @@ -483,7 +487,10 @@ export const siteResources = sqliteTable("siteResources", { aiProviderId: integer("aiProviderId").references( () => aiProviders.providerId, { onDelete: "set null" } - ) + ), + modelAccessMode: text("modelAccessMode").$type< + "passthrough" | "catalog" | "allowlist" + >() }); export const siteResourceAiModels = sqliteTable( @@ -1605,8 +1612,6 @@ export const aiProviders = sqliteTable("aiProviders", { skipTlsVerification: integer("skipTlsVerification", { mode: "boolean" }) .notNull() .default(false), - budgetAmount: real("budgetAmount"), - budgetUnit: text("budgetUnit").$type<"usd" | "tokens">(), enabled: integer("enabled", { mode: "boolean" }).notNull().default(true), createdAt: integer("createdAt").notNull(), updatedAt: integer("updatedAt").notNull() @@ -1621,8 +1626,6 @@ export const aiModels = sqliteTable( .references(() => aiProviders.providerId, { onDelete: "cascade" }), modelKey: text("modelKey").notNull(), name: text("name").notNull(), - budgetAmount: real("budgetAmount"), - budgetUnit: text("budgetUnit").$type<"usd" | "tokens">(), enabled: integer("enabled", { mode: "boolean" }) .notNull() .default(true), @@ -1632,6 +1635,72 @@ export const aiModels = sqliteTable( (t) => [unique("ai_model_provider_key_uniq").on(t.providerId, t.modelKey)] ); +export const aiBudgets = sqliteTable( + "aiBudgets", + { + budgetId: integer("budgetId").primaryKey({ autoIncrement: true }), + orgId: text("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + providerId: integer("providerId").references( + () => aiProviders.providerId, + { onDelete: "cascade" } + ), + modelId: integer("modelId").references(() => aiModels.modelId, { + onDelete: "cascade" + }), + resourceId: integer("resourceId").references( + () => resources.resourceId, + { onDelete: "cascade" } + ), + siteResourceId: integer("siteResourceId").references( + () => siteResources.siteResourceId, + { onDelete: "cascade" } + ), + amount: real("amount").notNull(), + unit: text("unit").$type<"usd" | "tokens">().notNull(), + period: text("period").$type<"monthly">().notNull().default("monthly"), + enforcement: text("enforcement") + .$type<"hard" | "soft">() + .notNull() + .default("hard"), + enabled: integer("enabled", { mode: "boolean" }) + .notNull() + .default(true), + createdAt: integer("createdAt").notNull(), + updatedAt: integer("updatedAt").notNull() + }, + (t) => [ + check( + "ai_budget_one_scope", + sql`( + (CASE WHEN ${t.providerId} IS NOT NULL THEN 1 ELSE 0 END) + + (CASE WHEN ${t.modelId} IS NOT NULL THEN 1 ELSE 0 END) + + (CASE WHEN ${t.resourceId} IS NOT NULL THEN 1 ELSE 0 END) + + (CASE WHEN ${t.siteResourceId} IS NOT NULL THEN 1 ELSE 0 END) + ) = 1` + ), + unique("ai_budget_provider_uniq").on(t.providerId), + unique("ai_budget_model_uniq").on(t.modelId), + unique("ai_budget_resource_uniq").on(t.resourceId), + unique("ai_budget_site_resource_uniq").on(t.siteResourceId) + ] +); + +export const aiBudgetPeriods = sqliteTable( + "aiBudgetPeriods", + { + periodId: integer("periodId").primaryKey({ autoIncrement: true }), + budgetId: integer("budgetId") + .notNull() + .references(() => aiBudgets.budgetId, { onDelete: "cascade" }), + periodStart: integer("periodStart").notNull(), + periodEnd: integer("periodEnd").notNull(), + usedAmount: real("usedAmount").notNull().default(0) + }, + (t) => [unique("ai_budget_period_start_uniq").on(t.budgetId, t.periodStart)] +); + export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1716,5 +1785,7 @@ export type RolePolicy = InferSelectModel; export type UserPolicy = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; +export type AiBudget = InferSelectModel; +export type AiBudgetPeriod = InferSelectModel; export type ResourceAiModel = InferSelectModel; export type SiteResourceAiModel = InferSelectModel; From 2bf426bc222aa41591eaab0edbbaf3d83a5d8c69 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 15:36:31 -0400 Subject: [PATCH 029/296] dont restrict to enterprise and dont link with providers yet --- .../private/lib/traefik/getTraefikConfig.ts | 50 ++++++++----------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 8e1b1eca2..3c8a1cab2 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -427,34 +427,26 @@ export async function getTraefikConfig( ) ); - let siteResourcesInference: { - siteResourceId: number; - alias: string | null; - ssl: boolean | null; - enabled: boolean | null; - }[] = []; - if (build == "enterprise") { - siteResourcesInference = await db - .select({ - siteResourceId: siteResources.siteResourceId, - alias: siteResources.alias, - ssl: siteResources.ssl, - enabled: siteResources.enabled - }) - .from(siteResources) - .innerJoin( - aiProviders, - eq(siteResources.aiProviderId, aiProviders.providerId) + const siteResourcesInference = await db + .select({ + siteResourceId: siteResources.siteResourceId, + alias: siteResources.alias, + ssl: siteResources.ssl, + enabled: siteResources.enabled + }) + .from(siteResources) + // .innerJoin( + // aiProviders, + // eq(siteResources.aiProviderId, aiProviders.providerId) + // ) + .where( + and( + eq(siteResources.mode, "inference"), + eq(siteResources.enabled, true), + // eq(aiProviders.enabled, true), + isNotNull(siteResources.alias) ) - .where( - and( - eq(siteResources.mode, "inference"), - eq(siteResources.enabled, true), - eq(aiProviders.enabled, true), - isNotNull(siteResources.alias) - ) - ); - } + ); let validCerts: CertificateResult[] = []; if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { @@ -1586,9 +1578,7 @@ export async function getTraefikConfig( tls = { certResolver: resolverName, - ...(preferWildcard - ? { domains: [{ main: wildCard }] } - : {}) + ...(preferWildcard ? { domains: [{ main: wildCard }] } : {}) }; } else { const matchingCert = validCerts.find( From ec5a2b0cbe9841f60a7c3bf7c63b202d83f1830c Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 15:36:40 -0400 Subject: [PATCH 030/296] add peers to exit node --- server/routers/olm/handleOlmRegisterMessage.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 95c5e0d47..c0cee86d9 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -28,6 +28,7 @@ import { verifyExitNodeOrgAccess } from "#dynamic/lib/exitNodes"; import { getUniqueSubnetForExitNode } from "@server/lib/exitNodes"; +import { addPeer } from "../gerbil/peers"; const HOLEPUNCH_STALE_CHAIN_THRESHOLD = 18; const HOLEPUNCH_STALE_CHAIN_TTL_SECONDS = 1800; @@ -393,6 +394,18 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { ); } + if (clientSubnet && exitNodeId) { + try { + // add the peer to the exit node so it can connect + await addPeer(exitNodeId, { + publicKey: publicKey, + allowedIps: [clientSubnet] + }); + } catch (error) { + logger.error(`Failed to add peer to exit node: ${error}`); + } + } + let staleHolePunchChainCount: number | undefined; const hasChainId = chainId !== undefined && chainId !== null && String(chainId) !== ""; @@ -490,7 +503,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { endpoint: `${exitNode.endpoint}:${exitNode.listenPort}`, publicKey: exitNode.publicKey, serverIP: exitNode.address.split("/")[0], - tunnelIP: client.exitNodeSubnet.split("/")[0] + tunnelIP: `${client.exitNodeSubnet.split("/")[0]}/${exitNode.address.split("/")[1]}` // we need to use the exit node's subnet mask here because the client will be using the exit node's subnet mask for its routing table so we can address it } : undefined, chainId: chainId From 2cfd7e867beaa01ba02390ed781c8d9fbb4a0b8e Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 4 Aug 2026 15:45:44 -0400 Subject: [PATCH 031/296] delete the client peers from the exit nodes properly --- server/lib/deleteOrg.ts | 3 +++ server/routers/olm/handleOlmRegisterMessage.ts | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/lib/deleteOrg.ts b/server/lib/deleteOrg.ts index 20c8a3e23..43921e41b 100644 --- a/server/lib/deleteOrg.ts +++ b/server/lib/deleteOrg.ts @@ -93,6 +93,9 @@ export async function deleteOrgById( await trx.delete(sites).where(eq(sites.siteId, site.siteId)); } for (const client of orgClients) { + if (client.exitNodeId && client.pubKey) { + await deletePeer(client.exitNodeId, client.pubKey); + } const [olm] = await trx .select() .from(olms) diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index c0cee86d9..5cd38dabe 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -28,7 +28,7 @@ import { verifyExitNodeOrgAccess } from "#dynamic/lib/exitNodes"; import { getUniqueSubnetForExitNode } from "@server/lib/exitNodes"; -import { addPeer } from "../gerbil/peers"; +import { addPeer, deletePeer } from "../gerbil/peers"; const HOLEPUNCH_STALE_CHAIN_THRESHOLD = 18; const HOLEPUNCH_STALE_CHAIN_TTL_SECONDS = 1800; @@ -394,6 +394,12 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { ); } + if (client.pubKey && client.pubKey !== publicKey && client.exitNodeId) { + // test the old client to see if its different then remove + logger.info("Public key mismatch. Deleting old peer..."); + await deletePeer(client.exitNodeId, client.pubKey); + } + if (clientSubnet && exitNodeId) { try { // add the peer to the exit node so it can connect From e38359c74f44fc0cdc0e9e0a29b40d46970ab83b Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 4 Aug 2026 15:54:24 -0400 Subject: [PATCH 032/296] add crud for adding providers and models to resources --- server/db/pg/schema/schema.ts | 58 +- server/db/sqlite/schema/schema.ts | 58 +- server/lib/aiInferenceResource.ts | 512 ++++++++++++++++++ server/lib/traefik/getTraefikConfig.ts | 10 +- .../private/lib/traefik/getTraefikConfig.ts | 23 +- server/routers/aiGateway/chatCompletions.ts | 288 +++++++--- server/routers/aiProvider/createAiModel.ts | 25 +- server/routers/aiProvider/createAiProvider.ts | 9 - server/routers/aiProvider/updateAiModel.ts | 26 +- server/routers/aiProvider/updateAiProvider.ts | 32 +- server/routers/aiProvider/validation.ts | 24 - server/routers/external.ts | 94 ++++ server/routers/integration.ts | 86 +++ .../routers/resource/addAiModelToResource.ts | 44 +- .../resource/addAiProviderToResource.ts | 152 ++++++ server/routers/resource/createResource.ts | 56 +- server/routers/resource/index.ts | 4 + .../routers/resource/listResourceAiModels.ts | 2 +- .../resource/listResourceAiProviders.ts | 94 ++++ .../resource/removeAiModelFromResource.ts | 8 +- .../resource/removeAiProviderFromResource.ts | 165 ++++++ .../routers/resource/setResourceAiModels.ts | 48 +- .../resource/setResourceAiProviders.ts | 137 +++++ server/routers/resource/updateResource.ts | 15 +- .../siteResource/addAiModelToSiteResource.ts | 49 +- .../addAiProviderToSiteResource.ts | 152 ++++++ .../siteResource/createSiteResource.ts | 50 +- server/routers/siteResource/index.ts | 4 + .../siteResource/listSiteResourceAiModels.ts | 14 +- .../listSiteResourceAiProviders.ts | 94 ++++ .../removeAiModelFromSiteResource.ts | 9 +- .../removeAiProviderFromSiteResource.ts | 164 ++++++ .../siteResource/setSiteResourceAiModels.ts | 72 +-- .../setSiteResourceAiProviders.ts | 139 +++++ .../siteResource/updateSiteResource.ts | 24 +- .../[providerId]/authentication/page.tsx | 4 - .../[providerId]/network/page.tsx | 4 - .../settings/ai-providers/create/page.tsx | 2 - src/lib/aiProviderFormSchema.ts | 30 - 39 files changed, 2343 insertions(+), 438 deletions(-) create mode 100644 server/lib/aiInferenceResource.ts create mode 100644 server/routers/resource/addAiProviderToResource.ts create mode 100644 server/routers/resource/listResourceAiProviders.ts create mode 100644 server/routers/resource/removeAiProviderFromResource.ts create mode 100644 server/routers/resource/setResourceAiProviders.ts create mode 100644 server/routers/siteResource/addAiProviderToSiteResource.ts create mode 100644 server/routers/siteResource/listSiteResourceAiProviders.ts create mode 100644 server/routers/siteResource/removeAiProviderFromSiteResource.ts create mode 100644 server/routers/siteResource/setSiteResourceAiProviders.ts diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 6a24064a6..b323ae411 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -211,14 +211,7 @@ export const resources = pgTable( authDaemonPort: integer("authDaemonPort").default(22123), status: varchar("status") .$type<"pending" | "approved">() - .default("approved"), - aiProviderId: integer("aiProviderId").references( - () => aiProviders.providerId, - { onDelete: "set null" } - ), - modelAccessMode: varchar("modelAccessMode").$type< - "passthrough" | "catalog" | "allowlist" - >() + .default("approved") }, (t) => [ index("idx_resources_fulldomain") @@ -229,6 +222,23 @@ export const resources = pgTable( ] ); +export const resourceAiProviders = pgTable( + "resourceAiProviders", + { + resourceId: integer("resourceId") + .notNull() + .references(() => resources.resourceId, { onDelete: "cascade" }), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + modelAccessMode: varchar("modelAccessMode") + .$type<"passthrough" | "catalog" | "allowlist">() + .notNull() + .default("passthrough") + }, + (t) => [primaryKey({ columns: [t.resourceId, t.providerId] })] +); + export const resourceAiModels = pgTable( "resourceAiModels", { @@ -496,18 +506,30 @@ export const siteResources = pgTable( fullDomain: varchar("fullDomain"), status: varchar("status") .$type<"pending" | "approved">() - .default("approved"), - aiProviderId: integer("aiProviderId").references( - () => aiProviders.providerId, - { onDelete: "set null" } - ), - modelAccessMode: varchar("modelAccessMode").$type< - "passthrough" | "catalog" | "allowlist" - >() + .default("approved") }, (t) => [index("idx_siteresources_orgid_niceid").on(t.orgId, t.niceId)] ); +export const siteResourceAiProviders = pgTable( + "siteResourceAiProviders", + { + siteResourceId: integer("siteResourceId") + .notNull() + .references(() => siteResources.siteResourceId, { + onDelete: "cascade" + }), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + modelAccessMode: varchar("modelAccessMode") + .$type<"passthrough" | "catalog" | "allowlist">() + .notNull() + .default("passthrough") + }, + (t) => [primaryKey({ columns: [t.siteResourceId, t.providerId] })] +); + export const siteResourceAiModels = pgTable( "siteResourceAiModels", { @@ -1806,5 +1828,9 @@ export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; export type AiBudget = InferSelectModel; export type AiBudgetPeriod = InferSelectModel; +export type ResourceAiProvider = InferSelectModel; +export type SiteResourceAiProvider = InferSelectModel< + typeof siteResourceAiProviders +>; export type ResourceAiModel = InferSelectModel; export type SiteResourceAiModel = InferSelectModel; diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index f453d36f7..c8f25dfed 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -216,16 +216,26 @@ export const resources = sqliteTable("resources", { .$type<"site" | "remote" | "native">() .default("site"), authDaemonPort: integer("authDaemonPort").default(22123), - status: text("status").$type<"pending" | "approved">().default("approved"), - aiProviderId: integer("aiProviderId").references( - () => aiProviders.providerId, - { onDelete: "set null" } - ), - modelAccessMode: text("modelAccessMode").$type< - "passthrough" | "catalog" | "allowlist" - >() + status: text("status").$type<"pending" | "approved">().default("approved") }); +export const resourceAiProviders = sqliteTable( + "resourceAiProviders", + { + resourceId: integer("resourceId") + .notNull() + .references(() => resources.resourceId, { onDelete: "cascade" }), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + modelAccessMode: text("modelAccessMode") + .$type<"passthrough" | "catalog" | "allowlist">() + .notNull() + .default("passthrough") + }, + (t) => [primaryKey({ columns: [t.resourceId, t.providerId] })] +); + export const resourceAiModels = sqliteTable( "resourceAiModels", { @@ -483,16 +493,28 @@ export const siteResources = sqliteTable("siteResources", { }), subdomain: text("subdomain"), fullDomain: text("fullDomain"), - status: text("status").$type<"pending" | "approved">().default("approved"), - aiProviderId: integer("aiProviderId").references( - () => aiProviders.providerId, - { onDelete: "set null" } - ), - modelAccessMode: text("modelAccessMode").$type< - "passthrough" | "catalog" | "allowlist" - >() + status: text("status").$type<"pending" | "approved">().default("approved") }); +export const siteResourceAiProviders = sqliteTable( + "siteResourceAiProviders", + { + siteResourceId: integer("siteResourceId") + .notNull() + .references(() => siteResources.siteResourceId, { + onDelete: "cascade" + }), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + modelAccessMode: text("modelAccessMode") + .$type<"passthrough" | "catalog" | "allowlist">() + .notNull() + .default("passthrough") + }, + (t) => [primaryKey({ columns: [t.siteResourceId, t.providerId] })] +); + export const siteResourceAiModels = sqliteTable( "siteResourceAiModels", { @@ -1787,5 +1809,9 @@ export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; export type AiBudget = InferSelectModel; export type AiBudgetPeriod = InferSelectModel; +export type ResourceAiProvider = InferSelectModel; +export type SiteResourceAiProvider = InferSelectModel< + typeof siteResourceAiProviders +>; export type ResourceAiModel = InferSelectModel; export type SiteResourceAiModel = InferSelectModel; diff --git a/server/lib/aiInferenceResource.ts b/server/lib/aiInferenceResource.ts new file mode 100644 index 000000000..4175a11e8 --- /dev/null +++ b/server/lib/aiInferenceResource.ts @@ -0,0 +1,512 @@ +import { and, eq, inArray } from "drizzle-orm"; +import { + aiModels, + aiProviders, + db, + resourceAiModels, + resourceAiProviders, + siteResourceAiModels, + siteResourceAiProviders, + type Transaction +} from "@server/db"; +import { z } from "zod"; + +type DbOrTrx = Transaction | typeof db; + +export const modelAccessModeSchema = z.enum([ + "passthrough", + "catalog", + "allowlist" +]); + +export type ModelAccessMode = z.infer; + +export const resourceAiProviderAttachmentSchema = z.strictObject({ + providerId: z.number().int().positive(), + modelAccessMode: modelAccessModeSchema.optional() +}); + +export type ResourceAiProviderInput = z.infer< + typeof resourceAiProviderAttachmentSchema +>; + +export type ResourceAiProviderAttachment = { + providerId: number; + modelAccessMode: ModelAccessMode; +}; + +export type InferenceFieldsError = { + error: string; +}; + +export function isInferenceFieldsError( + value: { error: string } | object +): value is InferenceFieldsError { + return "error" in value; +} + +function normalizeAttachments( + inputs: ResourceAiProviderInput[] +): ResourceAiProviderAttachment[] { + const byProvider = new Map(); + for (const input of inputs) { + byProvider.set( + input.providerId, + input.modelAccessMode ?? "passthrough" + ); + } + return [...byProvider.entries()].map(([providerId, modelAccessMode]) => ({ + providerId, + modelAccessMode + })); +} + +/** + * Validate provider attachments for an org. + * At most one passthrough provider is allowed per resource. + */ +export async function resolveProviderAttachments(input: { + orgId: string; + attachments: ResourceAiProviderInput[]; + requireAtLeastOne: boolean; +}): Promise { + const attachments = normalizeAttachments(input.attachments); + + if (input.requireAtLeastOne && attachments.length === 0) { + return { + error: "At least one AI provider is required for inference-mode resources" + }; + } + + const passthroughCount = attachments.filter( + (a) => a.modelAccessMode === "passthrough" + ).length; + if (passthroughCount > 1) { + return { + error: "A resource may have at most one AI provider in passthrough mode" + }; + } + + if (attachments.length === 0) { + return []; + } + + const providerIds = attachments.map((a) => a.providerId); + const providers = await db + .select({ + providerId: aiProviders.providerId, + orgId: aiProviders.orgId, + enabled: aiProviders.enabled + }) + .from(aiProviders) + .where( + and( + inArray(aiProviders.providerId, providerIds), + eq(aiProviders.orgId, input.orgId) + ) + ); + + if (providers.length !== providerIds.length) { + return { + error: "One or more AI providers were not found in this organization" + }; + } + + const disabled = providers.find((p) => !p.enabled); + if (disabled) { + return { + error: `AI provider with ID ${disabled.providerId} is disabled` + }; + } + + return attachments; +} + +export async function assertInferenceModeAllowsProviderFields(input: { + mode: string; + hasProviderAttachments: boolean; +}): Promise { + if (input.mode === "inference") { + return null; + } + if (input.hasProviderAttachments) { + return { + error: "AI providers can only be attached to inference-mode resources" + }; + } + return null; +} + +export async function setPublicResourceAiProviders( + resourceId: number, + attachments: ResourceAiProviderAttachment[], + trx: DbOrTrx = db +): Promise { + await trx + .delete(resourceAiProviders) + .where(eq(resourceAiProviders.resourceId, resourceId)); + + if (attachments.length > 0) { + await trx.insert(resourceAiProviders).values( + attachments.map((a) => ({ + resourceId, + providerId: a.providerId, + modelAccessMode: a.modelAccessMode + })) + ); + } + + await prunePublicResourceAllowlistToAllowlistProviders(resourceId, trx); +} + +export async function setSiteResourceAiProviders( + siteResourceId: number, + attachments: ResourceAiProviderAttachment[], + trx: DbOrTrx = db +): Promise { + await trx + .delete(siteResourceAiProviders) + .where(eq(siteResourceAiProviders.siteResourceId, siteResourceId)); + + if (attachments.length > 0) { + await trx.insert(siteResourceAiProviders).values( + attachments.map((a) => ({ + siteResourceId, + providerId: a.providerId, + modelAccessMode: a.modelAccessMode + })) + ); + } + + await pruneSiteResourceAllowlistToAllowlistProviders(siteResourceId, trx); +} + +export async function clearPublicResourceAiConfig( + resourceId: number, + trx: DbOrTrx = db +): Promise { + await trx + .delete(resourceAiModels) + .where(eq(resourceAiModels.resourceId, resourceId)); + await trx + .delete(resourceAiProviders) + .where(eq(resourceAiProviders.resourceId, resourceId)); +} + +export async function clearSiteResourceAiConfig( + siteResourceId: number, + trx: DbOrTrx = db +): Promise { + await trx + .delete(siteResourceAiModels) + .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); + await trx + .delete(siteResourceAiProviders) + .where(eq(siteResourceAiProviders.siteResourceId, siteResourceId)); +} + +async function prunePublicResourceAllowlistToAllowlistProviders( + resourceId: number, + trx: DbOrTrx = db +): Promise { + const allowlistProviders = await trx + .select({ providerId: resourceAiProviders.providerId }) + .from(resourceAiProviders) + .where( + and( + eq(resourceAiProviders.resourceId, resourceId), + eq(resourceAiProviders.modelAccessMode, "allowlist") + ) + ); + + if (allowlistProviders.length === 0) { + await trx + .delete(resourceAiModels) + .where(eq(resourceAiModels.resourceId, resourceId)); + return; + } + + const validModels = await trx + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .where( + inArray( + aiModels.providerId, + allowlistProviders.map((p) => p.providerId) + ) + ); + const validIds = validModels.map((m) => m.modelId); + + const existing = await trx + .select({ modelId: resourceAiModels.modelId }) + .from(resourceAiModels) + .where(eq(resourceAiModels.resourceId, resourceId)); + + const toRemove = existing + .map((e) => e.modelId) + .filter((id) => !validIds.includes(id)); + + if (toRemove.length > 0) { + await trx + .delete(resourceAiModels) + .where( + and( + eq(resourceAiModels.resourceId, resourceId), + inArray(resourceAiModels.modelId, toRemove) + ) + ); + } +} + +async function pruneSiteResourceAllowlistToAllowlistProviders( + siteResourceId: number, + trx: DbOrTrx = db +): Promise { + const allowlistProviders = await trx + .select({ providerId: siteResourceAiProviders.providerId }) + .from(siteResourceAiProviders) + .where( + and( + eq(siteResourceAiProviders.siteResourceId, siteResourceId), + eq(siteResourceAiProviders.modelAccessMode, "allowlist") + ) + ); + + if (allowlistProviders.length === 0) { + await trx + .delete(siteResourceAiModels) + .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); + return; + } + + const validModels = await trx + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .where( + inArray( + aiModels.providerId, + allowlistProviders.map((p) => p.providerId) + ) + ); + const validIds = validModels.map((m) => m.modelId); + + const existing = await trx + .select({ modelId: siteResourceAiModels.modelId }) + .from(siteResourceAiModels) + .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); + + const toRemove = existing + .map((e) => e.modelId) + .filter((id) => !validIds.includes(id)); + + if (toRemove.length > 0) { + await trx + .delete(siteResourceAiModels) + .where( + and( + eq(siteResourceAiModels.siteResourceId, siteResourceId), + inArray(siteResourceAiModels.modelId, toRemove) + ) + ); + } +} + +export async function listPublicResourceAiProviders(resourceId: number) { + return db + .select({ + providerId: resourceAiProviders.providerId, + modelAccessMode: resourceAiProviders.modelAccessMode, + name: aiProviders.name, + type: aiProviders.type, + enabled: aiProviders.enabled + }) + .from(resourceAiProviders) + .innerJoin( + aiProviders, + eq(resourceAiProviders.providerId, aiProviders.providerId) + ) + .where(eq(resourceAiProviders.resourceId, resourceId)); +} + +export async function listSiteResourceAiProviders(siteResourceId: number) { + return db + .select({ + providerId: siteResourceAiProviders.providerId, + modelAccessMode: siteResourceAiProviders.modelAccessMode, + name: aiProviders.name, + type: aiProviders.type, + enabled: aiProviders.enabled + }) + .from(siteResourceAiProviders) + .innerJoin( + aiProviders, + eq(siteResourceAiProviders.providerId, aiProviders.providerId) + ) + .where(eq(siteResourceAiProviders.siteResourceId, siteResourceId)); +} + +/** + * Allowlist APIs require an inference resource with at least one + * attached provider in allowlist mode. + */ +export async function assertPublicAllowlistApiEligible(resource: { + resourceId: number; + mode: string; +}): Promise { + if (resource.mode !== "inference") { + return "AI model allowlists are only supported on inference-mode resources"; + } + + const [row] = await db + .select({ providerId: resourceAiProviders.providerId }) + .from(resourceAiProviders) + .where( + and( + eq(resourceAiProviders.resourceId, resource.resourceId), + eq(resourceAiProviders.modelAccessMode, "allowlist") + ) + ) + .limit(1); + + if (!row) { + return "Attach at least one AI provider with modelAccessMode=allowlist before managing allowed models"; + } + return null; +} + +export async function assertSiteAllowlistApiEligible(siteResource: { + siteResourceId: number; + mode: string; +}): Promise { + if (siteResource.mode !== "inference") { + return "AI model allowlists are only supported on inference-mode resources"; + } + + const [row] = await db + .select({ providerId: siteResourceAiProviders.providerId }) + .from(siteResourceAiProviders) + .where( + and( + eq( + siteResourceAiProviders.siteResourceId, + siteResource.siteResourceId + ), + eq(siteResourceAiProviders.modelAccessMode, "allowlist") + ) + ) + .limit(1); + + if (!row) { + return "Attach at least one AI provider with modelAccessMode=allowlist before managing allowed models"; + } + return null; +} + +/** + * Models must belong to providers attached to this resource in allowlist mode, + * and those providers must belong to the resource's org. + */ +export async function assertModelsBelongToPublicAllowlistProviders(input: { + orgId: string; + resourceId: number; + modelIds: number[]; +}): Promise { + const uniqueIds = [...new Set(input.modelIds)]; + if (uniqueIds.length === 0) { + return null; + } + + const allowlistProviders = await db + .select({ providerId: resourceAiProviders.providerId }) + .from(resourceAiProviders) + .innerJoin( + aiProviders, + eq(resourceAiProviders.providerId, aiProviders.providerId) + ) + .where( + and( + eq(resourceAiProviders.resourceId, input.resourceId), + eq(resourceAiProviders.modelAccessMode, "allowlist"), + eq(aiProviders.orgId, input.orgId) + ) + ); + + if (allowlistProviders.length === 0) { + return "No allowlist AI providers are attached to this resource"; + } + + const validModels = await db + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .innerJoin(aiProviders, eq(aiModels.providerId, aiProviders.providerId)) + .where( + and( + inArray(aiModels.modelId, uniqueIds), + inArray( + aiModels.providerId, + allowlistProviders.map((p) => p.providerId) + ), + eq(aiProviders.orgId, input.orgId) + ) + ); + + if (validModels.length !== uniqueIds.length) { + return "One or more model IDs do not exist or do not belong to an allowlist provider on this resource"; + } + + return null; +} + +export async function assertModelsBelongToSiteAllowlistProviders(input: { + orgId: string; + siteResourceId: number; + modelIds: number[]; +}): Promise { + const uniqueIds = [...new Set(input.modelIds)]; + if (uniqueIds.length === 0) { + return null; + } + + const allowlistProviders = await db + .select({ providerId: siteResourceAiProviders.providerId }) + .from(siteResourceAiProviders) + .innerJoin( + aiProviders, + eq(siteResourceAiProviders.providerId, aiProviders.providerId) + ) + .where( + and( + eq( + siteResourceAiProviders.siteResourceId, + input.siteResourceId + ), + eq(siteResourceAiProviders.modelAccessMode, "allowlist"), + eq(aiProviders.orgId, input.orgId) + ) + ); + + if (allowlistProviders.length === 0) { + return "No allowlist AI providers are attached to this site resource"; + } + + const validModels = await db + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .innerJoin(aiProviders, eq(aiModels.providerId, aiProviders.providerId)) + .where( + and( + inArray(aiModels.modelId, uniqueIds), + inArray( + aiModels.providerId, + allowlistProviders.map((p) => p.providerId) + ), + eq(aiProviders.orgId, input.orgId) + ) + ); + + if (validModels.length !== uniqueIds.length) { + return "One or more model IDs do not exist or do not belong to an allowlist provider on this site resource"; + } + + return null; +} diff --git a/server/lib/traefik/getTraefikConfig.ts b/server/lib/traefik/getTraefikConfig.ts index 4f5dd3860..574b8a374 100644 --- a/server/lib/traefik/getTraefikConfig.ts +++ b/server/lib/traefik/getTraefikConfig.ts @@ -1,4 +1,4 @@ -import { db, targetHealthCheck, domains, aiProviders } from "@server/db"; +import { db, targetHealthCheck, domains, aiProviders, resourceAiProviders } from "@server/db"; import { and, eq, @@ -214,7 +214,7 @@ export async function getTraefikConfig( // central AI gateway), so they can't be reached via the targets->sites // join above - query them separately and include them on every exit node. const inferenceResources = await db - .select({ + .selectDistinct({ resourceId: resources.resourceId, resourceName: resources.name, fullDomain: resources.fullDomain, @@ -226,9 +226,13 @@ export async function getTraefikConfig( preferWildcardCert: domains.preferWildcardCert }) .from(resources) + .innerJoin( + resourceAiProviders, + eq(resources.resourceId, resourceAiProviders.resourceId) + ) .innerJoin( aiProviders, - eq(resources.aiProviderId, aiProviders.providerId) + eq(resourceAiProviders.providerId, aiProviders.providerId) ) .leftJoin(domains, eq(domains.domainId, resources.domainId)) .where( diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 8e1b1eca2..abe2fc691 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -42,7 +42,9 @@ import { siteResources, Target, targets, - aiProviders + aiProviders, + resourceAiProviders, + siteResourceAiProviders } from "@server/db"; import { sanitize, @@ -402,7 +404,7 @@ export async function getTraefikConfig( // so they can't be reached via the joins above - query them separately // and include them on every exit node. const inferenceResources = await db - .select({ + .selectDistinct({ resourceId: resources.resourceId, fullDomain: resources.fullDomain, ssl: resources.ssl, @@ -414,9 +416,13 @@ export async function getTraefikConfig( preferWildcardCert: domains.preferWildcardCert }) .from(resources) + .innerJoin( + resourceAiProviders, + eq(resources.resourceId, resourceAiProviders.resourceId) + ) .innerJoin( aiProviders, - eq(resources.aiProviderId, aiProviders.providerId) + eq(resourceAiProviders.providerId, aiProviders.providerId) ) .leftJoin(domains, eq(domains.domainId, resources.domainId)) .where( @@ -435,16 +441,23 @@ export async function getTraefikConfig( }[] = []; if (build == "enterprise") { siteResourcesInference = await db - .select({ + .selectDistinct({ siteResourceId: siteResources.siteResourceId, alias: siteResources.alias, ssl: siteResources.ssl, enabled: siteResources.enabled }) .from(siteResources) + .innerJoin( + siteResourceAiProviders, + eq( + siteResources.siteResourceId, + siteResourceAiProviders.siteResourceId + ) + ) .innerJoin( aiProviders, - eq(siteResources.aiProviderId, aiProviders.providerId) + eq(siteResourceAiProviders.providerId, aiProviders.providerId) ) .where( and( diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index c9c32c2ba..51508778a 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -8,8 +8,10 @@ import { db, exitNodes, resourceAiModels, + resourceAiProviders, resources, siteResourceAiModels, + siteResourceAiProviders, siteResources, users } from "@server/db"; @@ -21,7 +23,6 @@ import { AiProviderType, resolveAiProviderConfig } from "@server/lib/aiProviderDefaults"; -import { verifyResourceAccessToken } from "@server/auth/verifyResourceAccessToken"; import { SESSION_COOKIE_NAME, validateSessionToken @@ -31,6 +32,7 @@ import { isIpInCidr } from "@server/lib/ip"; import { localCache } from "@server/lib/cache"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; +import type { ModelAccessMode } from "@server/lib/aiInferenceResource"; // Short-lived local caches so a burst of requests from the same IP/user // doesn't hit the database on every single request. None of this is @@ -85,14 +87,23 @@ async function findClientByIp(ip: string): Promise { return result; } +type ProviderAttachment = { + provider: AiProvider; + modelAccessMode: ModelAccessMode; +}; + type ResolvedTarget = { resourceId: number | null; orgId: string | null; - provider: AiProvider; - // null = no restriction; every enabled model on the provider is allowed - allowedModelIds: number[] | null; + attachments: ProviderAttachment[]; + // model IDs on this resource's allowlist (resource-wide) + allowedModelIds: number[]; }; +type ProviderSelection = + | { ok: true; provider: AiProvider } + | { ok: false; status: number; message: string }; + export type RequestUser = { userId: string; username: string; @@ -184,88 +195,245 @@ async function resolveTarget(host: string): Promise { const [resourceRow] = await db .select({ resourceId: resources.resourceId, - orgId: resources.orgId, - provider: aiProviders + orgId: resources.orgId }) .from(resources) - .innerJoin( - aiProviders, - eq(resources.aiProviderId, aiProviders.providerId) - ) .where( and( eq(resources.fullDomain, host), eq(resources.mode, "inference"), - eq(resources.enabled, true), - eq(aiProviders.enabled, true) + eq(resources.enabled, true) ) ) .limit(1); if (resourceRow) { - const restrictions = await db - .select({ modelId: resourceAiModels.modelId }) - .from(resourceAiModels) - .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)); + const attachmentRows = await db + .select({ + modelAccessMode: resourceAiProviders.modelAccessMode, + provider: aiProviders + }) + .from(resourceAiProviders) + .innerJoin( + aiProviders, + eq(resourceAiProviders.providerId, aiProviders.providerId) + ) + .where( + and( + eq(resourceAiProviders.resourceId, resourceRow.resourceId), + eq(aiProviders.enabled, true) + ) + ); + + if (attachmentRows.length === 0) { + return null; + } + + const hasAllowlist = attachmentRows.some( + (a) => a.modelAccessMode === "allowlist" + ); + let allowedModelIds: number[] = []; + if (hasAllowlist) { + const restrictions = await db + .select({ modelId: resourceAiModels.modelId }) + .from(resourceAiModels) + .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)); + allowedModelIds = restrictions.map((r) => r.modelId); + } return { resourceId: resourceRow.resourceId, orgId: resourceRow.orgId, - provider: resourceRow.provider, - allowedModelIds: restrictions.length - ? restrictions.map((r) => r.modelId) - : null + attachments: attachmentRows.map((a) => ({ + provider: a.provider, + modelAccessMode: a.modelAccessMode as ModelAccessMode + })), + allowedModelIds }; } const [siteResourceRow] = await db .select({ siteResourceId: siteResources.siteResourceId, - orgId: siteResources.orgId, - provider: aiProviders + orgId: siteResources.orgId }) .from(siteResources) - .innerJoin( - aiProviders, - eq(siteResources.aiProviderId, aiProviders.providerId) - ) .where( and( eq(siteResources.alias, host), eq(siteResources.mode, "inference"), - eq(siteResources.enabled, true), - eq(aiProviders.enabled, true) + eq(siteResources.enabled, true) ) ) .limit(1); if (siteResourceRow) { - const restrictions = await db - .select({ modelId: siteResourceAiModels.modelId }) - .from(siteResourceAiModels) + const attachmentRows = await db + .select({ + modelAccessMode: siteResourceAiProviders.modelAccessMode, + provider: aiProviders + }) + .from(siteResourceAiProviders) + .innerJoin( + aiProviders, + eq(siteResourceAiProviders.providerId, aiProviders.providerId) + ) .where( - eq( - siteResourceAiModels.siteResourceId, - siteResourceRow.siteResourceId + and( + eq( + siteResourceAiProviders.siteResourceId, + siteResourceRow.siteResourceId + ), + eq(aiProviders.enabled, true) ) ); + if (attachmentRows.length === 0) { + return null; + } + + const hasAllowlist = attachmentRows.some( + (a) => a.modelAccessMode === "allowlist" + ); + let allowedModelIds: number[] = []; + if (hasAllowlist) { + const restrictions = await db + .select({ modelId: siteResourceAiModels.modelId }) + .from(siteResourceAiModels) + .where( + eq( + siteResourceAiModels.siteResourceId, + siteResourceRow.siteResourceId + ) + ); + allowedModelIds = restrictions.map((r) => r.modelId); + } + return { // siteResources have no per-user auth/policy stack today (see // the routing comment in getTraefikConfig.ts), so there's no // resource access token scope to validate a user token against. resourceId: null, orgId: siteResourceRow.orgId, - provider: siteResourceRow.provider, - allowedModelIds: restrictions.length - ? restrictions.map((r) => r.modelId) - : null + attachments: attachmentRows.map((a) => ({ + provider: a.provider, + modelAccessMode: a.modelAccessMode as ModelAccessMode + })), + allowedModelIds }; } return null; } +async function providerMatchesModel( + attachment: ProviderAttachment, + requestedModel: string, + allowedModelIds: number[] +): Promise { + if (attachment.modelAccessMode === "passthrough") { + return true; + } + + const [matchedModel] = await db + .select({ + modelId: aiModels.modelId, + enabled: aiModels.enabled + }) + .from(aiModels) + .where( + and( + eq(aiModels.providerId, attachment.provider.providerId), + eq(aiModels.modelKey, requestedModel) + ) + ) + .limit(1); + + if (!matchedModel) { + return false; + } + + if (attachment.modelAccessMode === "catalog") { + return matchedModel.enabled; + } + + // allowlist + return allowedModelIds.includes(matchedModel.modelId); +} + +async function selectProvider( + attachments: ProviderAttachment[], + allowedModelIds: number[], + requestedModel: string | undefined +): Promise { + const passthroughAttachments = attachments.filter( + (a) => a.modelAccessMode === "passthrough" + ); + const hasRestricted = attachments.some( + (a) => + a.modelAccessMode === "catalog" || a.modelAccessMode === "allowlist" + ); + + if (!requestedModel) { + if (hasRestricted) { + return { + ok: false, + status: HttpCode.FORBIDDEN, + message: + "This resource restricts access to specific models; a model must be specified" + }; + } + if (passthroughAttachments.length === 1) { + return { ok: true, provider: passthroughAttachments[0].provider }; + } + return { + ok: false, + status: HttpCode.FORBIDDEN, + message: "A model must be specified for this resource" + }; + } + + const candidates: ProviderAttachment[] = []; + for (const attachment of attachments) { + if (attachment.modelAccessMode === "passthrough") { + candidates.push(attachment); + continue; + } + if ( + await providerMatchesModel( + attachment, + requestedModel, + allowedModelIds + ) + ) { + candidates.push(attachment); + } + } + + if (candidates.length === 1) { + return { ok: true, provider: candidates[0].provider }; + } + + if (candidates.length > 1) { + return { + ok: false, + status: HttpCode.FORBIDDEN, + message: `Model "${requestedModel}" is ambiguous across multiple AI providers on this resource` + }; + } + + // Zero candidates: fall back to a single passthrough attachment if present + if (passthroughAttachments.length === 1) { + return { ok: true, provider: passthroughAttachments[0].provider }; + } + + return { + ok: false, + status: HttpCode.FORBIDDEN, + message: `Model "${requestedModel}" is not permitted on this resource` + }; +} + // Generic OpenAI-wire-compatible passthrough. Anthropic's native API uses a // different path/schema; everything else here is OpenAI-compatible today. function getCompletionsPath(type: AiProviderType): string { @@ -296,7 +464,7 @@ export async function chatCompletions( }); } - const { provider, allowedModelIds, resourceId, orgId } = target; + const { attachments, allowedModelIds, resourceId, orgId } = target; // Best-effort identity resolution - not yet enforced, but lets us // start making per-user access decisions (e.g. model/role-based @@ -311,39 +479,19 @@ export async function chatCompletions( const requestedModel = typeof req.body?.model === "string" ? req.body.model : undefined; - if (allowedModelIds) { - if (!requestedModel) { - return res.status(HttpCode.FORBIDDEN).json({ - error: { - message: - "This resource restricts access to specific models; a model must be specified" - } - }); - } - - const [matchedModel] = await db - .select({ modelId: aiModels.modelId }) - .from(aiModels) - .where( - and( - eq(aiModels.providerId, provider.providerId), - eq(aiModels.modelKey, requestedModel) - ) - ) - .limit(1); - - if ( - !matchedModel || - !allowedModelIds.includes(matchedModel.modelId) - ) { - return res.status(HttpCode.FORBIDDEN).json({ - error: { - message: `Model "${requestedModel}" is not permitted on this resource` - } - }); - } + const selection = await selectProvider( + attachments, + allowedModelIds, + requestedModel + ); + if (!selection.ok) { + return res.status(selection.status).json({ + error: { message: selection.message } + }); } + const { provider } = selection; + if (!provider.apiKey) { return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ error: { message: "AI provider has no API key configured" } diff --git a/server/routers/aiProvider/createAiModel.ts b/server/routers/aiProvider/createAiModel.ts index 7fda989d2..dfe2a62e1 100644 --- a/server/routers/aiProvider/createAiModel.ts +++ b/server/routers/aiProvider/createAiModel.ts @@ -9,26 +9,16 @@ import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { and, eq } from "drizzle-orm"; import type { CreateOrEditAiModelResponse } from "@server/routers/aiProvider/types"; -import { - aiBudgetUnitSchema, - refineBudgetFields -} from "@server/routers/aiProvider/validation"; const paramsSchema = z.strictObject({ providerId: z.coerce.number().int().positive() }); -const bodySchema = z - .strictObject({ - modelKey: z.string().nonempty(), - name: z.string().nonempty(), - budgetAmount: z.number().positive().optional().nullable(), - budgetUnit: aiBudgetUnitSchema.optional().nullable(), - enabled: z.boolean().optional() - }) - .superRefine((data, ctx) => { - refineBudgetFields(data, ctx); - }); +const bodySchema = z.strictObject({ + modelKey: z.string().nonempty(), + name: z.string().nonempty(), + enabled: z.boolean().optional() +}); registry.registerPath({ method: "put", @@ -79,8 +69,7 @@ export async function createAiModel( } const { providerId } = parsedParams.data; - const { modelKey, name, budgetAmount, budgetUnit, enabled } = - parsedBody.data; + const { modelKey, name, enabled } = parsedBody.data; const [provider] = req.aiProvider && req.aiProvider.providerId === providerId @@ -127,8 +116,6 @@ export async function createAiModel( providerId, modelKey, name, - budgetAmount: budgetAmount ?? null, - budgetUnit: budgetUnit ?? null, enabled: enabled ?? true, createdAt: now, updatedAt: now diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index 1d58344cb..6fb3f395a 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -13,10 +13,8 @@ import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/ import { toPublicAiProvider } from "@server/routers/aiProvider/types"; import { aiAuthTypeSchema, - aiBudgetUnitSchema, aiProviderTypeSchema, aiRoutingModeSchema, - refineBudgetFields, refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; @@ -33,13 +31,10 @@ const bodySchema = z authType: aiAuthTypeSchema.optional().nullable(), routingMode: aiRoutingModeSchema.optional(), skipTlsVerification: z.boolean().optional(), - budgetAmount: z.number().positive().optional().nullable(), - budgetUnit: aiBudgetUnitSchema.optional().nullable(), enabled: z.boolean().optional() }) .superRefine((data, ctx) => { refineProviderUpstreamFields(data, ctx); - refineBudgetFields(data, ctx); }); registry.registerPath({ @@ -99,8 +94,6 @@ export async function createAiProvider( authType, routingMode, skipTlsVerification, - budgetAmount, - budgetUnit, enabled } = parsedBody.data; @@ -126,8 +119,6 @@ export async function createAiProvider( authType: authType ?? null, routingMode: resolvedRoutingMode, skipTlsVerification: skipTlsVerification ?? false, - budgetAmount: budgetAmount ?? null, - budgetUnit: budgetUnit ?? null, enabled: enabled ?? true, createdAt: now, updatedAt: now diff --git a/server/routers/aiProvider/updateAiModel.ts b/server/routers/aiProvider/updateAiModel.ts index fa94e8da4..fe9ccf273 100644 --- a/server/routers/aiProvider/updateAiModel.ts +++ b/server/routers/aiProvider/updateAiModel.ts @@ -9,26 +9,16 @@ import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { and, eq, ne } from "drizzle-orm"; import type { CreateOrEditAiModelResponse } from "@server/routers/aiProvider/types"; -import { - aiBudgetUnitSchema, - refineBudgetFields -} from "@server/routers/aiProvider/validation"; const paramsSchema = z.strictObject({ modelId: z.coerce.number().int().positive() }); -const bodySchema = z - .strictObject({ - modelKey: z.string().nonempty().optional(), - name: z.string().nonempty().optional(), - budgetAmount: z.number().positive().optional().nullable(), - budgetUnit: aiBudgetUnitSchema.optional().nullable(), - enabled: z.boolean().optional() - }) - .superRefine((data, ctx) => { - refineBudgetFields(data, ctx); - }); +const bodySchema = z.strictObject({ + modelKey: z.string().nonempty().optional(), + name: z.string().nonempty().optional(), + enabled: z.boolean().optional() +}); registry.registerPath({ method: "post", @@ -135,12 +125,6 @@ export async function updateAiModel( if (body.name !== undefined) { updateData.name = body.name; } - if (body.budgetAmount !== undefined) { - updateData.budgetAmount = body.budgetAmount; - } - if (body.budgetUnit !== undefined) { - updateData.budgetUnit = body.budgetUnit; - } if (body.enabled !== undefined) { updateData.enabled = body.enabled; } diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts index 39787fb93..35e27fba2 100644 --- a/server/routers/aiProvider/updateAiProvider.ts +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -14,10 +14,8 @@ import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/ import { toPublicAiProvider } from "@server/routers/aiProvider/types"; import { aiAuthTypeSchema, - aiBudgetUnitSchema, aiProviderTypeSchema, aiRoutingModeSchema, - refineBudgetFields, refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; import type { @@ -29,21 +27,15 @@ const paramsSchema = z.strictObject({ providerId: z.coerce.number().int().positive() }); -const bodySchema = z - .strictObject({ - name: z.string().nonempty().optional(), - upstreamUrl: z.url().optional().nullable(), - apiKey: z.string().optional(), - authType: aiAuthTypeSchema.optional().nullable(), - routingMode: aiRoutingModeSchema.optional(), - skipTlsVerification: z.boolean().optional(), - budgetAmount: z.number().positive().optional().nullable(), - budgetUnit: aiBudgetUnitSchema.optional().nullable(), - enabled: z.boolean().optional() - }) - .superRefine((data, ctx) => { - refineBudgetFields(data, ctx); - }); +const bodySchema = z.strictObject({ + name: z.string().nonempty().optional(), + upstreamUrl: z.url().optional().nullable(), + apiKey: z.string().optional(), + authType: aiAuthTypeSchema.optional().nullable(), + routingMode: aiRoutingModeSchema.optional(), + skipTlsVerification: z.boolean().optional(), + enabled: z.boolean().optional() +}); registry.registerPath({ method: "post", @@ -168,12 +160,6 @@ export async function updateAiProvider( if (body.enabled !== undefined) { updateData.enabled = body.enabled; } - if (body.budgetAmount !== undefined) { - updateData.budgetAmount = body.budgetAmount; - } - if (body.budgetUnit !== undefined) { - updateData.budgetUnit = body.budgetUnit; - } if (nextRoutingMode === "target") { updateData.upstreamUrl = null; } else if (body.upstreamUrl !== undefined) { diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts index c830c08de..a5e2aa962 100644 --- a/server/routers/aiProvider/validation.ts +++ b/server/routers/aiProvider/validation.ts @@ -1,7 +1,6 @@ import { z } from "zod"; import { providerRequiresUpstreamUrl, - type AiBudgetUnit, type AiProviderRoutingMode, type AiProviderType } from "@server/lib/aiProviderDefaults"; @@ -18,33 +17,10 @@ export const aiProviderTypeSchema = z.enum([ "custom" ]); -export const aiBudgetUnitSchema = z.enum(["usd", "tokens"]); - export const aiAuthTypeSchema = z.enum(["bearer"]); export const aiRoutingModeSchema = z.enum(["url", "target"]); -export function refineBudgetFields( - data: { - budgetAmount?: number | null; - budgetUnit?: AiBudgetUnit | null; - }, - ctx: z.RefinementCtx -) { - const hasAmount = - data.budgetAmount !== undefined && data.budgetAmount !== null; - const hasUnit = data.budgetUnit !== undefined && data.budgetUnit !== null; - - if (hasAmount !== hasUnit) { - ctx.addIssue({ - code: "custom", - message: - "budgetAmount and budgetUnit must both be set or both omitted", - path: hasAmount ? ["budgetUnit"] : ["budgetAmount"] - }); - } -} - export function refineProviderUpstreamFields( data: { type: AiProviderType; diff --git a/server/routers/external.ts b/server/routers/external.ts index 4ebf58cae..94165cc8d 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -414,6 +414,13 @@ authenticated.get( siteResource.listSiteResourceAiModels ); +authenticated.get( + "/site-resource/:siteResourceId/ai-providers", + verifySiteResourceAccess, + verifyUserHasAction(ActionsEnum.listResourceAiModels), + siteResource.listSiteResourceAiProviders +); + authenticated.post( "/site-resource/:siteResourceId/roles", verifySiteResourceAccess, @@ -432,6 +439,46 @@ authenticated.post( siteResource.setSiteResourceAiModels ); +authenticated.post( + "/site-resource/:siteResourceId/ai-models/add", + verifySiteResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.addAiModelToSiteResource +); + +authenticated.post( + "/site-resource/:siteResourceId/ai-models/remove", + verifySiteResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.removeAiModelFromSiteResource +); + +authenticated.post( + "/site-resource/:siteResourceId/ai-providers", + verifySiteResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.setSiteResourceAiProviders +); + +authenticated.post( + "/site-resource/:siteResourceId/ai-providers/add", + verifySiteResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.addAiProviderToSiteResource +); + +authenticated.post( + "/site-resource/:siteResourceId/ai-providers/remove", + verifySiteResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.removeAiProviderFromSiteResource +); + authenticated.post( "/site-resource/:siteResourceId/users", verifySiteResourceAccess, @@ -673,6 +720,13 @@ authenticated.get( resource.listResourceAiModels ); +authenticated.get( + "/resource/:resourceId/ai-providers", + verifyResourceAccess, + verifyUserHasAction(ActionsEnum.listResourceAiModels), + resource.listResourceAiProviders +); + authenticated.get( "/resource/:resourceId", verifyResourceAccess, @@ -884,6 +938,46 @@ authenticated.post( resource.setResourceAiModels ); +authenticated.post( + "/resource/:resourceId/ai-models/add", + verifyResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.addAiModelToResource +); + +authenticated.post( + "/resource/:resourceId/ai-models/remove", + verifyResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.removeAiModelFromResource +); + +authenticated.post( + "/resource/:resourceId/ai-providers", + verifyResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.setResourceAiProviders +); + +authenticated.post( + "/resource/:resourceId/ai-providers/add", + verifyResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.addAiProviderToResource +); + +authenticated.post( + "/resource/:resourceId/ai-providers/remove", + verifyResourceAccess, + verifyUserHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.removeAiProviderFromResource +); + authenticated.put( "/resource-policy/:resourcePolicyId/access-control", verifyResourcePolicyAccess, diff --git a/server/routers/integration.ts b/server/routers/integration.ts index 41757e7a0..e08c14920 100644 --- a/server/routers/integration.ts +++ b/server/routers/integration.ts @@ -256,6 +256,16 @@ authenticated.get( siteResource.listSiteResourceAiModels ); +authenticated.get( + [ + "/site-resource/:siteResourceId/ai-providers", + "/private-resource/:siteResourceId/ai-providers" + ], + verifyApiKeySiteResourceAccess, + verifyApiKeyHasAction(ActionsEnum.listResourceAiModels), + siteResource.listSiteResourceAiProviders +); + authenticated.post( [ "/site-resource/:siteResourceId/roles", @@ -341,6 +351,39 @@ authenticated.post( siteResource.removeAiModelFromSiteResource ); +authenticated.post( + [ + "/site-resource/:siteResourceId/ai-providers", + "/private-resource/:siteResourceId/ai-providers" + ], + verifyApiKeySiteResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.setSiteResourceAiProviders +); + +authenticated.post( + [ + "/site-resource/:siteResourceId/ai-providers/add", + "/private-resource/:siteResourceId/ai-providers/add" + ], + verifyApiKeySiteResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.addAiProviderToSiteResource +); + +authenticated.post( + [ + "/site-resource/:siteResourceId/ai-providers/remove", + "/private-resource/:siteResourceId/ai-providers/remove" + ], + verifyApiKeySiteResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + siteResource.removeAiProviderFromSiteResource +); + authenticated.post( [ "/site-resource/:siteResourceId/users/add", @@ -563,6 +606,16 @@ authenticated.get( resource.listResourceAiModels ); +authenticated.get( + [ + "/resource/:resourceId/ai-providers", + "/public-resource/:resourceId/ai-providers" + ], + verifyApiKeyResourceAccess, + verifyApiKeyHasAction(ActionsEnum.listResourceAiModels), + resource.listResourceAiProviders +); + authenticated.get( ["/resource/:resourceId", "/public-resource/:resourceId"], verifyApiKeyResourceAccess, @@ -775,6 +828,17 @@ authenticated.post( resource.setResourceAiModels ); +authenticated.post( + [ + "/resource/:resourceId/ai-providers", + "/public-resource/:resourceId/ai-providers" + ], + verifyApiKeyResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.setResourceAiProviders +); + authenticated.post( ["/resource/:resourceId/users", "/public-resource/:resourceId/users"], verifyApiKeyResourceAccess, @@ -989,6 +1053,28 @@ authenticated.post( resource.removeAiModelFromResource ); +authenticated.post( + [ + "/resource/:resourceId/ai-providers/add", + "/public-resource/:resourceId/ai-providers/add" + ], + verifyApiKeyResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.addAiProviderToResource +); + +authenticated.post( + [ + "/resource/:resourceId/ai-providers/remove", + "/public-resource/:resourceId/ai-providers/remove" + ], + verifyApiKeyResourceAccess, + verifyApiKeyHasAction(ActionsEnum.setResourceAiModels), + logActionAudit(ActionsEnum.setResourceAiModels), + resource.removeAiProviderFromResource +); + authenticated.post( [ "/resource/:resourceId/users/add", diff --git a/server/routers/resource/addAiModelToResource.ts b/server/routers/resource/addAiModelToResource.ts index fd33a3e91..3121c9ffa 100644 --- a/server/routers/resource/addAiModelToResource.ts +++ b/server/routers/resource/addAiModelToResource.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { db, resources, resourceAiModels, aiModels } from "@server/db"; +import { db, resources, resourceAiModels } from "@server/db"; import { eq, and } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -8,7 +8,10 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; - +import { + assertPublicAllowlistApiEligible, + assertModelsBelongToPublicAllowlistProviders +} from "@server/lib/aiInferenceResource"; const addAiModelToResourceBodySchema = z.strictObject({ modelId: z.int().positive() }); @@ -21,7 +24,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-models/add", description: - "Add a single AI model to a resource's model restriction allow-list.", + "Add a single catalog model to an inference resource allowlist. Requires at least one attached AI provider in allowlist mode. The model must belong to a provider attached in allowlist mode.", tags: [OpenAPITags.PublicResource], request: { params: addAiModelToResourceParamsSchema, @@ -95,33 +98,18 @@ export async function addAiModelToResource( ); } - if (!resource.aiProviderId) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "Resource has no AI provider linked" - ) - ); + const eligibleError = await assertPublicAllowlistApiEligible(resource); + if (eligibleError) { + return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); } - const [model] = await db - .select() - .from(aiModels) - .where( - and( - eq(aiModels.modelId, modelId), - eq(aiModels.providerId, resource.aiProviderId) - ) - ) - .limit(1); - - if (!model) { - return next( - createHttpError( - HttpCode.NOT_FOUND, - "Model not found or does not belong to this resource's AI provider" - ) - ); + const modelError = await assertModelsBelongToPublicAllowlistProviders({ + orgId: resource.orgId, + resourceId, + modelIds: [modelId] + }); + if (modelError) { + return next(createHttpError(HttpCode.BAD_REQUEST, modelError)); } const existingEntry = await db diff --git a/server/routers/resource/addAiProviderToResource.ts b/server/routers/resource/addAiProviderToResource.ts new file mode 100644 index 000000000..d51cf244c --- /dev/null +++ b/server/routers/resource/addAiProviderToResource.ts @@ -0,0 +1,152 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, resources } from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { + isInferenceFieldsError, + listPublicResourceAiProviders, + modelAccessModeSchema, + resolveProviderAttachments, + setPublicResourceAiProviders +} from "@server/lib/aiInferenceResource"; + +const addAiProviderToResourceBodySchema = z.strictObject({ + providerId: z.number().int().positive(), + modelAccessMode: modelAccessModeSchema.optional() +}); + +const addAiProviderToResourceParamsSchema = z.strictObject({ + resourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/resource/{resourceId}/ai-providers/add", + description: + "Add or replace a single AI provider attachment on an inference resource.", + tags: [OpenAPITags.PublicResource], + request: { + params: addAiProviderToResourceParamsSchema, + body: { + content: { + "application/json": { + schema: addAiProviderToResourceBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function addAiProviderToResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = addAiProviderToResourceBodySchema.safeParse( + req.body + ); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { providerId, modelAccessMode } = parsedBody.data; + + const parsedParams = addAiProviderToResourceParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { resourceId } = parsedParams.data; + + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Resource not found") + ); + } + + if (resource.mode !== "inference") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI providers can only be attached to inference-mode resources" + ) + ); + } + + const existing = await listPublicResourceAiProviders(resourceId); + const nextAttachments = [ + ...existing + .filter((a) => a.providerId !== providerId) + .map((a) => ({ + providerId: a.providerId, + modelAccessMode: a.modelAccessMode + })), + { providerId, modelAccessMode } + ]; + + const attachments = await resolveProviderAttachments({ + orgId: resource.orgId, + attachments: nextAttachments, + requireAtLeastOne: true + }); + if (isInferenceFieldsError(attachments)) { + return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + } + + await setPublicResourceAiProviders(resourceId, attachments); + + return response(res, { + data: {}, + success: true, + error: false, + message: "AI provider added to resource successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index e56896e55..4c880c3af 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -38,6 +38,13 @@ import { } from "@server/db/names"; import { usageService } from "@server/lib/billing/usageService"; import { LimitId } from "@server/lib/billing"; +import { + isInferenceFieldsError, + resolveProviderAttachments, + resourceAiProviderAttachmentSchema, + setPublicResourceAiProviders, + type ResourceAiProviderAttachment +} from "@server/lib/aiInferenceResource"; const createResourceParamsSchema = z.strictObject({ orgId: z.string() @@ -98,13 +105,11 @@ const createHttpResourceSchema = z authDaemonPort: z.int().positive().optional(), authDaemonMode: z.enum(["site", "remote", "native"]).optional(), // Inference settings - aiProviderId: z - .number() - .int() - .positive() + aiProviders: z + .array(resourceAiProviderAttachmentSchema) .optional() .describe( - "For inference-mode resources: the AI provider this resource proxies chat completions to." + "For inference-mode resources: AI providers to attach. Each entry may set modelAccessMode (passthrough, catalog, or allowlist); defaults to passthrough. At most one passthrough provider is allowed." ) }) .refine( @@ -377,11 +382,35 @@ async function createHttpResource( authDaemonPort, authDaemonMode, pamMode, - aiProviderId + aiProviders: aiProviderInputs } = parsedBody.data; const subdomain = parsedBody.data.subdomain; const stickySession = parsedBody.data.stickySession; + const effectiveMode = mode ?? "http"; + + let providerAttachments: ResourceAiProviderAttachment[] = []; + if (effectiveMode === "inference") { + const resolved = await resolveProviderAttachments({ + orgId, + attachments: aiProviderInputs ?? [], + requireAtLeastOne: true + }); + if (isInferenceFieldsError(resolved)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, resolved.error) + ); + } + providerAttachments = resolved; + } else if (aiProviderInputs && aiProviderInputs.length > 0) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI providers can only be attached to inference-mode resources" + ) + ); + } + // Wildcard subdomains are a paid feature if (subdomain && subdomain.includes("*")) { const isLicensed = await isLicensedOrSubscribed( @@ -422,7 +451,7 @@ async function createHttpResource( } if ( - ["ssh", "rdp", "vnc"].includes(mode!) && + ["ssh", "rdp", "vnc"].includes(effectiveMode) && !isLicensedOrSubscribed( orgId!, tierMatrix[TierFeature.AdvancedPublicResources] @@ -555,7 +584,7 @@ async function createHttpResource( orgId, name, subdomain: finalSubdomain, - mode: mode, + mode: effectiveMode, pamMode: pamMode, authDaemonMode: authDaemonMode, authDaemonPort: authDaemonPort, @@ -564,11 +593,18 @@ async function createHttpResource( postAuthPath: postAuthPath, wildcard, health: "unknown", - defaultResourcePolicyId: defaultPolicy.resourcePolicyId, - aiProviderId: aiProviderId ?? null + defaultResourcePolicyId: defaultPolicy.resourcePolicyId }) .returning(); + if (providerAttachments.length > 0) { + await setPublicResourceAiProviders( + newResource[0].resourceId, + providerAttachments, + trx + ); + } + await trx.insert(roleResources).values({ roleId: adminRole[0].roleId, resourceId: newResource[0].resourceId diff --git a/server/routers/resource/index.ts b/server/routers/resource/index.ts index 32992d364..a09cf0452 100644 --- a/server/routers/resource/index.ts +++ b/server/routers/resource/index.ts @@ -39,3 +39,7 @@ export * from "./listResourceAiModels"; export * from "./setResourceAiModels"; export * from "./addAiModelToResource"; export * from "./removeAiModelFromResource"; +export * from "./listResourceAiProviders"; +export * from "./setResourceAiProviders"; +export * from "./addAiProviderToResource"; +export * from "./removeAiProviderFromResource"; diff --git a/server/routers/resource/listResourceAiModels.ts b/server/routers/resource/listResourceAiModels.ts index f6875faff..0a5dea778 100644 --- a/server/routers/resource/listResourceAiModels.ts +++ b/server/routers/resource/listResourceAiModels.ts @@ -34,7 +34,7 @@ registry.registerPath({ method: "get", path: "/resource/{resourceId}/ai-models", description: - "List the AI models a resource is restricted to. An empty list means the resource is not restricted and every enabled model on its linked AI provider is allowed.", + "List catalog models on this resource's allowlist. Only enforced when modelAccessMode=allowlist; an empty allowlist denies all models.", tags: [OpenAPITags.PublicResource], request: { params: listResourceAiModelsParamsSchema diff --git a/server/routers/resource/listResourceAiProviders.ts b/server/routers/resource/listResourceAiProviders.ts new file mode 100644 index 000000000..2c81c45d5 --- /dev/null +++ b/server/routers/resource/listResourceAiProviders.ts @@ -0,0 +1,94 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, resources } from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { listPublicResourceAiProviders } from "@server/lib/aiInferenceResource"; + +const listResourceAiProvidersParamsSchema = z.strictObject({ + resourceId: z.coerce.number().int().positive() +}); + +export type ListResourceAiProvidersResponse = { + providers: Awaited>; +}; + +registry.registerPath({ + method: "get", + path: "/resource/{resourceId}/ai-providers", + description: "List AI providers attached to an inference resource.", + tags: [OpenAPITags.PublicResource], + request: { + params: listResourceAiProvidersParamsSchema + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function listResourceAiProviders( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = listResourceAiProvidersParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { resourceId } = parsedParams.data; + + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Resource not found") + ); + } + + const providers = await listPublicResourceAiProviders(resourceId); + + return response(res, { + data: { providers }, + success: true, + error: false, + message: "Resource AI providers retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/resource/removeAiModelFromResource.ts b/server/routers/resource/removeAiModelFromResource.ts index 47445bbf5..7b5cb012f 100644 --- a/server/routers/resource/removeAiModelFromResource.ts +++ b/server/routers/resource/removeAiModelFromResource.ts @@ -8,6 +8,7 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; +import { assertPublicAllowlistApiEligible } from "@server/lib/aiInferenceResource"; const removeAiModelFromResourceBodySchema = z.strictObject({ modelId: z.int().positive() @@ -21,7 +22,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-models/remove", description: - "Remove a single AI model from a resource's model restriction allow-list.", + "Remove a single catalog model from an inference resource allowlist. Requires at least one attached AI provider in allowlist mode.", tags: [OpenAPITags.PublicResource], request: { params: removeAiModelFromResourceParamsSchema, @@ -97,6 +98,11 @@ export async function removeAiModelFromResource( ); } + const eligibleError = await assertPublicAllowlistApiEligible(resource); + if (eligibleError) { + return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); + } + const existingEntry = await db .select() .from(resourceAiModels) diff --git a/server/routers/resource/removeAiProviderFromResource.ts b/server/routers/resource/removeAiProviderFromResource.ts new file mode 100644 index 000000000..d3a4294ad --- /dev/null +++ b/server/routers/resource/removeAiProviderFromResource.ts @@ -0,0 +1,165 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, resources } from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { + isInferenceFieldsError, + listPublicResourceAiProviders, + resolveProviderAttachments, + setPublicResourceAiProviders +} from "@server/lib/aiInferenceResource"; + +const removeAiProviderFromResourceBodySchema = z.strictObject({ + providerId: z.number().int().positive() +}); + +const removeAiProviderFromResourceParamsSchema = z.strictObject({ + resourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/resource/{resourceId}/ai-providers/remove", + description: + "Remove an AI provider attachment from an inference resource. At least one provider must remain.", + tags: [OpenAPITags.PublicResource], + request: { + params: removeAiProviderFromResourceParamsSchema, + body: { + content: { + "application/json": { + schema: removeAiProviderFromResourceBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function removeAiProviderFromResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = removeAiProviderFromResourceBodySchema.safeParse( + req.body + ); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { providerId } = parsedBody.data; + + const parsedParams = + removeAiProviderFromResourceParamsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { resourceId } = parsedParams.data; + + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Resource not found") + ); + } + + if (resource.mode !== "inference") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI providers can only be attached to inference-mode resources" + ) + ); + } + + const existing = await listPublicResourceAiProviders(resourceId); + const found = existing.find((a) => a.providerId === providerId); + if (!found) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + "AI provider is not attached to this resource" + ) + ); + } + + const remaining = existing + .filter((a) => a.providerId !== providerId) + .map((a) => ({ + providerId: a.providerId, + modelAccessMode: a.modelAccessMode + })); + + if (remaining.length === 0) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "At least one AI provider is required for inference-mode resources" + ) + ); + } + + const attachments = await resolveProviderAttachments({ + orgId: resource.orgId, + attachments: remaining, + requireAtLeastOne: true + }); + if (isInferenceFieldsError(attachments)) { + return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + } + + await setPublicResourceAiProviders(resourceId, attachments); + + return response(res, { + data: {}, + success: true, + error: false, + message: "AI provider removed from resource successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/resource/setResourceAiModels.ts b/server/routers/resource/setResourceAiModels.ts index 2ef56f4fc..0c3323ebe 100644 --- a/server/routers/resource/setResourceAiModels.ts +++ b/server/routers/resource/setResourceAiModels.ts @@ -1,13 +1,17 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { db, resources, resourceAiModels, aiModels } from "@server/db"; -import { eq, and, inArray } from "drizzle-orm"; +import { db, resources, resourceAiModels } from "@server/db"; +import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; +import { + assertPublicAllowlistApiEligible, + assertModelsBelongToPublicAllowlistProviders +} from "@server/lib/aiInferenceResource"; const setResourceAiModelsBodySchema = z.strictObject({ modelIds: z.array(z.int().positive()) @@ -21,7 +25,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-models", description: - "Set the AI models a resource is restricted to. This replaces all existing restrictions. Pass an empty array to remove the restriction (allow every enabled model on the linked provider).", + "Replace the allowlist of catalog models for an inference resource. Requires at least one attached AI provider in allowlist mode. Models must belong to a provider attached in allowlist mode. An empty array denies all models.", tags: [OpenAPITags.PublicResource], request: { params: setResourceAiModelsParamsSchema, @@ -95,34 +99,18 @@ export async function setResourceAiModels( ); } - if (modelIds.length > 0) { - if (!resource.aiProviderId) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "Resource has no AI provider linked" - ) - ); - } + const eligibleError = await assertPublicAllowlistApiEligible(resource); + if (eligibleError) { + return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); + } - const validModels = await db - .select({ modelId: aiModels.modelId }) - .from(aiModels) - .where( - and( - inArray(aiModels.modelId, modelIds), - eq(aiModels.providerId, resource.aiProviderId) - ) - ); - - if (validModels.length !== new Set(modelIds).size) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "One or more model IDs do not exist or do not belong to this resource's AI provider" - ) - ); - } + const modelError = await assertModelsBelongToPublicAllowlistProviders({ + orgId: resource.orgId, + resourceId, + modelIds + }); + if (modelError) { + return next(createHttpError(HttpCode.BAD_REQUEST, modelError)); } await db.transaction(async (trx) => { diff --git a/server/routers/resource/setResourceAiProviders.ts b/server/routers/resource/setResourceAiProviders.ts new file mode 100644 index 000000000..a1de8f176 --- /dev/null +++ b/server/routers/resource/setResourceAiProviders.ts @@ -0,0 +1,137 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, resources } from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { + isInferenceFieldsError, + resolveProviderAttachments, + resourceAiProviderAttachmentSchema, + setPublicResourceAiProviders +} from "@server/lib/aiInferenceResource"; + +const setResourceAiProvidersBodySchema = z.strictObject({ + providers: z.array(resourceAiProviderAttachmentSchema) +}); + +const setResourceAiProvidersParamsSchema = z.strictObject({ + resourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/resource/{resourceId}/ai-providers", + description: + "Replace the AI providers attached to an inference resource. At least one provider is required. At most one may use passthrough mode.", + tags: [OpenAPITags.PublicResource], + request: { + params: setResourceAiProvidersParamsSchema, + body: { + content: { + "application/json": { + schema: setResourceAiProvidersBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function setResourceAiProviders( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = setResourceAiProvidersBodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { providers } = parsedBody.data; + + const parsedParams = setResourceAiProvidersParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { resourceId } = parsedParams.data; + + const [resource] = await db + .select() + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + + if (!resource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Resource not found") + ); + } + + if (resource.mode !== "inference") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI providers can only be attached to inference-mode resources" + ) + ); + } + + const attachments = await resolveProviderAttachments({ + orgId: resource.orgId, + attachments: providers, + requireAtLeastOne: true + }); + if (isInferenceFieldsError(attachments)) { + return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + } + + await setPublicResourceAiProviders(resourceId, attachments); + + return response(res, { + data: {}, + success: true, + error: false, + message: "AI providers set for resource successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/resource/updateResource.ts b/server/routers/resource/updateResource.ts index c67cb2795..0d6f8f8aa 100644 --- a/server/routers/resource/updateResource.ts +++ b/server/routers/resource/updateResource.ts @@ -120,15 +120,6 @@ const updateHttpResourceBodySchema = z .optional() .describe( "ID of the resource policy to apply to this resource. Set to null to remove the resource policy and fall back to the inline policy settings." - ), - aiProviderId: z - .number() - .int() - .positive() - .nullable() - .optional() - .describe( - "For inference-mode resources: the AI provider this resource proxies chat completions to. Set to null to unlink." ) }) .refine((data) => Object.keys(data).length > 0, { @@ -354,8 +345,10 @@ export async function updateResource( ); } - if (["http", "ssh", "rdp", "vnc"].includes(resource.mode)) { - // HANDLE UPDATING HTTP RESOURCES + if ( + ["http", "ssh", "rdp", "vnc", "inference"].includes(resource.mode) + ) { + // HANDLE UPDATING HTTP / BROWSER / INFERENCE RESOURCES return await updateHttpResource( { req, diff --git a/server/routers/siteResource/addAiModelToSiteResource.ts b/server/routers/siteResource/addAiModelToSiteResource.ts index 2e55f100b..0b6a004d7 100644 --- a/server/routers/siteResource/addAiModelToSiteResource.ts +++ b/server/routers/siteResource/addAiModelToSiteResource.ts @@ -1,11 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { - db, - siteResources, - siteResourceAiModels, - aiModels -} from "@server/db"; +import { db, siteResources, siteResourceAiModels } from "@server/db"; import { eq, and } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -13,6 +8,10 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; +import { + assertSiteAllowlistApiEligible, + assertModelsBelongToSiteAllowlistProviders +} from "@server/lib/aiInferenceResource"; const addAiModelToSiteResourceBodySchema = z.strictObject({ modelId: z.int().positive() @@ -26,7 +25,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-models/add", description: - "Add a single AI model to a site resource's model restriction allow-list.", + "Add a single catalog model to an inference site resource allowlist. Requires at least one attached AI provider in allowlist mode. The model must belong to a provider attached in allowlist mode.", tags: [OpenAPITags.PrivateResource], request: { params: addAiModelToSiteResourceParamsSchema, @@ -102,33 +101,19 @@ export async function addAiModelToSiteResource( ); } - if (!siteResource.aiProviderId) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "Site resource has no AI provider linked" - ) - ); + const eligibleError = + await assertSiteAllowlistApiEligible(siteResource); + if (eligibleError) { + return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); } - const [model] = await db - .select() - .from(aiModels) - .where( - and( - eq(aiModels.modelId, modelId), - eq(aiModels.providerId, siteResource.aiProviderId) - ) - ) - .limit(1); - - if (!model) { - return next( - createHttpError( - HttpCode.NOT_FOUND, - "Model not found or does not belong to this site resource's AI provider" - ) - ); + const modelError = await assertModelsBelongToSiteAllowlistProviders({ + orgId: siteResource.orgId, + siteResourceId, + modelIds: [modelId] + }); + if (modelError) { + return next(createHttpError(HttpCode.BAD_REQUEST, modelError)); } const existingEntry = await db diff --git a/server/routers/siteResource/addAiProviderToSiteResource.ts b/server/routers/siteResource/addAiProviderToSiteResource.ts new file mode 100644 index 000000000..bae346347 --- /dev/null +++ b/server/routers/siteResource/addAiProviderToSiteResource.ts @@ -0,0 +1,152 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, siteResources } from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { + isInferenceFieldsError, + listSiteResourceAiProviders, + modelAccessModeSchema, + resolveProviderAttachments, + setSiteResourceAiProviders +} from "@server/lib/aiInferenceResource"; + +const addAiProviderToSiteResourceBodySchema = z.strictObject({ + providerId: z.number().int().positive(), + modelAccessMode: modelAccessModeSchema.optional() +}); + +const addAiProviderToSiteResourceParamsSchema = z.strictObject({ + siteResourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/site-resource/{siteResourceId}/ai-providers/add", + description: + "Add or replace a single AI provider attachment on an inference site resource.", + tags: [OpenAPITags.PrivateResource], + request: { + params: addAiProviderToSiteResourceParamsSchema, + body: { + content: { + "application/json": { + schema: addAiProviderToSiteResourceBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function addAiProviderToSiteResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = addAiProviderToSiteResourceBodySchema.safeParse( + req.body + ); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { providerId, modelAccessMode } = parsedBody.data; + + const parsedParams = addAiProviderToSiteResourceParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteResourceId } = parsedParams.data; + + const [siteResource] = await db + .select() + .from(siteResources) + .where(eq(siteResources.siteResourceId, siteResourceId)) + .limit(1); + + if (!siteResource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Site resource not found") + ); + } + + if (siteResource.mode !== "inference") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI providers can only be attached to inference-mode resources" + ) + ); + } + + const existing = await listSiteResourceAiProviders(siteResourceId); + const nextAttachments = [ + ...existing + .filter((a) => a.providerId !== providerId) + .map((a) => ({ + providerId: a.providerId, + modelAccessMode: a.modelAccessMode + })), + { providerId, modelAccessMode } + ]; + + const attachments = await resolveProviderAttachments({ + orgId: siteResource.orgId, + attachments: nextAttachments, + requireAtLeastOne: true + }); + if (isInferenceFieldsError(attachments)) { + return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + } + + await setSiteResourceAiProviders(siteResourceId, attachments); + + return response(res, { + data: {}, + success: true, + error: false, + message: "AI provider added to site resource successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 3244668f5..47dcfc59b 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -39,6 +39,13 @@ import { createCertificate } from "#dynamic/routers/certificates/createCertifica import { build } from "@server/build"; import { usageService } from "@server/lib/billing/usageService"; import { LimitId } from "@server/lib/billing"; +import { + isInferenceFieldsError, + resolveProviderAttachments, + resourceAiProviderAttachmentSchema, + setSiteResourceAiProviders, + type ResourceAiProviderAttachment +} from "@server/lib/aiInferenceResource"; const createSiteResourceParamsSchema = z.strictObject({ orgId: z.string() @@ -79,13 +86,11 @@ const createSiteResourceSchema = z pamMode: z.enum(["passthrough", "push"]).optional(), domainId: z.string().optional(), // only used for http mode, we need this to verify the alias is unique within the org subdomain: z.string().optional(), // only used for http mode, we need this to verify the alias is unique within the org - aiProviderId: z - .number() - .int() - .positive() + aiProviders: z + .array(resourceAiProviderAttachmentSchema) .optional() .describe( - "For inference-mode site resources: the AI provider this resource proxies chat completions to." + "For inference-mode site resources: AI providers to attach. Each entry may set modelAccessMode (passthrough, catalog, or allowlist); defaults to passthrough. At most one passthrough provider is allowed." ) }) .strict() @@ -334,7 +339,7 @@ export async function createSiteResource( pamMode, domainId, subdomain, - aiProviderId + aiProviders: aiProviderInputs } = parsedBody.data; // Backward compatibility: merge deprecated siteId into siteIds array @@ -343,6 +348,28 @@ export async function createSiteResource( siteIds.push(siteId); } + let providerAttachments: ResourceAiProviderAttachment[] = []; + if (mode === "inference") { + const resolved = await resolveProviderAttachments({ + orgId, + attachments: aiProviderInputs ?? [], + requireAtLeastOne: true + }); + if (isInferenceFieldsError(resolved)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, resolved.error) + ); + } + providerAttachments = resolved; + } else if (aiProviderInputs && aiProviderInputs.length > 0) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI providers can only be attached to inference-mode resources" + ) + ); + } + if (build == "saas") { const usage = await usageService.getUsage( orgId, @@ -608,8 +635,7 @@ export async function createSiteResource( domainId, subdomain: finalSubdomain, fullDomain, - requiresExitNodeConnection: mode === "inference", // in the future we might want to have different modes that do this - aiProviderId: aiProviderId ?? null + requiresExitNodeConnection: mode === "inference" // in the future we might want to have different modes that do this }; if (isLicensedSshPam) { if (authDaemonPort !== undefined) @@ -625,6 +651,14 @@ export async function createSiteResource( const siteResourceId = newSiteResource.siteResourceId; + if (providerAttachments.length > 0) { + await setSiteResourceAiProviders( + siteResourceId, + providerAttachments, + trx + ); + } + //////////////////// update the associations //////////////////// if (network) { diff --git a/server/routers/siteResource/index.ts b/server/routers/siteResource/index.ts index 2acaf1e33..046ed0a13 100644 --- a/server/routers/siteResource/index.ts +++ b/server/routers/siteResource/index.ts @@ -21,3 +21,7 @@ export * from "./listSiteResourceAiModels"; export * from "./setSiteResourceAiModels"; export * from "./addAiModelToSiteResource"; export * from "./removeAiModelFromSiteResource"; +export * from "./listSiteResourceAiProviders"; +export * from "./setSiteResourceAiProviders"; +export * from "./addAiProviderToSiteResource"; +export * from "./removeAiProviderFromSiteResource"; diff --git a/server/routers/siteResource/listSiteResourceAiModels.ts b/server/routers/siteResource/listSiteResourceAiModels.ts index 26dea7cd0..26e709140 100644 --- a/server/routers/siteResource/listSiteResourceAiModels.ts +++ b/server/routers/siteResource/listSiteResourceAiModels.ts @@ -1,11 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { - db, - siteResources, - siteResourceAiModels, - aiModels -} from "@server/db"; +import { db, siteResources, siteResourceAiModels, aiModels } from "@server/db"; import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -27,10 +22,7 @@ async function query(siteResourceId: number) { enabled: aiModels.enabled }) .from(siteResourceAiModels) - .innerJoin( - aiModels, - eq(siteResourceAiModels.modelId, aiModels.modelId) - ) + .innerJoin(aiModels, eq(siteResourceAiModels.modelId, aiModels.modelId)) .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); } @@ -42,7 +34,7 @@ registry.registerPath({ method: "get", path: "/site-resource/{siteResourceId}/ai-models", description: - "List the AI models a site resource is restricted to. An empty list means the site resource is not restricted and every enabled model on its linked AI provider is allowed.", + "List catalog models on this site resource's allowlist. Only enforced when modelAccessMode=allowlist; an empty allowlist denies all models.", tags: [OpenAPITags.PrivateResource], request: { params: listSiteResourceAiModelsParamsSchema diff --git a/server/routers/siteResource/listSiteResourceAiProviders.ts b/server/routers/siteResource/listSiteResourceAiProviders.ts new file mode 100644 index 000000000..bcbe37e8b --- /dev/null +++ b/server/routers/siteResource/listSiteResourceAiProviders.ts @@ -0,0 +1,94 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, siteResources } from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { listSiteResourceAiProviders as listAttachments } from "@server/lib/aiInferenceResource"; + +const listSiteResourceAiProvidersParamsSchema = z.strictObject({ + siteResourceId: z.coerce.number().int().positive() +}); + +export type ListSiteResourceAiProvidersResponse = { + providers: Awaited>; +}; + +registry.registerPath({ + method: "get", + path: "/site-resource/{siteResourceId}/ai-providers", + description: "List AI providers attached to an inference site resource.", + tags: [OpenAPITags.PrivateResource], + request: { + params: listSiteResourceAiProvidersParamsSchema + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function listSiteResourceAiProviders( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = listSiteResourceAiProvidersParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteResourceId } = parsedParams.data; + + const [siteResource] = await db + .select() + .from(siteResources) + .where(eq(siteResources.siteResourceId, siteResourceId)) + .limit(1); + + if (!siteResource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Site resource not found") + ); + } + + const providers = await listAttachments(siteResourceId); + + return response(res, { + data: { providers }, + success: true, + error: false, + message: "Site resource AI providers retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/siteResource/removeAiModelFromSiteResource.ts b/server/routers/siteResource/removeAiModelFromSiteResource.ts index 5af2f4d87..eb8d649a2 100644 --- a/server/routers/siteResource/removeAiModelFromSiteResource.ts +++ b/server/routers/siteResource/removeAiModelFromSiteResource.ts @@ -8,6 +8,7 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; +import { assertSiteAllowlistApiEligible } from "@server/lib/aiInferenceResource"; const removeAiModelFromSiteResourceBodySchema = z.strictObject({ modelId: z.int().positive() @@ -21,7 +22,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-models/remove", description: - "Remove a single AI model from a site resource's model restriction allow-list.", + "Remove a single catalog model from an inference site resource allowlist. Requires at least one attached AI provider in allowlist mode.", tags: [OpenAPITags.PrivateResource], request: { params: removeAiModelFromSiteResourceParamsSchema, @@ -96,6 +97,12 @@ export async function removeAiModelFromSiteResource( ); } + const eligibleError = + await assertSiteAllowlistApiEligible(siteResource); + if (eligibleError) { + return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); + } + const existingEntry = await db .select() .from(siteResourceAiModels) diff --git a/server/routers/siteResource/removeAiProviderFromSiteResource.ts b/server/routers/siteResource/removeAiProviderFromSiteResource.ts new file mode 100644 index 000000000..65781859c --- /dev/null +++ b/server/routers/siteResource/removeAiProviderFromSiteResource.ts @@ -0,0 +1,164 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, siteResources } from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { + isInferenceFieldsError, + listSiteResourceAiProviders, + resolveProviderAttachments, + setSiteResourceAiProviders +} from "@server/lib/aiInferenceResource"; + +const removeAiProviderFromSiteResourceBodySchema = z.strictObject({ + providerId: z.number().int().positive() +}); + +const removeAiProviderFromSiteResourceParamsSchema = z.strictObject({ + siteResourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/site-resource/{siteResourceId}/ai-providers/remove", + description: + "Remove an AI provider attachment from an inference site resource. At least one provider must remain.", + tags: [OpenAPITags.PrivateResource], + request: { + params: removeAiProviderFromSiteResourceParamsSchema, + body: { + content: { + "application/json": { + schema: removeAiProviderFromSiteResourceBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function removeAiProviderFromSiteResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = + removeAiProviderFromSiteResourceBodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { providerId } = parsedBody.data; + + const parsedParams = + removeAiProviderFromSiteResourceParamsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteResourceId } = parsedParams.data; + + const [siteResource] = await db + .select() + .from(siteResources) + .where(eq(siteResources.siteResourceId, siteResourceId)) + .limit(1); + + if (!siteResource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Site resource not found") + ); + } + + if (siteResource.mode !== "inference") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI providers can only be attached to inference-mode resources" + ) + ); + } + + const existing = await listSiteResourceAiProviders(siteResourceId); + const found = existing.find((a) => a.providerId === providerId); + if (!found) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + "AI provider is not attached to this site resource" + ) + ); + } + + const remaining = existing + .filter((a) => a.providerId !== providerId) + .map((a) => ({ + providerId: a.providerId, + modelAccessMode: a.modelAccessMode + })); + + if (remaining.length === 0) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "At least one AI provider is required for inference-mode resources" + ) + ); + } + + const attachments = await resolveProviderAttachments({ + orgId: siteResource.orgId, + attachments: remaining, + requireAtLeastOne: true + }); + if (isInferenceFieldsError(attachments)) { + return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + } + + await setSiteResourceAiProviders(siteResourceId, attachments); + + return response(res, { + data: {}, + success: true, + error: false, + message: "AI provider removed from site resource successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/siteResource/setSiteResourceAiModels.ts b/server/routers/siteResource/setSiteResourceAiModels.ts index 1366187af..391634553 100644 --- a/server/routers/siteResource/setSiteResourceAiModels.ts +++ b/server/routers/siteResource/setSiteResourceAiModels.ts @@ -1,18 +1,17 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { - db, - siteResources, - siteResourceAiModels, - aiModels -} from "@server/db"; -import { eq, and, inArray } from "drizzle-orm"; +import { db, siteResources, siteResourceAiModels } from "@server/db"; +import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; +import { + assertSiteAllowlistApiEligible, + assertModelsBelongToSiteAllowlistProviders +} from "@server/lib/aiInferenceResource"; const setSiteResourceAiModelsBodySchema = z.strictObject({ modelIds: z.array(z.int().positive()) @@ -26,7 +25,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-models", description: - "Set the AI models a site resource is restricted to. This replaces all existing restrictions. Pass an empty array to remove the restriction (allow every enabled model on the linked provider).", + "Replace the allowlist of catalog models for an inference site resource. Requires at least one attached AI provider in allowlist mode. Models must belong to a provider attached in allowlist mode. An empty array denies all models.", tags: [OpenAPITags.PrivateResource], request: { params: setSiteResourceAiModelsParamsSchema, @@ -102,52 +101,33 @@ export async function setSiteResourceAiModels( ); } - if (modelIds.length > 0) { - if (!siteResource.aiProviderId) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "Site resource has no AI provider linked" - ) - ); - } + const eligibleError = + await assertSiteAllowlistApiEligible(siteResource); + if (eligibleError) { + return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); + } - const validModels = await db - .select({ modelId: aiModels.modelId }) - .from(aiModels) - .where( - and( - inArray(aiModels.modelId, modelIds), - eq(aiModels.providerId, siteResource.aiProviderId) - ) - ); - - if (validModels.length !== new Set(modelIds).size) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "One or more model IDs do not exist or do not belong to this site resource's AI provider" - ) - ); - } + const modelError = await assertModelsBelongToSiteAllowlistProviders({ + orgId: siteResource.orgId, + siteResourceId, + modelIds + }); + if (modelError) { + return next(createHttpError(HttpCode.BAD_REQUEST, modelError)); } await db.transaction(async (trx) => { await trx .delete(siteResourceAiModels) - .where( - eq(siteResourceAiModels.siteResourceId, siteResourceId) - ); + .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); if (modelIds.length > 0) { - await trx - .insert(siteResourceAiModels) - .values( - modelIds.map((modelId) => ({ - siteResourceId, - modelId - })) - ); + await trx.insert(siteResourceAiModels).values( + modelIds.map((modelId) => ({ + siteResourceId, + modelId + })) + ); } }); diff --git a/server/routers/siteResource/setSiteResourceAiProviders.ts b/server/routers/siteResource/setSiteResourceAiProviders.ts new file mode 100644 index 000000000..93bf8a866 --- /dev/null +++ b/server/routers/siteResource/setSiteResourceAiProviders.ts @@ -0,0 +1,139 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, siteResources } from "@server/db"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { + isInferenceFieldsError, + resolveProviderAttachments, + resourceAiProviderAttachmentSchema, + setSiteResourceAiProviders as replaceAttachments +} from "@server/lib/aiInferenceResource"; + +const setSiteResourceAiProvidersBodySchema = z.strictObject({ + providers: z.array(resourceAiProviderAttachmentSchema) +}); + +const setSiteResourceAiProvidersParamsSchema = z.strictObject({ + siteResourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "post", + path: "/site-resource/{siteResourceId}/ai-providers", + description: + "Replace the AI providers attached to an inference site resource. At least one provider is required. At most one may use passthrough mode.", + tags: [OpenAPITags.PrivateResource], + request: { + params: setSiteResourceAiProvidersParamsSchema, + body: { + content: { + "application/json": { + schema: setSiteResourceAiProvidersBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export async function setSiteResourceAiProviders( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = setSiteResourceAiProvidersBodySchema.safeParse( + req.body + ); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { providers } = parsedBody.data; + + const parsedParams = setSiteResourceAiProvidersParamsSchema.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteResourceId } = parsedParams.data; + + const [siteResource] = await db + .select() + .from(siteResources) + .where(eq(siteResources.siteResourceId, siteResourceId)) + .limit(1); + + if (!siteResource) { + return next( + createHttpError(HttpCode.NOT_FOUND, "Site resource not found") + ); + } + + if (siteResource.mode !== "inference") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI providers can only be attached to inference-mode resources" + ) + ); + } + + const attachments = await resolveProviderAttachments({ + orgId: siteResource.orgId, + attachments: providers, + requireAtLeastOne: true + }); + if (isInferenceFieldsError(attachments)) { + return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + } + + await replaceAttachments(siteResourceId, attachments); + + return response(res, { + data: {}, + success: true, + error: false, + message: "AI providers set for site resource successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index caf524804..e99d60e1e 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -29,6 +29,7 @@ import { NextFunction, Request, Response } from "express"; import createHttpError from "http-errors"; import { z } from "zod"; import { fromError } from "zod-validation-error"; +import { clearSiteResourceAiConfig } from "@server/lib/aiInferenceResource"; const updateSiteResourceParamsSchema = z.strictObject({ siteResourceId: z.coerce.number().int().positive() @@ -78,16 +79,7 @@ const updateSiteResourceSchema = z authDaemonMode: z.enum(["site", "remote", "native"]).optional(), pamMode: z.enum(["passthrough", "push"]).optional(), domainId: z.string().optional(), - subdomain: z.string().optional(), - aiProviderId: z - .number() - .int() - .positive() - .nullable() - .optional() - .describe( - "For inference-mode site resources: the AI provider this resource proxies chat completions to. Set to null to unlink." - ) + subdomain: z.string().optional() }) .strict() .refine( @@ -341,8 +333,7 @@ export async function updateSiteResource( authDaemonMode, pamMode, domainId, - subdomain, - aiProviderId + subdomain } = parsedBody.data; // Backward compatibility: merge deprecated siteId into siteIds array @@ -608,12 +599,19 @@ export async function updateSiteResource( networkId: mode === "inference" ? null : undefined, requiresExitNodeConnection: mode !== undefined ? mode === "inference" : undefined, - aiProviderId: aiProviderId, ...sshPamSet }) .where(and(eq(siteResources.siteResourceId, siteResourceId))) .returning(); + const effectiveMode = mode ?? existingSiteResource.mode; + if ( + existingSiteResource.mode === "inference" && + effectiveMode !== "inference" + ) { + await clearSiteResourceAiConfig(siteResourceId, trx); + } + //////////////////// update the associations //////////////////// if (mode === "inference") { diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx index f054c7901..02d1d2142 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx @@ -66,8 +66,6 @@ export default function AiProviderAuthenticationPage() { authType: (provider.authType as "bearer" | null) ?? "bearer", routingMode: (provider.routingMode as "url" | "target") ?? "url", skipTlsVerification: provider.skipTlsVerification, - budgetAmount: provider.budgetAmount, - budgetUnit: provider.budgetUnit as "usd" | "tokens" | null, enabled: provider.enabled } }); @@ -96,8 +94,6 @@ export default function AiProviderAuthenticationPage() { authType: (updated.authType as "bearer" | null) ?? "bearer", routingMode: (updated.routingMode as "url" | "target") ?? "url", skipTlsVerification: updated.skipTlsVerification, - budgetAmount: updated.budgetAmount, - budgetUnit: updated.budgetUnit as "usd" | "tokens" | null, enabled: updated.enabled }); toast({ diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx index 90ef74d10..2aa06d01d 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx @@ -75,8 +75,6 @@ export default function AiProviderNetworkPage() { authType: (provider.authType as "bearer" | null) ?? "bearer", routingMode: (provider.routingMode as "url" | "target") ?? "url", skipTlsVerification: provider.skipTlsVerification, - budgetAmount: provider.budgetAmount, - budgetUnit: provider.budgetUnit as "usd" | "tokens" | null, enabled: provider.enabled } }); @@ -120,8 +118,6 @@ export default function AiProviderNetworkPage() { authType: (updated.authType as "bearer" | null) ?? "bearer", routingMode: (updated.routingMode as "url" | "target") ?? "url", skipTlsVerification: updated.skipTlsVerification, - budgetAmount: updated.budgetAmount, - budgetUnit: updated.budgetUnit as "usd" | "tokens" | null, enabled: updated.enabled }); diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index ec6eee6c7..3111f0b3e 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -79,8 +79,6 @@ export default function CreateAiProviderPage() { authType: "bearer", routingMode: "url", skipTlsVerification: false, - budgetAmount: null, - budgetUnit: null, enabled: true } }); diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index 8861dcdbf..26245620c 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -26,8 +26,6 @@ export const aiProviderFormSchema = z authType: z.enum(["bearer"]).optional().nullable(), routingMode: z.enum(["url", "target"]).optional(), skipTlsVerification: z.boolean().optional(), - budgetAmount: z.number().positive().nullable().optional(), - budgetUnit: z.enum(["usd", "tokens"]).optional().nullable(), enabled: z.boolean().optional() }) .superRefine((data, ctx) => { @@ -78,20 +76,6 @@ export const aiProviderFormSchema = z path: ["authType"] }); } - - const hasAmount = - data.budgetAmount !== undefined && data.budgetAmount !== null; - const hasUnit = - data.budgetUnit !== undefined && data.budgetUnit !== null; - - if (hasAmount !== hasUnit) { - ctx.addIssue({ - code: "custom", - message: - "budgetAmount and budgetUnit must both be set or both omitted", - path: hasAmount ? ["budgetUnit"] : ["budgetAmount"] - }); - } }); export type AiProviderFormValues = z.infer; @@ -145,11 +129,6 @@ export function toAiProviderCreatePayload(values: AiProviderFormValues) { ? upstreamRaw : null; - const hasBudget = - values.budgetAmount !== undefined && - values.budgetAmount !== null && - values.budgetUnit; - return { name: values.name.trim(), type: values.type, @@ -161,8 +140,6 @@ export function toAiProviderCreatePayload(values: AiProviderFormValues) { ? (values.authType ?? "bearer") : (values.authType ?? undefined), skipTlsVerification: values.skipTlsVerification, - budgetAmount: hasBudget ? values.budgetAmount : null, - budgetUnit: hasBudget ? values.budgetUnit : null, enabled: values.enabled ?? true }; } @@ -178,11 +155,6 @@ export function toAiProviderUpdatePayload(values: AiProviderFormValues) { ? upstreamRaw : null; - const hasBudget = - values.budgetAmount !== undefined && - values.budgetAmount !== null && - values.budgetUnit; - const payload: Record = { name: values.name.trim(), routingMode: values.type === "custom" ? routingMode : "url", @@ -192,8 +164,6 @@ export function toAiProviderUpdatePayload(values: AiProviderFormValues) { ? (values.authType ?? "bearer") : (values.authType ?? null), skipTlsVerification: values.skipTlsVerification ?? false, - budgetAmount: hasBudget ? values.budgetAmount : null, - budgetUnit: hasBudget ? values.budgetUnit : null, enabled: values.enabled ?? true }; From e99cd52e99c1a693001aa53e8e5948bc1acba0aa Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Tue, 4 Aug 2026 21:55:34 +0200 Subject: [PATCH 033/296] =?UTF-8?q?=F0=9F=90=9B=20Fix=20log=20retention=20?= =?UTF-8?q?access=20pulling=20from=20the=20wrong=20org=20settings=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/logAccessAudit.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/private/lib/logAccessAudit.ts b/server/private/lib/logAccessAudit.ts index ff0c1d779..12b30f99f 100644 --- a/server/private/lib/logAccessAudit.ts +++ b/server/private/lib/logAccessAudit.ts @@ -28,7 +28,7 @@ async function getAccessDays(orgId: string): Promise { const [org] = await db .select({ - settingsLogRetentionDaysAction: orgs.settingsLogRetentionDaysAction + settingsLogRetentionDaysAccess: orgs.settingsLogRetentionDaysAccess }) .from(orgs) .where(eq(orgs.orgId, orgId)) @@ -41,11 +41,11 @@ async function getAccessDays(orgId: string): Promise { // store the result in cache await cache.set( `org_${orgId}_accessDays`, - org.settingsLogRetentionDaysAction, + org.settingsLogRetentionDaysAccess, 300 ); - return org.settingsLogRetentionDaysAction; + return org.settingsLogRetentionDaysAccess; } export async function cleanUpOldLogs(orgId: string, retentionDays: number) { From 7759d878352d09ab1cfde3d8cd7b89fc0e13fb39 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 4 Aug 2026 16:54:23 -0400 Subject: [PATCH 034/296] add basic ui for private inference resource --- messages/en-US.json | 14 ++ server/db/pg/schema/schema.ts | 8 +- server/db/sqlite/schema/schema.ts | 8 +- server/lib/aiInferenceResource.ts | 79 ++++-- server/routers/aiGateway/chatCompletions.ts | 32 --- server/routers/resource/createResource.ts | 6 +- .../resource/setResourceAiProviders.ts | 6 +- .../siteResource/createSiteResource.ts | 2 +- .../setSiteResourceAiProviders.ts | 6 +- .../ai-providers/[providerId]/layout.tsx | 4 + .../ai-providers/[providerId]/models/page.tsx | 150 +++++++++++ .../private/[niceId]/general/page.tsx | 30 ++- .../private/[niceId]/inference/page.tsx | 112 +-------- .../resources/private/[niceId]/layout.tsx | 45 ++-- .../private/[niceId]/providers/page.tsx | 232 ++++++++++++++++++ .../resources/private/create/page.tsx | 91 ++++++- src/components/AiProvidersSelector.tsx | 61 +++++ src/lib/privateResourceForm.ts | 92 +++++-- src/lib/queries.ts | 58 +++++ 19 files changed, 828 insertions(+), 208 deletions(-) create mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx create mode 100644 src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx create mode 100644 src/components/AiProvidersSelector.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 81a60f0a1..2fc4e778e 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1714,6 +1714,20 @@ "aiProviderQuestionRemove": "Are you sure you want to delete this AI provider?", "aiProviderMessageRemove": "This will permanently delete the provider and its models and targets. This cannot be undone.", "aiProviderErrorNoUpdate": "AI provider is not available to update", + "aiProviderModels": "Models", + "aiProviderModelsDescription": "Define model names available on this provider. Requests must use one of these model keys.", + "aiProviderModelsPlaceholder": "Type a model name and press Enter", + "aiProviderModelsUpdated": "Models updated", + "aiProviderModelsErrorUpdate": "Failed to update models", + "aiResourceProviders": "Providers", + "aiResourceProvidersDescription": "Choose which AI providers this inference resource can use", + "aiResourceProvidersHelp": "Models must be defined on each provider. Model names cannot overlap across selected providers.", + "aiResourceProvidersSelect": "Select providers", + "aiResourceProvidersEmpty": "No AI providers found", + "aiResourceProvidersRequired": "Select at least one AI provider", + "aiResourceProvidersUpdated": "Providers updated", + "aiResourceProvidersErrorUpdate": "Failed to update providers", + "aiResourceAliasRequired": "Alias is required for inference resources", "sidebarApiKeys": "API Keys", "sidebarProvisioning": "Provisioning", "sidebarSettings": "Settings", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index b323ae411..868b2a121 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -232,9 +232,9 @@ export const resourceAiProviders = pgTable( .notNull() .references(() => aiProviders.providerId, { onDelete: "cascade" }), modelAccessMode: varchar("modelAccessMode") - .$type<"passthrough" | "catalog" | "allowlist">() + .$type<"catalog" | "allowlist">() .notNull() - .default("passthrough") + .default("catalog") }, (t) => [primaryKey({ columns: [t.resourceId, t.providerId] })] ); @@ -523,9 +523,9 @@ export const siteResourceAiProviders = pgTable( .notNull() .references(() => aiProviders.providerId, { onDelete: "cascade" }), modelAccessMode: varchar("modelAccessMode") - .$type<"passthrough" | "catalog" | "allowlist">() + .$type<"catalog" | "allowlist">() .notNull() - .default("passthrough") + .default("catalog") }, (t) => [primaryKey({ columns: [t.siteResourceId, t.providerId] })] ); diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index c8f25dfed..2ae941895 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -229,9 +229,9 @@ export const resourceAiProviders = sqliteTable( .notNull() .references(() => aiProviders.providerId, { onDelete: "cascade" }), modelAccessMode: text("modelAccessMode") - .$type<"passthrough" | "catalog" | "allowlist">() + .$type<"catalog" | "allowlist">() .notNull() - .default("passthrough") + .default("catalog") }, (t) => [primaryKey({ columns: [t.resourceId, t.providerId] })] ); @@ -508,9 +508,9 @@ export const siteResourceAiProviders = sqliteTable( .notNull() .references(() => aiProviders.providerId, { onDelete: "cascade" }), modelAccessMode: text("modelAccessMode") - .$type<"passthrough" | "catalog" | "allowlist">() + .$type<"catalog" | "allowlist">() .notNull() - .default("passthrough") + .default("catalog") }, (t) => [primaryKey({ columns: [t.siteResourceId, t.providerId] })] ); diff --git a/server/lib/aiInferenceResource.ts b/server/lib/aiInferenceResource.ts index 4175a11e8..6e0f490d8 100644 --- a/server/lib/aiInferenceResource.ts +++ b/server/lib/aiInferenceResource.ts @@ -13,11 +13,7 @@ import { z } from "zod"; type DbOrTrx = Transaction | typeof db; -export const modelAccessModeSchema = z.enum([ - "passthrough", - "catalog", - "allowlist" -]); +export const modelAccessModeSchema = z.enum(["catalog", "allowlist"]); export type ModelAccessMode = z.infer; @@ -50,10 +46,7 @@ function normalizeAttachments( ): ResourceAiProviderAttachment[] { const byProvider = new Map(); for (const input of inputs) { - byProvider.set( - input.providerId, - input.modelAccessMode ?? "passthrough" - ); + byProvider.set(input.providerId, input.modelAccessMode ?? "catalog"); } return [...byProvider.entries()].map(([providerId, modelAccessMode]) => ({ providerId, @@ -61,9 +54,61 @@ function normalizeAttachments( })); } +/** + * Ensure enabled catalog modelKeys are unique across attached providers. + * Catalog attachments contribute all enabled models on the provider. + * Allowlist attachments contribute nothing until models are allowlisted + * (those are checked when the allowlist is set). + */ +export async function assertNoOverlappingModelKeys( + attachments: ResourceAiProviderAttachment[], + trx: DbOrTrx = db +): Promise { + const catalogProviderIds = attachments + .filter((a) => a.modelAccessMode === "catalog") + .map((a) => a.providerId); + + if (catalogProviderIds.length < 2) { + return null; + } + + const models = await trx + .select({ + providerId: aiModels.providerId, + modelKey: aiModels.modelKey + }) + .from(aiModels) + .where( + and( + inArray(aiModels.providerId, catalogProviderIds), + eq(aiModels.enabled, true) + ) + ); + + const keyToProviders = new Map(); + for (const model of models) { + const existing = keyToProviders.get(model.modelKey) ?? []; + if (!existing.includes(model.providerId)) { + existing.push(model.providerId); + } + keyToProviders.set(model.modelKey, existing); + } + + const overlaps = [...keyToProviders.entries()].filter( + ([, providerIds]) => providerIds.length > 1 + ); + if (overlaps.length === 0) { + return null; + } + + const keys = overlaps.map(([key]) => key).sort(); + return { + error: `Model keys must be unique across providers on a resource. Overlapping keys: ${keys.join(", ")}` + }; +} + /** * Validate provider attachments for an org. - * At most one passthrough provider is allowed per resource. */ export async function resolveProviderAttachments(input: { orgId: string; @@ -78,15 +123,6 @@ export async function resolveProviderAttachments(input: { }; } - const passthroughCount = attachments.filter( - (a) => a.modelAccessMode === "passthrough" - ).length; - if (passthroughCount > 1) { - return { - error: "A resource may have at most one AI provider in passthrough mode" - }; - } - if (attachments.length === 0) { return []; } @@ -119,6 +155,11 @@ export async function resolveProviderAttachments(input: { }; } + const overlapError = await assertNoOverlappingModelKeys(attachments); + if (overlapError) { + return overlapError; + } + return attachments; } diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index 51508778a..f1ac231ff 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -331,10 +331,6 @@ async function providerMatchesModel( requestedModel: string, allowedModelIds: number[] ): Promise { - if (attachment.modelAccessMode === "passthrough") { - return true; - } - const [matchedModel] = await db .select({ modelId: aiModels.modelId, @@ -366,26 +362,7 @@ async function selectProvider( allowedModelIds: number[], requestedModel: string | undefined ): Promise { - const passthroughAttachments = attachments.filter( - (a) => a.modelAccessMode === "passthrough" - ); - const hasRestricted = attachments.some( - (a) => - a.modelAccessMode === "catalog" || a.modelAccessMode === "allowlist" - ); - if (!requestedModel) { - if (hasRestricted) { - return { - ok: false, - status: HttpCode.FORBIDDEN, - message: - "This resource restricts access to specific models; a model must be specified" - }; - } - if (passthroughAttachments.length === 1) { - return { ok: true, provider: passthroughAttachments[0].provider }; - } return { ok: false, status: HttpCode.FORBIDDEN, @@ -395,10 +372,6 @@ async function selectProvider( const candidates: ProviderAttachment[] = []; for (const attachment of attachments) { - if (attachment.modelAccessMode === "passthrough") { - candidates.push(attachment); - continue; - } if ( await providerMatchesModel( attachment, @@ -422,11 +395,6 @@ async function selectProvider( }; } - // Zero candidates: fall back to a single passthrough attachment if present - if (passthroughAttachments.length === 1) { - return { ok: true, provider: passthroughAttachments[0].provider }; - } - return { ok: false, status: HttpCode.FORBIDDEN, diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index 4c880c3af..4d546468f 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -109,7 +109,7 @@ const createHttpResourceSchema = z .array(resourceAiProviderAttachmentSchema) .optional() .describe( - "For inference-mode resources: AI providers to attach. Each entry may set modelAccessMode (passthrough, catalog, or allowlist); defaults to passthrough. At most one passthrough provider is allowed." + "For inference-mode resources: AI providers to attach. Each entry may set modelAccessMode (catalog or allowlist); defaults to catalog. Model keys must be unique across attached catalog providers." ) }) .refine( @@ -397,9 +397,7 @@ async function createHttpResource( requireAtLeastOne: true }); if (isInferenceFieldsError(resolved)) { - return next( - createHttpError(HttpCode.BAD_REQUEST, resolved.error) - ); + return next(createHttpError(HttpCode.BAD_REQUEST, resolved.error)); } providerAttachments = resolved; } else if (aiProviderInputs && aiProviderInputs.length > 0) { diff --git a/server/routers/resource/setResourceAiProviders.ts b/server/routers/resource/setResourceAiProviders.ts index a1de8f176..0cd0d5d84 100644 --- a/server/routers/resource/setResourceAiProviders.ts +++ b/server/routers/resource/setResourceAiProviders.ts @@ -27,7 +27,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-providers", description: - "Replace the AI providers attached to an inference resource. At least one provider is required. At most one may use passthrough mode.", + "Replace the AI providers attached to an inference resource. At least one provider is required. Model keys must be unique across attached catalog providers.", tags: [OpenAPITags.PublicResource], request: { params: setResourceAiProvidersParamsSchema, @@ -116,7 +116,9 @@ export async function setResourceAiProviders( requireAtLeastOne: true }); if (isInferenceFieldsError(attachments)) { - return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + return next( + createHttpError(HttpCode.BAD_REQUEST, attachments.error) + ); } await setPublicResourceAiProviders(resourceId, attachments); diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 47dcfc59b..77d9d5f51 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -90,7 +90,7 @@ const createSiteResourceSchema = z .array(resourceAiProviderAttachmentSchema) .optional() .describe( - "For inference-mode site resources: AI providers to attach. Each entry may set modelAccessMode (passthrough, catalog, or allowlist); defaults to passthrough. At most one passthrough provider is allowed." + "For inference-mode site resources: AI providers to attach. Each entry may set modelAccessMode (catalog or allowlist); defaults to catalog. Model keys must be unique across attached catalog providers." ) }) .strict() diff --git a/server/routers/siteResource/setSiteResourceAiProviders.ts b/server/routers/siteResource/setSiteResourceAiProviders.ts index 93bf8a866..c6e708064 100644 --- a/server/routers/siteResource/setSiteResourceAiProviders.ts +++ b/server/routers/siteResource/setSiteResourceAiProviders.ts @@ -27,7 +27,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-providers", description: - "Replace the AI providers attached to an inference site resource. At least one provider is required. At most one may use passthrough mode.", + "Replace the AI providers attached to an inference site resource. At least one provider is required. Model keys must be unique across attached catalog providers.", tags: [OpenAPITags.PrivateResource], request: { params: setSiteResourceAiProvidersParamsSchema, @@ -118,7 +118,9 @@ export async function setSiteResourceAiProviders( requireAtLeastOne: true }); if (isInferenceFieldsError(attachments)) { - return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + return next( + createHttpError(HttpCode.BAD_REQUEST, attachments.error) + ); } await replaceAttachments(siteResourceId, attachments); diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx index 5846ad86d..4647347f5 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx @@ -69,6 +69,10 @@ export default async function AiProviderLayout({ children, params }: Props) { title: t("aiProviderNetworkSettings"), href: "/{orgId}/settings/ai-providers/{providerId}/network" }, + { + title: t("aiProviderModels"), + href: "/{orgId}/settings/ai-providers/{providerId}/models" + }, { title: t("aiProviderAuthSettings"), href: "/{orgId}/settings/ai-providers/{providerId}/authentication" diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx new file mode 100644 index 000000000..f90143492 --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx @@ -0,0 +1,150 @@ +"use client"; + +import { + SettingsContainer, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { TagInput, type Tag } from "@app/components/tags/tag-input"; +import { Button } from "@app/components/ui/button"; +import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { aiProviderQueries } from "@app/lib/queries"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useTranslations } from "next-intl"; +import { useEffect, useState } from "react"; + +export default function AiProviderModelsPage() { + const { provider } = useAiProviderContext(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const queryClient = useQueryClient(); + const t = useTranslations(); + const [saveLoading, setSaveLoading] = useState(false); + const [tags, setTags] = useState([]); + const [activeTagIndex, setActiveTagIndex] = useState(null); + + const modelsQuery = useQuery( + aiProviderQueries.providerModels({ providerId: provider.providerId }) + ); + + useEffect(() => { + if (!modelsQuery.data) return; + setTags( + modelsQuery.data.map((model) => ({ + id: String(model.modelId), + text: model.modelKey + })) + ); + }, [modelsQuery.data]); + + async function onSave() { + setSaveLoading(true); + try { + const existing = modelsQuery.data ?? []; + const existingByKey = new Map( + existing.map((model) => [model.modelKey, model]) + ); + const nextKeys = new Set( + tags.map((tag) => tag.text.trim()).filter(Boolean) + ); + + const toCreate = [...nextKeys].filter( + (key) => !existingByKey.has(key) + ); + const toDelete = existing.filter( + (model) => !nextKeys.has(model.modelKey) + ); + + await Promise.all([ + ...toCreate.map((modelKey) => + api.put(`/ai-provider/${provider.providerId}/model`, { + modelKey, + name: modelKey + }) + ), + ...toDelete.map((model) => + api.delete(`/ai-model/${model.modelId}`) + ) + ]); + + await queryClient.invalidateQueries( + aiProviderQueries.providerModels({ + providerId: provider.providerId + }) + ); + + toast({ + title: t("success"), + description: t("aiProviderModelsUpdated") + }); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiProviderModelsErrorUpdate"), + description: formatAxiosError( + e, + t("aiProviderModelsErrorUpdate") + ) + }); + } finally { + setSaveLoading(false); + } + } + + return ( + + + + + {t("aiProviderModels")} + + + {t("aiProviderModelsDescription")} + + + + + + { + const next = + typeof newTags === "function" + ? newTags(tags) + : newTags; + setTags(next as Tag[]); + }} + allowDuplicates={false} + sortTags + delimiterList={[",", "Enter"]} + disabled={modelsQuery.isLoading || saveLoading} + /> + + + + + + + + + ); +} diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/general/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/general/page.tsx index 8dbdb3809..b0af75ec0 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/general/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/general/page.tsx @@ -24,7 +24,9 @@ import { } from "@app/components/ui/form"; import { Input } from "@app/components/ui/input"; import { SwitchInput } from "@app/components/SwitchInput"; +import { PrivateResourceAliasField } from "@app/components/PrivateResourceDestinationFields"; import { createGeneralFormSchema } from "@app/lib/privateResourceForm"; +import { asAnyControl, asAnyWatch } from "@app/lib/formControlUtils"; import { zodResolver } from "@hookform/resolvers/zod"; import { useTranslations } from "next-intl"; import { useActionState, useMemo } from "react"; @@ -35,8 +37,12 @@ import { useSaveSiteResource } from "@app/hooks/useSaveSiteResource"; export default function PrivateResourceGeneralPage() { const t = useTranslations(); const { save, siteResource } = useSaveSiteResource(); + const isInference = siteResource.mode === "inference"; - const formSchema = useMemo(() => createGeneralFormSchema(t), [t]); + const formSchema = useMemo( + () => createGeneralFormSchema(t, { requireAlias: isInference }), + [t, isInference] + ); type FormValues = z.infer; const form = useForm({ @@ -44,7 +50,8 @@ export default function PrivateResourceGeneralPage() { defaultValues: { name: siteResource.name, niceId: siteResource.niceId, - enabled: siteResource.enabled + enabled: siteResource.enabled, + alias: siteResource.alias ?? null } }); @@ -56,7 +63,13 @@ export default function PrivateResourceGeneralPage() { await save({ name: data.name, niceId: data.niceId, - enabled: data.enabled + enabled: data.enabled, + ...(isInference + ? { + mode: "inference" as const, + alias: data.alias + } + : {}) }); }, null); @@ -152,6 +165,17 @@ export default function PrivateResourceGeneralPage() { )} /> + {isInference && ( + + + + )} diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx index 24aab5067..34534fdd4 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx @@ -1,105 +1,15 @@ -"use client"; +import type { Metadata } from "next"; +import { redirect } from "next/navigation"; -import { - SettingsContainer, - SettingsFormCell, - SettingsFormGrid, - SettingsSection, - SettingsSectionBody, - SettingsSectionDescription, - SettingsSectionFooter, - SettingsSectionForm, - SettingsSectionHeader, - SettingsSectionTitle -} from "@app/components/Settings"; -import { Button } from "@app/components/ui/button"; -import { Form } from "@app/components/ui/form"; -import { createInferenceFormSchema } from "@app/lib/privateResourceForm"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { useTranslations } from "next-intl"; -import { useActionState, useMemo, useState } from "react"; -import { useForm } from "react-hook-form"; -import { z } from "zod"; -import { PrivateResourceInferenceDestinationFields } from "@app/components/PrivateResourceDestinationFields"; -import { useSaveSiteResource } from "@app/hooks/useSaveSiteResource"; -import { - asAnyControl, - asAnySetValue, - asAnyWatch -} from "@app/lib/formControlUtils"; -import { buildSelectedSitesForResource } from "@app/lib/privateResourceUtils"; +export const metadata: Metadata = { + title: "Private Resource" +}; -export default function PrivateResourceInferencePage() { - const t = useTranslations(); - const { save, siteResource } = useSaveSiteResource(); - const [selectedSites, setSelectedSites] = useState(() => - buildSelectedSitesForResource(siteResource) - ); - - const formSchema = useMemo(() => createInferenceFormSchema(t), [t]); - type FormValues = z.infer; - - const form = useForm({ - resolver: zodResolver(formSchema), - defaultValues: { - mode: "inference", - alias: siteResource.alias ?? null - } - }); - - const [, formAction, saveLoading] = useActionState(async () => { - const isValid = await form.trigger(); - if (!isValid) return; - - const data = form.getValues(); - await save({ - mode: "inference", - alias: data.alias - }); - }, null); - - return ( - - - - - {t("hostSettings")} - - - {t("editInternalResourceDialogDestinationDescription")} - - - - - -
- - - - - - -
- - - - - - - - - +export default async function PrivateResourceInferencePage(props: { + params: Promise<{ niceId: string; orgId: string }>; +}) { + const params = await props.params; + redirect( + `/${params.orgId}/settings/resources/private/${params.niceId}/providers` ); } diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx index f07b92673..57064e545 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx @@ -55,20 +55,37 @@ export default async function PrivateResourceLayout( | "sshSettings" | "inferenceSettings"; - const navItems = [ - { - title: t("general"), - href: `/{orgId}/settings/resources/private/{niceId}/general` - }, - { - title: t(modeSettingsKey), - href: `/{orgId}/settings/resources/private/{niceId}/${siteResource.mode}` - }, - { - title: t("authentication"), - href: `/{orgId}/settings/resources/private/{niceId}/access` - } - ]; + const isInference = siteResource.mode === "inference"; + + const navItems = isInference + ? [ + { + title: t("general"), + href: `/{orgId}/settings/resources/private/{niceId}/general` + }, + { + title: t("aiResourceProviders"), + href: `/{orgId}/settings/resources/private/{niceId}/providers` + }, + { + title: t("authentication"), + href: `/{orgId}/settings/resources/private/{niceId}/access` + } + ] + : [ + { + title: t("general"), + href: `/{orgId}/settings/resources/private/{niceId}/general` + }, + { + title: t(modeSettingsKey), + href: `/{orgId}/settings/resources/private/{niceId}/${siteResource.mode}` + }, + { + title: t("authentication"), + href: `/{orgId}/settings/resources/private/{niceId}/access` + } + ]; return ( <> diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx new file mode 100644 index 000000000..ead7ab24b --- /dev/null +++ b/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx @@ -0,0 +1,232 @@ +"use client"; + +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { + AiProvidersSelector, + type SelectedAiProvider +} from "@app/components/AiProvidersSelector"; +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { useSiteResourceContext } from "@app/hooks/useSiteResourceContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { resourceQueries } from "@app/lib/queries"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useTranslations } from "next-intl"; +import { useRouter } from "next/navigation"; +import { useActionState, useEffect, useMemo, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; + +export default function PrivateResourceProvidersPage() { + const t = useTranslations(); + const router = useRouter(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const queryClient = useQueryClient(); + const { siteResource } = useSiteResourceContext(); + + useEffect(() => { + if (siteResource.mode !== "inference") { + router.replace( + `/${siteResource.orgId}/settings/resources/private/${siteResource.niceId}/general` + ); + } + }, [router, siteResource.mode, siteResource.niceId, siteResource.orgId]); + + const formSchema = useMemo( + () => + z.object({ + providerIds: z + .array(z.number().int().positive()) + .min(1, t("aiResourceProvidersRequired")) + }), + [t] + ); + type FormValues = z.infer; + + const [selectedProviders, setSelectedProviders] = useState< + SelectedAiProvider[] + >([]); + + const attachedQuery = useQuery({ + ...resourceQueries.siteResourceAiProviders({ + siteResourceId: siteResource.id + }), + enabled: siteResource.mode === "inference" + }); + + const form = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + providerIds: [] + } + }); + + useEffect(() => { + if (!attachedQuery.data) return; + const providers = attachedQuery.data.map((provider) => ({ + id: String(provider.providerId), + text: provider.name + })); + setSelectedProviders(providers); + form.reset({ + providerIds: attachedQuery.data.map((p) => p.providerId) + }); + }, [attachedQuery.data, form]); + + const [, formAction, saveLoading] = useActionState(async () => { + const isValid = await form.trigger(); + if (!isValid) return; + + const data = form.getValues(); + try { + await api.post(`/site-resource/${siteResource.id}/ai-providers`, { + providers: data.providerIds.map((providerId) => ({ + providerId, + modelAccessMode: "catalog" + })) + }); + + await queryClient.invalidateQueries( + resourceQueries.siteResourceAiProviders({ + siteResourceId: siteResource.id + }) + ); + + toast({ + title: t("success"), + description: t("aiResourceProvidersUpdated") + }); + } catch (error) { + toast({ + variant: "destructive", + title: t("aiResourceProvidersErrorUpdate"), + description: formatAxiosError( + error, + t("aiResourceProvidersErrorUpdate") + ) + }); + } + }, null); + + if (siteResource.mode !== "inference") { + return null; + } + + return ( + + + + + {t("aiResourceProviders")} + + + {t("aiResourceProvidersDescription")} + + + + + +
+ + + + ( + + + {t( + "aiResourceProviders" + )} + + + { + setSelectedProviders( + providers + ); + form.setValue( + "providerIds", + providers.map( + (p) => + parseInt( + p.id, + 10 + ) + ), + { + shouldValidate: true + } + ); + }} + /> + + + {t( + "aiResourceProvidersHelp" + )} + + + + )} + /> + + +
+ + + + + + + + + + ); +} diff --git a/src/app/[orgId]/settings/resources/private/create/page.tsx b/src/app/[orgId]/settings/resources/private/create/page.tsx index 936409d40..9420da17b 100644 --- a/src/app/[orgId]/settings/resources/private/create/page.tsx +++ b/src/app/[orgId]/settings/resources/private/create/page.tsx @@ -64,6 +64,10 @@ import { asAnySetValue, asAnyWatch } from "@app/lib/formControlUtils"; +import { + AiProvidersSelector, + type SelectedAiProvider +} from "@app/components/AiProvidersSelector"; export default function CreatePrivateResourcePage() { const params = useParams(); @@ -88,6 +92,9 @@ export default function CreatePrivateResourcePage() { : null; const [selectedSites, setSelectedSites] = useState([]); + const [selectedProviders, setSelectedProviders] = useState< + SelectedAiProvider[] + >([]); const formSchema = useMemo(() => createCreateFormSchema(t), [t]); type FormValues = z.infer; @@ -112,7 +119,8 @@ export default function CreatePrivateResourcePage() { pamMode: "passthrough", tcpPortRangeString: "*", udpPortRangeString: "*", - disableIcmp: false + disableIcmp: false, + providerIds: [] } }); @@ -196,7 +204,9 @@ export default function CreatePrivateResourcePage() { } router.push( - `/${orgId}/settings/resources/private/${created.niceId}/${created.mode}` + created.mode === "inference" + ? `/${orgId}/settings/resources/private/${created.niceId}/general` + : `/${orgId}/settings/resources/private/${created.niceId}/${created.mode}` ); } catch (error) { toast({ @@ -336,6 +346,13 @@ export default function CreatePrivateResourcePage() { "destinationPort", null ); + form.setValue( + "providerIds", + [] + ); + setSelectedProviders( + [] + ); } else { form.setValue( "destinationPort", @@ -637,6 +654,76 @@ export default function CreatePrivateResourcePage() { )} + {mode === "inference" && ( + + + + {t("aiResourceProviders")} + + + {t("aiResourceProvidersDescription")} + + + + + + + ( + + + {t( + "aiResourceProviders" + )} + + + { + setSelectedProviders( + providers + ); + form.setValue( + "providerIds", + providers.map( + ( + p + ) => + parseInt( + p.id, + 10 + ) + ), + { + shouldValidate: true + } + ); + }} + /> + + + {t( + "aiResourceProvidersHelp" + )} + + + + )} + /> + + + + + + )} +
)} - { !["tcp", "udp"].includes( - resource.mode - ) && !env.flags.disableEnterpriseFeatures && ( + {!["tcp", "udp", "inference"].includes( + resource.mode + ) && + !env.flags + .disableEnterpriseFeatures && ( <> diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx new file mode 100644 index 000000000..8ea8d2dd5 --- /dev/null +++ b/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx @@ -0,0 +1,15 @@ +import type { Metadata } from "next"; +import { redirect } from "next/navigation"; + +export const metadata: Metadata = { + title: "Public Resource" +}; + +export default async function PublicResourceInferencePage(props: { + params: Promise<{ niceId: string; orgId: string }>; +}) { + const params = await props.params; + redirect( + `/${params.orgId}/settings/resources/public/${params.niceId}/providers` + ); +} diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx index 1599cdceb..1c786be19 100644 --- a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx @@ -82,16 +82,29 @@ export default async function ResourceLayout(props: ResourceLayoutProps) { redirect(`/${params.orgId}/settings/resources`); } - const navItems = [ - { - title: t("general"), - href: `/{orgId}/settings/resources/public/{niceId}/general` - }, - { - title: t(`${resource.mode}Settings`), - href: `/{orgId}/settings/resources/public/{niceId}/${resource.mode}` - } - ]; + const isInference = resource.mode === "inference"; + + const navItems = isInference + ? [ + { + title: t("general"), + href: `/{orgId}/settings/resources/public/{niceId}/general` + }, + { + title: t("aiResourceProviders"), + href: `/{orgId}/settings/resources/public/{niceId}/providers` + } + ] + : [ + { + title: t("general"), + href: `/{orgId}/settings/resources/public/{niceId}/general` + }, + { + title: t(`${resource.mode}Settings`), + href: `/{orgId}/settings/resources/public/{niceId}/${resource.mode}` + } + ]; if (["http", "ssh", "rdp", "vnc"].includes(resource.mode)) { navItems.push( diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx new file mode 100644 index 000000000..67ef0fa9e --- /dev/null +++ b/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx @@ -0,0 +1,232 @@ +"use client"; + +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { + AiProvidersSelector, + type SelectedAiProvider +} from "@app/components/AiProvidersSelector"; +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { useResourceContext } from "@app/hooks/useResourceContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { resourceQueries } from "@app/lib/queries"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useTranslations } from "next-intl"; +import { useRouter } from "next/navigation"; +import { useActionState, useEffect, useMemo, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; + +export default function PublicResourceProvidersPage() { + const t = useTranslations(); + const router = useRouter(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const queryClient = useQueryClient(); + const { resource } = useResourceContext(); + + useEffect(() => { + if (resource.mode !== "inference") { + router.replace( + `/${resource.orgId}/settings/resources/public/${resource.niceId}/general` + ); + } + }, [router, resource.mode, resource.niceId, resource.orgId]); + + const formSchema = useMemo( + () => + z.object({ + providerIds: z + .array(z.number().int().positive()) + .min(1, t("aiResourceProvidersRequired")) + }), + [t] + ); + type FormValues = z.infer; + + const [selectedProviders, setSelectedProviders] = useState< + SelectedAiProvider[] + >([]); + + const attachedQuery = useQuery({ + ...resourceQueries.resourceAiProviders({ + resourceId: resource.resourceId + }), + enabled: resource.mode === "inference" + }); + + const form = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + providerIds: [] + } + }); + + useEffect(() => { + if (!attachedQuery.data) return; + const providers = attachedQuery.data.map((provider) => ({ + id: String(provider.providerId), + text: provider.name + })); + setSelectedProviders(providers); + form.reset({ + providerIds: attachedQuery.data.map((p) => p.providerId) + }); + }, [attachedQuery.data, form]); + + const [, formAction, saveLoading] = useActionState(async () => { + const isValid = await form.trigger(); + if (!isValid) return; + + const data = form.getValues(); + try { + await api.post(`/resource/${resource.resourceId}/ai-providers`, { + providers: data.providerIds.map((providerId) => ({ + providerId, + modelAccessMode: "catalog" + })) + }); + + await queryClient.invalidateQueries( + resourceQueries.resourceAiProviders({ + resourceId: resource.resourceId + }) + ); + + toast({ + title: t("success"), + description: t("aiResourceProvidersUpdated") + }); + } catch (error) { + toast({ + variant: "destructive", + title: t("aiResourceProvidersErrorUpdate"), + description: formatAxiosError( + error, + t("aiResourceProvidersErrorUpdate") + ) + }); + } + }, null); + + if (resource.mode !== "inference") { + return null; + } + + return ( + + + + + {t("aiResourceProviders")} + + + {t("aiResourceProvidersDescription")} + + + + + +
+ + + + ( + + + {t( + "aiResourceProviders" + )} + + + { + setSelectedProviders( + providers + ); + form.setValue( + "providerIds", + providers.map( + (p) => + parseInt( + p.id, + 10 + ) + ), + { + shouldValidate: true + } + ); + }} + /> + + + {t( + "aiResourceProvidersHelp" + )} + + + + )} + /> + + +
+ + + + + + + + + + ); +} diff --git a/src/app/[orgId]/settings/resources/public/create/page.tsx b/src/app/[orgId]/settings/resources/public/create/page.tsx index eaf4e5527..c0e0a0f8e 100644 --- a/src/app/[orgId]/settings/resources/public/create/page.tsx +++ b/src/app/[orgId]/settings/resources/public/create/page.tsx @@ -72,6 +72,10 @@ import { LocalTarget, ProxyResourceTargetsForm } from "@app/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm"; +import { + AiProvidersSelector, + type SelectedAiProvider +} from "@app/components/AiProvidersSelector"; import { AxiosResponse } from "axios"; import { ChevronsUpDown, ExternalLink } from "lucide-react"; import { useTranslations } from "next-intl"; @@ -245,6 +249,10 @@ export default function Page() { // Target management state (managed by ProxyResourceTargetsForm; mirrored here for onSubmit) const [targets, setTargets] = useState([]); + const [selectedProviders, setSelectedProviders] = useState< + SelectedAiProvider[] + >([]); + const [showProvidersError, setShowProvidersError] = useState(false); // SSH-specific state const [sshServerMode, setSshServerMode] = useState<"standard" | "native">( @@ -333,7 +341,7 @@ export default function Page() { !env.flags.disableEnterpriseFeatures; const availableTypes = useMemo((): NewResourceType[] => { - const base: NewResourceType[] = ["http"]; + const base: NewResourceType[] = ["http", "inference"]; if (enterpriseModesAllowed) { base.push("ssh", "rdp", "vnc"); } @@ -347,6 +355,9 @@ export default function Page() { if (!availableTypes.includes(resourceType)) { setResourceType("http"); } + if (resourceType !== "inference") { + setShowProvidersError(false); + } }, [availableTypes, resourceType]); const baseResourceFormSchema = useMemo( @@ -478,29 +489,42 @@ export default function Page() { ? finalizeSubdomainSanitize(httpData.subdomain, true) : undefined; - const effectiveMode = isNative - ? "native" - : standardDaemonLocation; - const portVal = sshDaemonPortForm.getValues().authDaemonPort; - const effectivePort = - !isNative && - standardDaemonLocation === "remote" && - pamMode === "push" && - portVal - ? Number(portVal) - : undefined; - Object.assign(payload, { subdomain: sanitizedSubdomain ? toASCII(sanitizedSubdomain) : undefined, domainId: httpData.domainId, protocol: "tcp", - mode: resourceType, - pamMode, - authDaemonMode: effectiveMode, - authDaemonPort: effectivePort || undefined + mode: resourceType }); + + if (resourceType === "inference") { + Object.assign(payload, { + aiProviders: selectedProviders.map((provider) => ({ + providerId: parseInt(provider.id, 10), + modelAccessMode: "catalog" + })) + }); + } else if (resourceType === "ssh") { + const effectiveMode = isNative + ? "native" + : standardDaemonLocation; + const portVal = + sshDaemonPortForm.getValues().authDaemonPort; + const effectivePort = + !isNative && + standardDaemonLocation === "remote" && + pamMode === "push" && + portVal + ? Number(portVal) + : undefined; + + Object.assign(payload, { + pamMode, + authDaemonMode: effectiveMode, + authDaemonPort: effectivePort || undefined + }); + } } else { const tcpUdpData = tcpUdpForm.getValues(); Object.assign(payload, { @@ -529,7 +553,11 @@ export default function Page() { const newNiceId = res.data.data.niceId; setNiceId(newNiceId); - if (resourceType === "http") { + if (resourceType === "inference") { + router.push( + `/${orgId}/settings/resources/public/${newNiceId}/general` + ); + } else if (resourceType === "http") { if (targets.length > 0) { try { for (const target of targets) { @@ -754,16 +782,17 @@ export default function Page() { let typeLabels: Partial> = { http: "HTTP", + inference: t("createInternalResourceDialogModeInference"), tcp: "TCP", udp: "UDP" }; if (enterpriseModesAllowed) { - typeLabels = { + typeLabels = { ...typeLabels, ssh: "SSH", rdp: "RDP", - vnc: "VNC", + vnc: "VNC" }; } @@ -1376,6 +1405,71 @@ export default function Page() { /> )} + {resourceType === "inference" && ( + + + + {t("aiResourceProviders")} + + + {t( + "aiResourceProvidersDescription" + )} + + + + + + +
+ + { + setSelectedProviders( + providers + ); + if ( + providers.length > + 0 + ) { + setShowProvidersError( + false + ); + } + }} + /> +

+ {t( + "aiResourceProvidersHelp" + )} +

+ {showProvidersError && ( +

+ {t( + "aiResourceProvidersRequired" + )} +

+ )} +
+ + + + + + )} +
diff --git a/src/components/PublicResourcesTable.tsx b/src/components/PublicResourcesTable.tsx index 09f451a69..48d7b4486 100644 --- a/src/components/PublicResourcesTable.tsx +++ b/src/components/PublicResourcesTable.tsx @@ -312,6 +312,12 @@ export default function PublicResourcesTable({ { value: "vnc", label: t("vncTitle") + }, + { + value: "inference", + label: t( + "createInternalResourceDialogModeInference" + ) } ]} selectedValue={ @@ -334,7 +340,11 @@ export default function PublicResourcesTable({ ? resourceRow.ssl ? "HTTPS" : "HTTP" - : resourceRow.mode?.toUpperCase()} + : resourceRow.mode === "inference" + ? t( + "createInternalResourceDialogModeInference" + ) + : resourceRow.mode?.toUpperCase()} ); } @@ -428,7 +438,7 @@ export default function PublicResourcesTable({ const resourceRow = row.original; if ( - !["http", "ssh", "rdp", "vnc"].includes( + !["http", "ssh", "rdp", "vnc", "inference"].includes( resourceRow.mode || "" ) ) { @@ -894,7 +904,9 @@ function ResourceEnabledForm({ resource, onToggleResourceEnabled }: ResourceEnabledFormProps) { - const enabled = ["http", "ssh", "rdp", "vnc"].includes(resource.mode || "") + const enabled = ["http", "ssh", "rdp", "vnc", "inference"].includes( + resource.mode || "" + ) ? !!resource.domainId && resource.enabled : resource.enabled; const [optimisticEnabled, setOptimisticEnabled] = useOptimistic(enabled); @@ -912,7 +924,7 @@ function ResourceEnabledForm({ */} - {["http", "ssh", "rdp", "vnc"].includes(resource.mode) ? ( + {isDomainResource ? ( <> URL @@ -94,33 +100,39 @@ export default function ResourceInfoBox({}: ResourceInfoBoxType) { ? resource.ssl ? "HTTPS" : "HTTP" - : resource.mode?.toUpperCase()} + : resource.mode === "inference" + ? t( + "createInternalResourceDialogModeInference" + ) + : resource.mode?.toUpperCase()} )} - - - {t("authentication")} - - - {authInfo.password || - authInfo.pincode || - authInfo.sso || - authInfo.whitelist || - authInfo.headerAuth ? ( -
- - {t("protected")} -
- ) : ( -
- - {t("notProtected")} -
- )} - - + {showAuth && ( + + + {t("authentication")} + + + {authInfo.password || + authInfo.pincode || + authInfo.sso || + authInfo.whitelist || + authInfo.headerAuth ? ( +
+ + {t("protected")} +
+ ) : ( +
+ + {t("notProtected")} +
+ )} + + + )} ) : ( <> @@ -138,7 +150,9 @@ export default function ResourceInfoBox({}: ResourceInfoBoxType) { {t("port")} diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 27cf1f7db..9a0cb2403 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -1300,6 +1300,26 @@ export const resourceQueries = { return res.data.data.providers; } }), + resourceAiProviders: ({ resourceId }: { resourceId: number }) => + queryOptions({ + queryKey: ["RESOURCES", resourceId, "AI_PROVIDERS"] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse<{ + providers: Array<{ + providerId: number; + modelAccessMode: "catalog" | "allowlist"; + name: string; + type: string; + enabled: boolean; + }>; + }> + >(`/resource/${resourceId}/ai-providers`, { + signal + }); + return res.data.data.providers; + } + }), resourceTargets: ({ resourceId }: { resourceId: number }) => queryOptions({ queryKey: ["RESOURCES", resourceId, "TARGETS"] as const, From 83e20c2dfa3679b26bd21cb4c3b57d18b949734d Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 4 Aug 2026 17:36:00 -0400 Subject: [PATCH 042/296] adjust placeholder ui for rules and targets --- .../public/ProxyResourceTargetsForm.tsx | 1 + .../PolicyAccessRulesTable.tsx | 1 + src/components/ui/data-table-empty-state.tsx | 30 ++++++++++++++----- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx b/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx index db3bec315..bc2f985e5 100644 --- a/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx +++ b/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx @@ -930,6 +930,7 @@ export const ProxyResourceTargetsForm = forwardRef< colSpan={columns.length} message={emptyMessage ?? t("targetNoOne")} action={addTargetButton} + compact /> )} diff --git a/src/components/resource-policy/PolicyAccessRulesTable.tsx b/src/components/resource-policy/PolicyAccessRulesTable.tsx index 2ff2e1915..e47395c4e 100644 --- a/src/components/resource-policy/PolicyAccessRulesTable.tsx +++ b/src/components/resource-policy/PolicyAccessRulesTable.tsx @@ -961,6 +961,7 @@ export function PolicyAccessRulesTable({ colSpan={columns.length} message={t("rulesNoOne")} action={emptyStateAction} + compact /> )} diff --git a/src/components/ui/data-table-empty-state.tsx b/src/components/ui/data-table-empty-state.tsx index e734f7726..b510029a6 100644 --- a/src/components/ui/data-table-empty-state.tsx +++ b/src/components/ui/data-table-empty-state.tsx @@ -5,34 +5,48 @@ import { useTranslations } from "next-intl"; import { type ReactNode } from "react"; const PLACEHOLDER_ROW_COUNT = 5; +const COMPACT_PLACEHOLDER_ROW_COUNT = 2; type DataTableEmptyStateProps = { colSpan: number; action?: ReactNode; message?: string; + compact?: boolean; }; export function DataTableEmptyState({ colSpan, action, - message + message, + compact = false }: DataTableEmptyStateProps) { const t = useTranslations(); + const placeholderRows = compact + ? COMPACT_PLACEHOLDER_ROW_COUNT + : PLACEHOLDER_ROW_COUNT; + const minHeightClass = compact ? "min-h-[4.5rem]" : "min-h-[11rem]"; + return ( -
+
- {Array.from({ length: PLACEHOLDER_ROW_COUNT }).map( - (_, i) => ( -
- ) - )} + {Array.from({ length: placeholderRows }).map((_, i) => ( +
+ ))}
-
+

{message ?? t("noResults")}

From 425a99e5eef1395831a6d1cf4887a3674806a98b Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 5 Aug 2026 09:59:41 -0400 Subject: [PATCH 043/296] Show the auth settings for testing --- src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx index 1c786be19..302650bf2 100644 --- a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx @@ -106,7 +106,7 @@ export default async function ResourceLayout(props: ResourceLayoutProps) { } ]; - if (["http", "ssh", "rdp", "vnc"].includes(resource.mode)) { + if (["http", "ssh", "rdp", "vnc", "inference"].includes(resource.mode)) { navItems.push( { title: t("authentication"), From 7afddb5eb58e831b4f399650d2d07f3f504d1519 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 5 Aug 2026 10:00:36 -0400 Subject: [PATCH 044/296] Clean logging --- server/routers/aiGateway/chatCompletions.ts | 2 +- server/routers/olm/handleOlmRegisterMessage.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index 41fd1f4e3..f5de5cc75 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -427,7 +427,7 @@ export async function chatCompletions( .json({ error: { message: "Missing Host header" } }); } - logger.info(`++++++++++++++AI gateway request for host: ${host}`); + logger.info(`AI gateway request for host: ${host}`); const target = await resolveTarget(host); if (!target) { diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 5cd38dabe..c5cc7cc7c 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -489,9 +489,9 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { jitMode ); - logger.info( - `+++++++++++++++++++++++++++++++ ExitNode Aliases: ${exitNodeAliases}` - ); + // logger.info( + // `ExitNode Aliases: ${exitNodeAliases}` + // ); // Return connect message with all site configurations return { From 3d062389e9c79a479a5862b62cc2d1f2a4938712 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 5 Aug 2026 10:14:40 -0400 Subject: [PATCH 045/296] update oss with gateway traefik routers --- server/lib/traefik/getTraefikConfig.ts | 177 ++++++++++++++++++++++--- 1 file changed, 162 insertions(+), 15 deletions(-) diff --git a/server/lib/traefik/getTraefikConfig.ts b/server/lib/traefik/getTraefikConfig.ts index ee972ea68..f53c501ea 100644 --- a/server/lib/traefik/getTraefikConfig.ts +++ b/server/lib/traefik/getTraefikConfig.ts @@ -1,4 +1,11 @@ -import { db, targetHealthCheck, domains, aiProviders, resourceAiProviders } from "@server/db"; +import { + db, + targetHealthCheck, + domains, + aiProviders, + resourceAiProviders, + siteResources +} from "@server/db"; import { and, eq, @@ -222,24 +229,25 @@ export async function getTraefikConfig( subdomain: resources.subdomain, domainId: resources.domainId, enabled: resources.enabled, + wildcard: resources.wildcard, domainCertResolver: domains.certResolver, preferWildcardCert: domains.preferWildcardCert }) .from(resources) - .innerJoin( - resourceAiProviders, - eq(resources.resourceId, resourceAiProviders.resourceId) - ) - .innerJoin( - aiProviders, - eq(resourceAiProviders.providerId, aiProviders.providerId) - ) + // .innerJoin( + // resourceAiProviders, + // eq(resources.resourceId, resourceAiProviders.resourceId) + // ) + // .innerJoin( + // aiProviders, + // eq(resourceAiProviders.providerId, aiProviders.providerId) + // ) .leftJoin(domains, eq(domains.domainId, resources.domainId)) .where( and( eq(resources.mode, "inference"), - eq(resources.enabled, true), - eq(aiProviders.enabled, true) + eq(resources.enabled, true) + // eq(aiProviders.enabled, true) ) ); @@ -709,6 +717,23 @@ export async function getTraefikConfig( } if (aiGatewayUrl) { + // The AI gateway may live on a different host than the inference + // resource itself (e.g. a remote exit node forwarding to the + // central dashboard over a tunnel). passHostHeader would forward + // the resource's own Host, which that external host won't + // recognize, so we pin the Host header to the gateway's own host + // and smuggle the original resource host through in "p-host" + // instead. + let aiGatewayHost: string | undefined; + try { + aiGatewayHost = new URL(aiGatewayUrl).host; + } catch { + aiGatewayHost = undefined; + } + + // Public inference resources: same TLS/cert-resolver handling as + // plain http-mode resources, but the service points at the AI + // gateway instead of any real backend targets. for (const ir of inferenceResources) { if (!ir.enabled) continue; if (!ir.domainId || !ir.fullDomain) continue; @@ -720,7 +745,14 @@ export async function getTraefikConfig( const irKey = `inference-r${ir.resourceId}`; const routerName = `${irKey}-router`; const serviceName = `${irKey}-service`; - const rule = `Host(\`${fullDomain}\`)`; + + let rule: string; + if (ir.wildcard && fullDomain.startsWith("*.")) { + const escaped = fullDomain.slice(2).replace(/\./g, "\\."); + rule = `HostRegexp(\`^[^.]+\\.${escaped}$\`)`; + } else { + rule = `Host(\`${fullDomain}\`)`; + } const domainParts = fullDomain.split("."); let wildCard; @@ -751,13 +783,39 @@ export async function getTraefikConfig( ...(preferWildcard ? { domains: [{ main: wildCard }] } : {}) }; + const irHeadersMiddlewareName = `${irKey}-headers-middleware`; + if (!config_output.http.middlewares) { + config_output.http.middlewares = {}; + } + config_output.http.middlewares[irHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), + "p-host": fullDomain + } + } + }; + const additionalMiddlewares = config.getRawConfig().traefik.additional_middlewares || []; const routerMiddlewares = [ badgerMiddlewareName, + irHeadersMiddlewareName, ...additionalMiddlewares ]; + if (ir.ssl) { + config_output.http.routers[routerName + "-redirect"] = { + entryPoints: [ + config.getRawConfig().traefik.http_entrypoint + ], + middlewares: [redirectHttpsMiddlewareName], + service: serviceName, + rule, + priority: 100 + }; + } + config_output.http.routers[routerName] = { entryPoints: [ ir.ssl @@ -771,7 +829,84 @@ export async function getTraefikConfig( ...(ir.ssl ? { tls } : {}) }; - if (ir.ssl) { + config_output.http.services[serviceName] = { + loadBalancer: { + servers: [{ url: aiGatewayUrl }] + } + }; + } + + // Private (siteResource) inference resources: routed by their alias + // instead of a public fullDomain, and deliberately WITHOUT the + // badger middleware - no per-user auth/policy stack exists for + // siteResources today, so gating here is reachability-only for now. + const siteResourcesInference = await db + .selectDistinct({ + siteResourceId: siteResources.siteResourceId, + alias: siteResources.alias, + ssl: siteResources.ssl, + enabled: siteResources.enabled + }) + .from(siteResources) + .where( + and( + eq(siteResources.mode, "inference"), + eq(siteResources.enabled, true), + isNotNull(siteResources.alias) + ) + ); + + for (const sr of siteResourcesInference) { + if (!sr.enabled || !sr.alias) continue; + + if (!config_output.http.routers) config_output.http.routers = {}; + if (!config_output.http.services) config_output.http.services = {}; + + const alias = sr.alias; + const srKey = `inference-sr${sr.siteResourceId}`; + const routerName = `${srKey}-router`; + const serviceName = `${srKey}-service`; + const rule = `Host(\`${alias}\`)`; + + const domainParts = alias.split("."); + const wildCard = + domainParts.length <= 2 + ? `*.${domainParts.join(".")}` + : `*.${domainParts.slice(1).join(".")}`; + + const globalDefaultResolver = + config.getRawConfig().traefik.cert_resolver; + const globalDefaultPreferWildcard = + config.getRawConfig().traefik.prefer_wildcard_cert; + + const tls = { + certResolver: globalDefaultResolver, + ...(globalDefaultPreferWildcard + ? { domains: [{ main: wildCard }] } + : {}) + }; + + const srHeadersMiddlewareName = `${srKey}-headers-middleware`; + if (!config_output.http.middlewares) { + config_output.http.middlewares = {}; + } + config_output.http.middlewares[srHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), + "p-host": alias + } + } + }; + + const additionalMiddlewares = + config.getRawConfig().traefik.additional_middlewares || []; + const routerMiddlewares = [ + srHeadersMiddlewareName, + ...additionalMiddlewares + ]; + + if (sr.ssl) { config_output.http.routers[routerName + "-redirect"] = { entryPoints: [ config.getRawConfig().traefik.http_entrypoint @@ -783,10 +918,22 @@ export async function getTraefikConfig( }; } + config_output.http.routers[routerName] = { + entryPoints: [ + sr.ssl + ? config.getRawConfig().traefik.https_entrypoint + : config.getRawConfig().traefik.http_entrypoint + ], + middlewares: routerMiddlewares, + service: serviceName, + rule, + priority: 100, + ...(sr.ssl ? { tls } : {}) + }; + config_output.http.services[serviceName] = { loadBalancer: { - servers: [{ url: aiGatewayUrl }], - passHostHeader: true + servers: [{ url: aiGatewayUrl }] } }; } From bc80f91a45e9d9e1c8fd1166cc91349e31da9263 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 5 Aug 2026 10:24:26 -0400 Subject: [PATCH 046/296] gateway endpoint mvp --- server/routers/aiGateway/chatCompletions.ts | 251 ++++++++++++-------- 1 file changed, 157 insertions(+), 94 deletions(-) diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index f5de5cc75..2f1bfc451 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -1,5 +1,5 @@ import { Request, Response } from "express"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { AiProvider, aiModels, @@ -94,10 +94,12 @@ type ProviderAttachment = { type ResolvedTarget = { resourceId: number | null; + siteResourceId: number | null; orgId: string | null; attachments: ProviderAttachment[]; - // model IDs on this resource's allowlist (resource-wide) - allowedModelIds: number[]; + // Model IDs on the resource allowlist that belong to allowlist-mode + // providers. Empty when no attached provider uses allowlist mode. + allowlistedModelIds: Set; }; type ProviderSelection = @@ -192,6 +194,9 @@ async function resolveRequestUser( } async function resolveTarget(host: string): Promise { + // TODO: eventually we need to know if it's a private or public resource + // and not just simply check the fullDomain in case there is a private resource with the same fullDomain + const [resourceRow] = await db .select({ resourceId: resources.resourceId, @@ -229,26 +234,40 @@ async function resolveTarget(host: string): Promise { return null; } - const hasAllowlist = attachmentRows.some( - (a) => a.modelAccessMode === "allowlist" - ); - let allowedModelIds: number[] = []; - if (hasAllowlist) { + const attachments: ProviderAttachment[] = attachmentRows.map((a) => ({ + provider: a.provider, + modelAccessMode: a.modelAccessMode as ModelAccessMode + })); + + const allowlistProviderIds = attachments + .filter((a) => a.modelAccessMode === "allowlist") + .map((a) => a.provider.providerId); + const allowlistedModelIds = new Set(); + if (allowlistProviderIds.length > 0) { const restrictions = await db .select({ modelId: resourceAiModels.modelId }) .from(resourceAiModels) - .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)); - allowedModelIds = restrictions.map((r) => r.modelId); + .innerJoin( + aiModels, + eq(resourceAiModels.modelId, aiModels.modelId) + ) + .where( + and( + eq(resourceAiModels.resourceId, resourceRow.resourceId), + inArray(aiModels.providerId, allowlistProviderIds) + ) + ); + for (const row of restrictions) { + allowlistedModelIds.add(row.modelId); + } } return { resourceId: resourceRow.resourceId, + siteResourceId: null, orgId: resourceRow.orgId, - attachments: attachmentRows.map((a) => ({ - provider: a.provider, - modelAccessMode: a.modelAccessMode as ModelAccessMode - })), - allowedModelIds + attachments, + allowlistedModelIds }; } @@ -292,74 +311,52 @@ async function resolveTarget(host: string): Promise { return null; } - const hasAllowlist = attachmentRows.some( - (a) => a.modelAccessMode === "allowlist" - ); - let allowedModelIds: number[] = []; - if (hasAllowlist) { + const attachments: ProviderAttachment[] = attachmentRows.map((a) => ({ + provider: a.provider, + modelAccessMode: a.modelAccessMode as ModelAccessMode + })); + + const allowlistProviderIds = attachments + .filter((a) => a.modelAccessMode === "allowlist") + .map((a) => a.provider.providerId); + const allowlistedModelIds = new Set(); + if (allowlistProviderIds.length > 0) { const restrictions = await db .select({ modelId: siteResourceAiModels.modelId }) .from(siteResourceAiModels) + .innerJoin( + aiModels, + eq(siteResourceAiModels.modelId, aiModels.modelId) + ) .where( - eq( - siteResourceAiModels.siteResourceId, - siteResourceRow.siteResourceId + and( + eq( + siteResourceAiModels.siteResourceId, + siteResourceRow.siteResourceId + ), + inArray(aiModels.providerId, allowlistProviderIds) ) ); - allowedModelIds = restrictions.map((r) => r.modelId); + for (const row of restrictions) { + allowlistedModelIds.add(row.modelId); + } } return { - // siteResources have no per-user auth/policy stack today (see - // the routing comment in getTraefikConfig.ts), so there's no - // resource access token scope to validate a user token against. resourceId: null, + siteResourceId: siteResourceRow.siteResourceId, orgId: siteResourceRow.orgId, - attachments: attachmentRows.map((a) => ({ - provider: a.provider, - modelAccessMode: a.modelAccessMode as ModelAccessMode - })), - allowedModelIds + attachments, + allowlistedModelIds }; } return null; } -async function providerMatchesModel( - attachment: ProviderAttachment, - requestedModel: string, - allowedModelIds: number[] -): Promise { - const [matchedModel] = await db - .select({ - modelId: aiModels.modelId, - enabled: aiModels.enabled - }) - .from(aiModels) - .where( - and( - eq(aiModels.providerId, attachment.provider.providerId), - eq(aiModels.modelKey, requestedModel) - ) - ) - .limit(1); - - if (!matchedModel) { - return false; - } - - if (attachment.modelAccessMode === "catalog") { - return matchedModel.enabled; - } - - // allowlist - return allowedModelIds.includes(matchedModel.modelId); -} - async function selectProvider( attachments: ProviderAttachment[], - allowedModelIds: number[], + allowlistedModelIds: Set, requestedModel: string | undefined ): Promise { if (!requestedModel) { @@ -370,21 +367,55 @@ async function selectProvider( }; } - const candidates: ProviderAttachment[] = []; - for (const attachment of attachments) { - if ( - await providerMatchesModel( - attachment, - requestedModel, - allowedModelIds + const providerById = new Map( + attachments.map((a) => [a.provider.providerId, a]) + ); + const providerIds = [...providerById.keys()]; + if (providerIds.length === 0) { + return { + ok: false, + status: HttpCode.FORBIDDEN, + message: `Model "${requestedModel}" is not permitted on this resource` + }; + } + + // One lookup for the requested model key across all attached providers. + const matchingModels = await db + .select({ + modelId: aiModels.modelId, + providerId: aiModels.providerId, + enabled: aiModels.enabled + }) + .from(aiModels) + .where( + and( + inArray(aiModels.providerId, providerIds), + eq(aiModels.modelKey, requestedModel) ) - ) { - candidates.push(attachment); + ); + + const candidates: AiProvider[] = []; + for (const model of matchingModels) { + const attachment = providerById.get(model.providerId); + if (!attachment) { + continue; + } + + if (attachment.modelAccessMode === "catalog") { + if (model.enabled) { + candidates.push(attachment.provider); + } + continue; + } + + // allowlist: only models explicitly attached to the resource + if (allowlistedModelIds.has(model.modelId)) { + candidates.push(attachment.provider); } } if (candidates.length === 1) { - return { ok: true, provider: candidates[0].provider }; + return { ok: true, provider: candidates[0] }; } if (candidates.length > 1) { @@ -402,15 +433,6 @@ async function selectProvider( }; } -// Generic OpenAI-wire-compatible passthrough. Anthropic's native API uses a -// different path/schema; everything else here is OpenAI-compatible today. -function getCompletionsPath(type: AiProviderType): string { - if (type === "anthropic") { - return "/v1/messages"; - } - return "/chat/completions"; -} - export async function chatCompletions( req: Request, res: Response @@ -438,7 +460,9 @@ export async function chatCompletions( }); } - const { attachments, allowedModelIds, resourceId, orgId } = target; + const { attachments, allowlistedModelIds, resourceId, orgId } = target; + + logger.debug("+++++ gateway target: ", target); // Best-effort identity resolution - not yet enforced, but lets us // start making per-user access decisions (e.g. model/role-based @@ -455,7 +479,7 @@ export async function chatCompletions( const selection = await selectProvider( attachments, - allowedModelIds, + allowlistedModelIds, requestedModel ); if (!selection.ok) { @@ -490,16 +514,35 @@ export async function chatCompletions( }); } - const targetUrl = `${upstreamUrl.replace(/\/$/, "")}${getCompletionsPath( - provider.type as AiProviderType - )}`; + const targetUrl = `${upstreamUrl.replace(/\/$/, "")}`; - const headers: Record = { - "Content-Type": "application/json" - }; - if (authType === "bearer") { - headers["Authorization"] = `Bearer ${apiKey}`; + // Drop hop-by-hop / proxy-only headers. Forwarding Host especially + // breaks Node fetch (TLS/SNI targets the upstream URL while Host + // still says localhost). + const skipHeaders = new Set([ + "p-host", + "host", + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + "content-length", + "accept-encoding" + ]); + + const headers: Record = {}; + for (const [key, value] of Object.entries(req.headers)) { + if (skipHeaders.has(key.toLowerCase()) || value === undefined) { + continue; + } + headers[key] = Array.isArray(value) ? value.join(", ") : value; } + // TODO: temporary hardcoded auth for testing; restore bearer from authType + headers["x-api-key"] = apiKey; // No dedicated per-request TLS agent is wired up (no extra deps for // this v1 gateway) - toggle the process-wide Node TLS check instead. @@ -510,13 +553,33 @@ export async function chatCompletions( process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; } + const body = JSON.stringify(req.body); + + logger.debug("AI gateway upstream request", { + url: targetUrl, + method: "POST", + headers, + body: req.body + }); + let upstreamRes: globalThis.Response; try { upstreamRes = await fetch(targetUrl, { method: "POST", headers, - body: JSON.stringify(req.body) + body }); + } catch (fetchError) { + logger.error({ + message: "AI gateway upstream fetch failed", + url: targetUrl, + error: fetchError, + cause: + fetchError instanceof Error + ? (fetchError as Error & { cause?: unknown }).cause + : undefined + }); + throw fetchError; } finally { if (provider.skipTlsVerification) { if (restoreTlsReject === undefined) { From 346a0bdc98eb0207d16698e3a7ce8c4a8047339d Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 5 Aug 2026 10:31:14 -0400 Subject: [PATCH 047/296] Resolve syntax issue --- server/routers/aiProvider/listAiProviders.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/routers/aiProvider/listAiProviders.ts b/server/routers/aiProvider/listAiProviders.ts index 7ec89df51..eb4477bc2 100644 --- a/server/routers/aiProvider/listAiProviders.ts +++ b/server/routers/aiProvider/listAiProviders.ts @@ -131,7 +131,7 @@ export async function listAiProviders( return response(res, { data: { - providers: rows.map(toPublicAiProvider), + providers: rows.map((row) => toPublicAiProvider(row)), pagination: { total: totalCount, pageSize, From e1dd8965dc9f3f8aa1fe617680b2ae72df627bef Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 5 Aug 2026 11:31:12 -0400 Subject: [PATCH 048/296] Remove extra checks --- server/db/sqlite/schema/schema.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 2ae941895..8af639238 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1693,15 +1693,6 @@ export const aiBudgets = sqliteTable( updatedAt: integer("updatedAt").notNull() }, (t) => [ - check( - "ai_budget_one_scope", - sql`( - (CASE WHEN ${t.providerId} IS NOT NULL THEN 1 ELSE 0 END) + - (CASE WHEN ${t.modelId} IS NOT NULL THEN 1 ELSE 0 END) + - (CASE WHEN ${t.resourceId} IS NOT NULL THEN 1 ELSE 0 END) + - (CASE WHEN ${t.siteResourceId} IS NOT NULL THEN 1 ELSE 0 END) - ) = 1` - ), unique("ai_budget_provider_uniq").on(t.providerId), unique("ai_budget_model_uniq").on(t.modelId), unique("ai_budget_resource_uniq").on(t.resourceId), From c673dce4844b414d332734cf109530c05863ecf5 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 5 Aug 2026 13:34:45 -0400 Subject: [PATCH 049/296] pin docker base image --- Dockerfile | 8 ++++---- Dockerfile.dev | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9af37f89c..ecd2daa28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# FROM node:24-slim AS base -FROM public.ecr.aws/docker/library/node:24-slim AS base +# FROM node:24.18.1-slim AS base +FROM public.ecr.aws/docker/library/node:24.18.1-slim AS base WORKDIR /app @@ -32,8 +32,8 @@ FROM base AS builder RUN npm ci --omit=dev -# FROM node:24-slim AS runner -FROM public.ecr.aws/docker/library/node:24-slim AS runner +# FROM node:24.18.1-slim AS runner +FROM public.ecr.aws/docker/library/node:24.18.1-slim AS runner WORKDIR /app diff --git a/Dockerfile.dev b/Dockerfile.dev index 3e5965fc1..1372fe33a 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM node:24-alpine +FROM node:24.18.1-alpine WORKDIR /app From 2e9bd501721d8d4fb7a6cc62ad6cae6a38ad2455 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 5 Aug 2026 15:17:50 -0400 Subject: [PATCH 050/296] add provider specific auth modes --- messages/en-US.json | 6 +- server/db/pg/schema/schema.ts | 10 ++- server/db/sqlite/schema/schema.ts | 10 ++- server/lib/aiProviderDefaults.ts | 74 ++++++++++++++++--- server/routers/aiGateway/chatCompletions.ts | 15 +--- server/routers/aiProvider/createAiProvider.ts | 20 ++--- server/routers/aiProvider/types.ts | 19 +---- server/routers/aiProvider/updateAiProvider.ts | 17 ++--- server/routers/aiProvider/validation.ts | 6 +- .../[providerId]/authentication/page.tsx | 29 +++++++- .../[providerId]/network/page.tsx | 9 ++- .../settings/ai-providers/create/page.tsx | 20 +++++ src/lib/aiProviderFormSchema.ts | 21 +++--- 13 files changed, 178 insertions(+), 78 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index fa9202b2f..257c56c42 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1652,7 +1652,7 @@ "aiProviderNetworkSettings": "Network Settings", "aiProviderNetworkSettingsDescription": "Choose how traffic reaches this provider", "aiProviderAuthSettings": "Authentication", - "aiProviderAuthSettingsDescription": "Credentials used for both upstream URL and Pangolin target routing", + "aiProviderAuthSettingsDescription": "Configure how this provider authenticates requests to its upstream URL", "aiProviderType": "Provider Type", "aiProviderTypeSearch": "Search providers...", "aiProviderTypeNotFound": "No provider type found", @@ -1683,6 +1683,10 @@ "aiProviderApiKeyLastChars": "API Key", "aiProviderAuthType": "Auth Type", "aiProviderAuthTypeBearer": "Bearer", + "aiProviderAuthTypeXApiKey": "x-api-key", + "aiProviderAuthTypeXGoogApiKey": "x-goog-api-key", + "aiProviderAuthTypeHec": "Splunk HEC", + "aiProviderAuthTypeCfAigAuthorization": "Cloudflare AI Gateway", "aiProviderAuthTypeDescription": "How the upstream API authenticates requests", "aiProviderRoutingMode": "Routing Mode", "aiProviderRoutingModeDescription": "Send traffic to an upstream URL or to HTTP targets on your sites", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 868b2a121..7883345da 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1644,7 +1644,15 @@ export const aiProviders = pgTable("aiProviders", { upstreamUrl: text("upstreamUrl"), apiKey: text("apiKey"), apiKeyLastChars: varchar("apiKeyLastChars"), - authType: varchar("authType").$type<"bearer">(), + authType: varchar("authType") + .$type< + | "bearer" + | "x-api-key" + | "x-goog-api-key" + | "hec" + | "cf-aig-authorization" + >() + .notNull(), routingMode: varchar("routingMode") .$type<"url" | "target">() .notNull() diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 8af639238..e586c4e02 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1626,7 +1626,15 @@ export const aiProviders = sqliteTable("aiProviders", { upstreamUrl: text("upstreamUrl"), apiKey: text("apiKey"), apiKeyLastChars: text("apiKeyLastChars"), - authType: text("authType").$type<"bearer">(), + authType: text("authType") + .$type< + | "bearer" + | "x-api-key" + | "x-goog-api-key" + | "hec" + | "cf-aig-authorization" + >() + .notNull(), routingMode: text("routingMode") .$type<"url" | "target">() .notNull() diff --git a/server/lib/aiProviderDefaults.ts b/server/lib/aiProviderDefaults.ts index 6cfb816b7..d803e7613 100644 --- a/server/lib/aiProviderDefaults.ts +++ b/server/lib/aiProviderDefaults.ts @@ -9,7 +9,15 @@ export type AiProviderType = | "vercelAiGateway" | "custom"; -export type AiProviderAuthType = "bearer"; +export const AI_PROVIDER_AUTH_TYPES = [ + "bearer", + "x-api-key", + "x-goog-api-key", + "hec", + "cf-aig-authorization" +] as const; + +export type AiProviderAuthType = (typeof AI_PROVIDER_AUTH_TYPES)[number]; export type AiBudgetUnit = "usd" | "tokens"; export type AiProviderRoutingMode = "url" | "target"; @@ -28,11 +36,11 @@ export const AI_PROVIDER_DEFAULTS: Record< }, anthropic: { upstreamUrl: "https://api.anthropic.com", - authType: "bearer" + authType: "x-api-key" }, googleGemini: { upstreamUrl: "https://generativelanguage.googleapis.com/v1beta/openai/", - authType: "bearer" + authType: "x-goog-api-key" }, vertexAi: { upstreamUrl: null, @@ -56,6 +64,13 @@ export const AI_PROVIDER_DEFAULTS: Record< } }; +const CONFLICTING_AUTH_HEADERS = [ + "authorization", + "x-api-key", + "x-goog-api-key", + "cf-aig-authorization" +] as const; + export function providerRequiresUpstreamUrl( type: AiProviderType, routingMode: AiProviderRoutingMode = "url" @@ -69,17 +84,18 @@ export function providerRequiresUpstreamUrl( return AI_PROVIDER_DEFAULTS[type].upstreamUrl === null; } -export function resolveAiProviderConfig(input: { +export function resolveAiProviderCreateFields(input: { type: AiProviderType; - upstreamUrl: string | null; - authType: AiProviderAuthType | null; + upstreamUrl?: string | null; + authType?: AiProviderAuthType | null; routingMode?: AiProviderRoutingMode | null; }): { upstreamUrl: string | null; - authType: AiProviderAuthType | null; + authType: AiProviderAuthType; routingMode: AiProviderRoutingMode; } { - const routingMode = input.routingMode ?? "url"; + const routingMode = + input.type === "custom" ? (input.routingMode ?? "url") : "url"; if (routingMode === "target") { return { @@ -91,8 +107,8 @@ export function resolveAiProviderConfig(input: { if (input.type === "custom") { return { - upstreamUrl: input.upstreamUrl, - authType: input.authType, + upstreamUrl: input.upstreamUrl ?? null, + authType: input.authType ?? "bearer", routingMode }; } @@ -100,7 +116,43 @@ export function resolveAiProviderConfig(input: { const defaults = AI_PROVIDER_DEFAULTS[input.type]; return { upstreamUrl: input.upstreamUrl ?? defaults.upstreamUrl, - authType: input.authType ?? defaults.authType, + authType: defaults.authType, routingMode }; } + +/** + * Strip inbound client auth headers, then set the provider auth header + * for the given authType. + */ +export function applyAiProviderAuthHeaders( + headers: Record, + authType: AiProviderAuthType, + apiKey: string +): void { + for (const name of CONFLICTING_AUTH_HEADERS) { + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === name) { + delete headers[key]; + } + } + } + + switch (authType) { + case "bearer": + headers["Authorization"] = `Bearer ${apiKey}`; + break; + case "x-api-key": + headers["x-api-key"] = apiKey; + break; + case "x-goog-api-key": + headers["x-goog-api-key"] = apiKey; + break; + case "hec": + headers["Authorization"] = `Splunk ${apiKey}`; + break; + case "cf-aig-authorization": + headers["cf-aig-authorization"] = `Bearer ${apiKey}`; + break; + } +} diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index 2f1bfc451..e1bb8bbc9 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -19,9 +19,7 @@ import config from "@server/lib/config"; import { decrypt } from "@server/lib/crypto"; import { AiProviderAuthType, - AiProviderRoutingMode, - AiProviderType, - resolveAiProviderConfig + applyAiProviderAuthHeaders } from "@server/lib/aiProviderDefaults"; import { SESSION_COOKIE_NAME, @@ -499,12 +497,8 @@ export async function chatCompletions( const secret = config.getRawConfig().server.secret!; const apiKey = decrypt(provider.apiKey, secret); - const { upstreamUrl, authType } = resolveAiProviderConfig({ - type: provider.type as AiProviderType, - upstreamUrl: provider.upstreamUrl, - authType: provider.authType as AiProviderAuthType | null, - routingMode: provider.routingMode as AiProviderRoutingMode | null - }); + const upstreamUrl = provider.upstreamUrl; + const authType = provider.authType as AiProviderAuthType; if (!upstreamUrl) { return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ @@ -541,8 +535,7 @@ export async function chatCompletions( } headers[key] = Array.isArray(value) ? value.join(", ") : value; } - // TODO: temporary hardcoded auth for testing; restore bearer from authType - headers["x-api-key"] = apiKey; + applyAiProviderAuthHeaders(headers, authType, apiKey); // No dedicated per-request TLS agent is wired up (no extra deps for // this v1 gateway) - toggle the process-wide Node TLS check instead. diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index 6fb3f395a..6188601a3 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -9,6 +9,7 @@ import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { encrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; +import { resolveAiProviderCreateFields } from "@server/lib/aiProviderDefaults"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import { toPublicAiProvider } from "@server/routers/aiProvider/types"; import { @@ -28,7 +29,7 @@ const bodySchema = z type: aiProviderTypeSchema, upstreamUrl: z.url().optional().nullable(), apiKey: z.string().optional(), - authType: aiAuthTypeSchema.optional().nullable(), + authType: aiAuthTypeSchema.optional(), routingMode: aiRoutingModeSchema.optional(), skipTlsVerification: z.boolean().optional(), enabled: z.boolean().optional() @@ -101,8 +102,12 @@ export async function createAiProvider( const encryptedApiKey = apiKey ? encrypt(apiKey, key) : null; const apiKeyLastChars = apiKey ? apiKey.slice(-4) : null; const now = Date.now(); - const resolvedRoutingMode = - type === "custom" ? (routingMode ?? "url") : "url"; + const resolved = resolveAiProviderCreateFields({ + type, + upstreamUrl, + authType, + routingMode + }); const [provider] = await db .insert(aiProviders) @@ -110,14 +115,11 @@ export async function createAiProvider( orgId, name, type, - upstreamUrl: - resolvedRoutingMode === "target" - ? null - : (upstreamUrl ?? null), + upstreamUrl: resolved.upstreamUrl, apiKey: encryptedApiKey, apiKeyLastChars, - authType: authType ?? null, - routingMode: resolvedRoutingMode, + authType: resolved.authType, + routingMode: resolved.routingMode, skipTlsVerification: skipTlsVerification ?? false, enabled: enabled ?? true, createdAt: now, diff --git a/server/routers/aiProvider/types.ts b/server/routers/aiProvider/types.ts index 3fa966785..7ac492056 100644 --- a/server/routers/aiProvider/types.ts +++ b/server/routers/aiProvider/types.ts @@ -1,11 +1,6 @@ import type { AiModel, AiProvider } from "@server/db"; import type { PaginatedResponse } from "@server/types/Pagination"; -import { - resolveAiProviderConfig, - type AiProviderAuthType, - type AiProviderRoutingMode, - type AiProviderType -} from "@server/lib/aiProviderDefaults"; +import type { AiProviderAuthType } from "@server/lib/aiProviderDefaults"; import { decrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; @@ -13,7 +8,7 @@ export type AiProviderPublic = Omit & { /** Decrypted API key. Only included on get/create/update of a single provider. */ apiKey?: string | null; effectiveUpstreamUrl: string | null; - effectiveAuthType: AiProviderAuthType | null; + effectiveAuthType: AiProviderAuthType; }; export type ListAiProvidersResponse = PaginatedResponse<{ @@ -45,12 +40,6 @@ export function toPublicAiProvider( options?: { includeApiKey?: boolean } ): AiProviderPublic { const { apiKey: encryptedApiKey, ...rest } = provider; - const resolved = resolveAiProviderConfig({ - type: provider.type as AiProviderType, - upstreamUrl: provider.upstreamUrl, - authType: provider.authType as AiProviderAuthType | null, - routingMode: provider.routingMode as AiProviderRoutingMode | null - }); let apiKey: string | null | undefined; if (options?.includeApiKey) { @@ -67,7 +56,7 @@ export function toPublicAiProvider( return { ...rest, ...(options?.includeApiKey ? { apiKey } : {}), - effectiveUpstreamUrl: resolved.upstreamUrl, - effectiveAuthType: resolved.authType + effectiveUpstreamUrl: provider.upstreamUrl, + effectiveAuthType: provider.authType as AiProviderAuthType }; } diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts index 35e27fba2..82224c28b 100644 --- a/server/routers/aiProvider/updateAiProvider.ts +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -19,6 +19,7 @@ import { refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; import type { + AiProviderAuthType, AiProviderRoutingMode, AiProviderType } from "@server/lib/aiProviderDefaults"; @@ -31,7 +32,7 @@ const bodySchema = z.strictObject({ name: z.string().nonempty().optional(), upstreamUrl: z.url().optional().nullable(), apiKey: z.string().optional(), - authType: aiAuthTypeSchema.optional().nullable(), + authType: aiAuthTypeSchema.optional(), routingMode: aiRoutingModeSchema.optional(), skipTlsVerification: z.boolean().optional(), enabled: z.boolean().optional() @@ -116,17 +117,16 @@ export async function updateAiProvider( body.upstreamUrl !== undefined ? body.upstreamUrl : existing.upstreamUrl; - const nextAuthType = + const nextAuthType: AiProviderAuthType = body.authType !== undefined ? body.authType - : (existing.authType ?? - (providerType === "custom" ? "bearer" : null)); + : (existing.authType as AiProviderAuthType); const validation = z .object({ type: aiProviderTypeSchema, upstreamUrl: z.string().nullable().optional(), - authType: aiAuthTypeSchema.nullable().optional(), + authType: aiAuthTypeSchema, routingMode: aiRoutingModeSchema.optional() }) .superRefine((data, ctx) => refineProviderUpstreamFields(data, ctx)) @@ -167,13 +167,6 @@ export async function updateAiProvider( } if (body.authType !== undefined) { updateData.authType = body.authType; - } else if ( - providerType === "custom" && - !existing.authType && - nextAuthType - ) { - // Backfill required authType for custom providers created without one - updateData.authType = nextAuthType; } if (body.apiKey !== undefined) { diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts index a5e2aa962..42476c710 100644 --- a/server/routers/aiProvider/validation.ts +++ b/server/routers/aiProvider/validation.ts @@ -1,6 +1,8 @@ import { z } from "zod"; import { + AI_PROVIDER_AUTH_TYPES, providerRequiresUpstreamUrl, + type AiProviderAuthType, type AiProviderRoutingMode, type AiProviderType } from "@server/lib/aiProviderDefaults"; @@ -17,7 +19,7 @@ export const aiProviderTypeSchema = z.enum([ "custom" ]); -export const aiAuthTypeSchema = z.enum(["bearer"]); +export const aiAuthTypeSchema = z.enum(AI_PROVIDER_AUTH_TYPES); export const aiRoutingModeSchema = z.enum(["url", "target"]); @@ -25,7 +27,7 @@ export function refineProviderUpstreamFields( data: { type: AiProviderType; upstreamUrl?: string | null; - authType?: "bearer" | null; + authType?: AiProviderAuthType | null; routingMode?: AiProviderRoutingMode | null; }, ctx: z.RefinementCtx diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx index 02d1d2142..264e6b252 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx @@ -40,7 +40,10 @@ import { type AiProviderFormValues } from "@app/lib/aiProviderFormSchema"; import { zodResolver } from "@hookform/resolvers/zod"; -import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import type { + AiProviderAuthType, + AiProviderType +} from "@server/lib/aiProviderDefaults"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; @@ -63,7 +66,7 @@ export default function AiProviderAuthenticationPage() { type: provider.type as AiProviderType, upstreamUrl: provider.upstreamUrl ?? "", apiKey: provider.apiKey ?? "", - authType: (provider.authType as "bearer" | null) ?? "bearer", + authType: (provider.authType as AiProviderAuthType) ?? "bearer", routingMode: (provider.routingMode as "url" | "target") ?? "url", skipTlsVerification: provider.skipTlsVerification, enabled: provider.enabled @@ -91,7 +94,7 @@ export default function AiProviderAuthenticationPage() { type: updated.type as AiProviderType, upstreamUrl: updated.upstreamUrl ?? "", apiKey: updated.apiKey ?? "", - authType: (updated.authType as "bearer" | null) ?? "bearer", + authType: (updated.authType as AiProviderAuthType) ?? "bearer", routingMode: (updated.routingMode as "url" | "target") ?? "url", skipTlsVerification: updated.skipTlsVerification, enabled: updated.enabled @@ -164,6 +167,26 @@ export default function AiProviderAuthenticationPage() { "aiProviderAuthTypeBearer" )} + + {t( + "aiProviderAuthTypeXApiKey" + )} + + + {t( + "aiProviderAuthTypeXGoogApiKey" + )} + + + {t( + "aiProviderAuthTypeHec" + )} + + + {t( + "aiProviderAuthTypeCfAigAuthorization" + )} + diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx index fbe9669cf..270ba3c68 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx @@ -45,7 +45,10 @@ import { } from "@app/lib/aiProviderFormSchema"; import { aiProviderQueries } from "@app/lib/queries"; import { zodResolver } from "@hookform/resolvers/zod"; -import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import type { + AiProviderAuthType, + AiProviderType +} from "@server/lib/aiProviderDefaults"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import { useQuery } from "@tanstack/react-query"; import type { AxiosResponse } from "axios"; @@ -72,7 +75,7 @@ export default function AiProviderNetworkPage() { type: provider.type as AiProviderType, upstreamUrl: provider.upstreamUrl ?? "", apiKey: "", - authType: (provider.authType as "bearer" | null) ?? "bearer", + authType: (provider.authType as AiProviderAuthType) ?? "bearer", routingMode: (provider.routingMode as "url" | "target") ?? "url", skipTlsVerification: provider.skipTlsVerification, enabled: provider.enabled @@ -115,7 +118,7 @@ export default function AiProviderNetworkPage() { type: updated.type as AiProviderType, upstreamUrl: updated.upstreamUrl ?? "", apiKey: "", - authType: (updated.authType as "bearer" | null) ?? "bearer", + authType: (updated.authType as AiProviderAuthType) ?? "bearer", routingMode: (updated.routingMode as "url" | "target") ?? "url", skipTlsVerification: updated.skipTlsVerification, enabled: updated.enabled diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index b84462153..5ad4c4ec7 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -527,6 +527,26 @@ export default function CreateAiProviderPage() { "aiProviderAuthTypeBearer" )} + + {t( + "aiProviderAuthTypeXApiKey" + )} + + + {t( + "aiProviderAuthTypeXGoogApiKey" + )} + + + {t( + "aiProviderAuthTypeHec" + )} + + + {t( + "aiProviderAuthTypeCfAigAuthorization" + )} + diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index 26245620c..8eb038fba 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import { + AI_PROVIDER_AUTH_TYPES, AI_PROVIDER_DEFAULTS, providerRequiresUpstreamUrl, type AiProviderType @@ -23,7 +24,7 @@ export const aiProviderFormSchema = z type: z.enum(aiProviderTypeValues), upstreamUrl: z.string().optional().nullable(), apiKey: z.string().optional(), - authType: z.enum(["bearer"]).optional().nullable(), + authType: z.enum(AI_PROVIDER_AUTH_TYPES).optional().nullable(), routingMode: z.enum(["url", "target"]).optional(), skipTlsVerification: z.boolean().optional(), enabled: z.boolean().optional() @@ -138,7 +139,7 @@ export function toAiProviderCreatePayload(values: AiProviderFormValues) { authType: values.type === "custom" ? (values.authType ?? "bearer") - : (values.authType ?? undefined), + : undefined, skipTlsVerification: values.skipTlsVerification, enabled: values.enabled ?? true }; @@ -159,14 +160,14 @@ export function toAiProviderUpdatePayload(values: AiProviderFormValues) { name: values.name.trim(), routingMode: values.type === "custom" ? routingMode : "url", upstreamUrl, - authType: - values.type === "custom" - ? (values.authType ?? "bearer") - : (values.authType ?? null), skipTlsVerification: values.skipTlsVerification ?? false, enabled: values.enabled ?? true }; + if (values.type === "custom") { + payload.authType = values.authType ?? "bearer"; + } + if (values.apiKey?.trim()) { payload.apiKey = values.apiKey.trim(); } @@ -184,11 +185,13 @@ export function toAiProviderNetworkPayload(values: AiProviderFormValues) { } export function toAiProviderAuthPayload(values: AiProviderFormValues) { - const full = toAiProviderUpdatePayload(values); - return { - authType: full.authType, + const payload: Record = { ...(values.apiKey !== undefined ? { apiKey: values.apiKey.trim() } : {}) }; + if (values.type === "custom") { + payload.authType = values.authType ?? "bearer"; + } + return payload; } export function toAiProviderConfigurationPayload(values: AiProviderFormValues) { From bcf6b86b841911ae8711b755e2cbe688ebde5956 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 5 Aug 2026 15:39:08 -0400 Subject: [PATCH 051/296] add no auth and passthrough auth --- messages/en-US.json | 11 ++ server/db/pg/schema/schema.ts | 2 + server/db/sqlite/schema/schema.ts | 2 + server/lib/aiProviderDefaults.ts | 30 +++- server/routers/aiGateway/chatCompletions.ts | 25 ++-- server/routers/aiProvider/validation.ts | 8 - .../[providerId]/authentication/page.tsx | 137 +++++++---------- .../settings/ai-providers/create/page.tsx | 138 +++++++---------- src/components/AiProviderAuthTypeSelect.tsx | 139 ++++++++++++++++++ src/lib/aiProviderFormSchema.ts | 36 +++-- 10 files changed, 323 insertions(+), 205 deletions(-) create mode 100644 src/components/AiProviderAuthTypeSelect.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 257c56c42..bf498421a 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1682,12 +1682,23 @@ "aiProviderApiKeyDescription": "API key used to authenticate requests to this provider", "aiProviderApiKeyLastChars": "API Key", "aiProviderAuthType": "Auth Type", + "aiProviderAuthTypeSearch": "Search auth types...", + "aiProviderAuthTypeNotFound": "No auth type found", "aiProviderAuthTypeBearer": "Bearer", + "aiProviderAuthTypeBearerDescription": "Authorization: Bearer key. Used by OpenAI and most providers", "aiProviderAuthTypeXApiKey": "x-api-key", + "aiProviderAuthTypeXApiKeyDescription": "x-api-key header. Used by Anthropic", "aiProviderAuthTypeXGoogApiKey": "x-goog-api-key", + "aiProviderAuthTypeXGoogApiKeyDescription": "x-goog-api-key header. Used by Google Gemini", "aiProviderAuthTypeHec": "Splunk HEC", + "aiProviderAuthTypeHecDescription": "Authorization: Splunk key. Used by Splunk HTTP Event Collector", "aiProviderAuthTypeCfAigAuthorization": "Cloudflare AI Gateway", + "aiProviderAuthTypeCfAigAuthorizationDescription": "cf-aig-authorization: Bearer key. Used by Cloudflare AI Gateway", + "aiProviderAuthTypeNone": "No Auth", + "aiProviderAuthTypePassthrough": "Passthrough", "aiProviderAuthTypeDescription": "How the upstream API authenticates requests", + "aiProviderAuthTypePassthroughDescription": "Forward the caller's API key headers to the upstream", + "aiProviderAuthTypeNoneDescription": "Do not send authentication headers to the upstream", "aiProviderRoutingMode": "Routing Mode", "aiProviderRoutingModeDescription": "Send traffic to an upstream URL or to HTTP targets on your sites", "aiProviderRoutingModeUrl": "Upstream URL", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 7883345da..9bf11a21d 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1651,6 +1651,8 @@ export const aiProviders = pgTable("aiProviders", { | "x-goog-api-key" | "hec" | "cf-aig-authorization" + | "none" + | "passthrough" >() .notNull(), routingMode: varchar("routingMode") diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index e586c4e02..35028e815 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1633,6 +1633,8 @@ export const aiProviders = sqliteTable("aiProviders", { | "x-goog-api-key" | "hec" | "cf-aig-authorization" + | "none" + | "passthrough" >() .notNull(), routingMode: text("routingMode") diff --git a/server/lib/aiProviderDefaults.ts b/server/lib/aiProviderDefaults.ts index d803e7613..70b666954 100644 --- a/server/lib/aiProviderDefaults.ts +++ b/server/lib/aiProviderDefaults.ts @@ -14,7 +14,9 @@ export const AI_PROVIDER_AUTH_TYPES = [ "x-api-key", "x-goog-api-key", "hec", - "cf-aig-authorization" + "cf-aig-authorization", + "none", + "passthrough" ] as const; export type AiProviderAuthType = (typeof AI_PROVIDER_AUTH_TYPES)[number]; @@ -71,6 +73,10 @@ const CONFLICTING_AUTH_HEADERS = [ "cf-aig-authorization" ] as const; +export function authTypeRequiresApiKey(authType: AiProviderAuthType): boolean { + return authType !== "none" && authType !== "passthrough"; +} + export function providerRequiresUpstreamUrl( type: AiProviderType, routingMode: AiProviderRoutingMode = "url" @@ -116,20 +122,26 @@ export function resolveAiProviderCreateFields(input: { const defaults = AI_PROVIDER_DEFAULTS[input.type]; return { upstreamUrl: input.upstreamUrl ?? defaults.upstreamUrl, - authType: defaults.authType, + authType: input.authType ?? defaults.authType, routingMode }; } /** - * Strip inbound client auth headers, then set the provider auth header - * for the given authType. + * Apply provider auth to upstream headers. + * - Injected modes: strip client auth headers, then set the provider key. + * - none: strip client auth headers, send no auth. + * - passthrough: leave client auth headers as-is. */ export function applyAiProviderAuthHeaders( headers: Record, authType: AiProviderAuthType, - apiKey: string + apiKey: string | null ): void { + if (authType === "passthrough") { + return; + } + for (const name of CONFLICTING_AUTH_HEADERS) { for (const key of Object.keys(headers)) { if (key.toLowerCase() === name) { @@ -138,6 +150,14 @@ export function applyAiProviderAuthHeaders( } } + if (authType === "none") { + return; + } + + if (!apiKey) { + throw new Error(`API key required for authType ${authType}`); + } + switch (authType) { case "bearer": headers["Authorization"] = `Bearer ${apiKey}`; diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts index e1bb8bbc9..d2671aa63 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/chatCompletions.ts @@ -19,7 +19,8 @@ import config from "@server/lib/config"; import { decrypt } from "@server/lib/crypto"; import { AiProviderAuthType, - applyAiProviderAuthHeaders + applyAiProviderAuthHeaders, + authTypeRequiresApiKey } from "@server/lib/aiProviderDefaults"; import { SESSION_COOKIE_NAME, @@ -488,15 +489,6 @@ export async function chatCompletions( const { provider } = selection; - if (!provider.apiKey) { - return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ - error: { message: "AI provider has no API key configured" } - }); - } - - const secret = config.getRawConfig().server.secret!; - const apiKey = decrypt(provider.apiKey, secret); - const upstreamUrl = provider.upstreamUrl; const authType = provider.authType as AiProviderAuthType; @@ -508,6 +500,19 @@ export async function chatCompletions( }); } + let apiKey: string | null = null; + if (authTypeRequiresApiKey(authType)) { + if (!provider.apiKey) { + return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ + error: { + message: "AI provider has no API key configured" + } + }); + } + const secret = config.getRawConfig().server.secret!; + apiKey = decrypt(provider.apiKey, secret); + } + const targetUrl = `${upstreamUrl.replace(/\/$/, "")}`; // Drop hop-by-hop / proxy-only headers. Forwarding Host especially diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts index 42476c710..b4ab104aa 100644 --- a/server/routers/aiProvider/validation.ts +++ b/server/routers/aiProvider/validation.ts @@ -52,12 +52,4 @@ export function refineProviderUpstreamFields( path: ["upstreamUrl"] }); } - - if (data.type === "custom" && !data.authType) { - ctx.addIssue({ - code: "custom", - message: "authType is required for custom providers", - path: ["authType"] - }); - } } diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx index 264e6b252..a8f827e93 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx @@ -12,6 +12,7 @@ import { SettingsSectionHeader, SettingsSectionTitle } from "@app/components/Settings"; +import { AiProviderAuthTypeSelect } from "@app/components/AiProviderAuthTypeSelect"; import { Button } from "@app/components/ui/button"; import { Form, @@ -23,13 +24,6 @@ import { FormMessage } from "@app/components/ui/form"; import { Input } from "@app/components/ui/input"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue -} from "@app/components/ui/select"; import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; @@ -40,9 +34,10 @@ import { type AiProviderFormValues } from "@app/lib/aiProviderFormSchema"; import { zodResolver } from "@hookform/resolvers/zod"; -import type { - AiProviderAuthType, - AiProviderType +import { + authTypeRequiresApiKey, + type AiProviderAuthType, + type AiProviderType } from "@server/lib/aiProviderDefaults"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import type { AxiosResponse } from "axios"; @@ -73,7 +68,10 @@ export default function AiProviderAuthenticationPage() { } }); - const showAuthType = provider.type === "custom"; + const authType = form.watch("authType"); + const showApiKey = authTypeRequiresApiKey( + (authType as AiProviderAuthType | null) ?? "bearer" + ); async function onSubmit(values: AiProviderFormValues) { setSaveLoading(true); @@ -135,88 +133,22 @@ export default function AiProviderAuthenticationPage() { id="ai-provider-auth-form" > - {showAuthType && ( - - ( - - - {t( - "aiProviderAuthType" - )} - - - - {t( - "aiProviderAuthTypeDescription" - )} - - - - )} - /> - - )} - ( - {t("aiProviderApiKey")} + {t( + "aiProviderAuthType" + )} - {t( - "aiProviderApiKeyDescription" + "aiProviderAuthTypeDescription" )} @@ -233,6 +165,43 @@ export default function AiProviderAuthenticationPage() { )} /> + + {showApiKey && ( + + ( + + + {t( + "aiProviderApiKey" + )} + + + + + + {t( + "aiProviderApiKeyDescription" + )} + + + + )} + /> + + )} diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 5ad4c4ec7..1de013a2e 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -19,6 +19,7 @@ import { SettingsSubsectionTitle } from "@app/components/Settings"; import HeaderTitle from "@app/components/SettingsSectionTitle"; +import { AiProviderAuthTypeSelect } from "@app/components/AiProviderAuthTypeSelect"; import { AiProviderTypeSelect } from "@app/components/AiProviderTypeSelect"; import { StrategySelect } from "@app/components/StrategySelect"; import { SwitchInput } from "@app/components/SwitchInput"; @@ -33,18 +34,12 @@ import { FormMessage } from "@app/components/ui/form"; import { Input } from "@app/components/ui/input"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue -} from "@app/components/ui/select"; import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { aiProviderCreateFormSchema, + defaultAuthTypeForProvider, emptyUpstreamForType, showsUpstreamUrlField, toAiProviderCreatePayload, @@ -52,6 +47,7 @@ import { type AiProviderFormValues } from "@app/lib/aiProviderFormSchema"; import { zodResolver } from "@hookform/resolvers/zod"; +import { authTypeRequiresApiKey } from "@server/lib/aiProviderDefaults"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; @@ -76,7 +72,7 @@ export default function CreateAiProviderPage() { type: "openai", upstreamUrl: emptyUpstreamForType("openai"), apiKey: "", - authType: "bearer", + authType: defaultAuthTypeForProvider("openai"), routingMode: "url", skipTlsVerification: false, enabled: true @@ -85,12 +81,13 @@ export default function CreateAiProviderPage() { const providerType = form.watch("type"); const routingMode = form.watch("routingMode"); + const authType = form.watch("authType"); const showUpstream = showsUpstreamUrlField(providerType, routingMode); const requireUpstream = upstreamUrlRequired(providerType, routingMode); const showRoutingMode = providerType === "custom"; - const showAuthType = providerType === "custom"; const showTargets = providerType === "custom" && routingMode === "target"; + const showApiKey = authTypeRequiresApiKey(authType ?? "bearer"); async function createTargets( providerId: number, @@ -272,6 +269,12 @@ export default function CreateAiProviderPage() { value ) ); + form.setValue( + "authType", + defaultAuthTypeForProvider( + value + ) + ); if ( value !== "custom" @@ -495,88 +498,22 @@ export default function CreateAiProviderPage() { - {showAuthType && ( - - ( - - - {t( - "aiProviderAuthType" - )} - - - - {t( - "aiProviderAuthTypeDescription" - )} - - - - )} - /> - - )} - ( - {t("aiProviderApiKey")} + {t( + "aiProviderAuthType" + )} - {t( - "aiProviderApiKeyDescription" + "aiProviderAuthTypeDescription" )} @@ -593,6 +530,43 @@ export default function CreateAiProviderPage() { )} /> + + {showApiKey && ( + + ( + + + {t( + "aiProviderApiKey" + )} + + + + + + {t( + "aiProviderApiKeyDescription" + )} + + + + )} + /> + + )} diff --git a/src/components/AiProviderAuthTypeSelect.tsx b/src/components/AiProviderAuthTypeSelect.tsx new file mode 100644 index 000000000..a040c6fa1 --- /dev/null +++ b/src/components/AiProviderAuthTypeSelect.tsx @@ -0,0 +1,139 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList +} from "@app/components/ui/command"; +import { + Popover, + PopoverContent, + PopoverTrigger +} from "@app/components/ui/popover"; +import { cn } from "@app/lib/cn"; +import { + AI_PROVIDER_AUTH_TYPES, + type AiProviderAuthType +} from "@server/lib/aiProviderDefaults"; +import { CheckIcon, ChevronsUpDown } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useMemo, useState } from "react"; + +const authLabelMap = { + bearer: "aiProviderAuthTypeBearer", + "x-api-key": "aiProviderAuthTypeXApiKey", + "x-goog-api-key": "aiProviderAuthTypeXGoogApiKey", + hec: "aiProviderAuthTypeHec", + "cf-aig-authorization": "aiProviderAuthTypeCfAigAuthorization", + none: "aiProviderAuthTypeNone", + passthrough: "aiProviderAuthTypePassthrough" +} as const; + +const authDescriptionMap = { + bearer: "aiProviderAuthTypeBearerDescription", + "x-api-key": "aiProviderAuthTypeXApiKeyDescription", + "x-goog-api-key": "aiProviderAuthTypeXGoogApiKeyDescription", + hec: "aiProviderAuthTypeHecDescription", + "cf-aig-authorization": "aiProviderAuthTypeCfAigAuthorizationDescription", + none: "aiProviderAuthTypeNoneDescription", + passthrough: "aiProviderAuthTypePassthroughDescription" +} as const; + +type AiProviderAuthTypeSelectProps = { + value: AiProviderAuthType; + onChange: (value: AiProviderAuthType) => void; + disabled?: boolean; + className?: string; +}; + +export function AiProviderAuthTypeSelect({ + value, + onChange, + disabled, + className +}: AiProviderAuthTypeSelectProps) { + const t = useTranslations(); + const [open, setOpen] = useState(false); + + const options = useMemo( + () => + AI_PROVIDER_AUTH_TYPES.map((authType) => ({ + authType, + title: t(authLabelMap[authType]), + description: t(authDescriptionMap[authType]) + })), + [t] + ); + + const selected = options.find((option) => option.authType === value); + + return ( + + + + + + + + + + {t("aiProviderAuthTypeNotFound")} + + + {options.map((option) => ( + { + onChange(option.authType); + setOpen(false); + }} + > + +
+ + {option.title} + + + {option.description} + +
+ + ))} + + + + + + ); +} diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index 8eb038fba..67b540305 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -2,7 +2,9 @@ import { z } from "zod"; import { AI_PROVIDER_AUTH_TYPES, AI_PROVIDER_DEFAULTS, + authTypeRequiresApiKey, providerRequiresUpstreamUrl, + type AiProviderAuthType, type AiProviderType } from "@server/lib/aiProviderDefaults"; @@ -70,10 +72,10 @@ export const aiProviderFormSchema = z }); } - if (data.type === "custom" && !data.authType) { + if (!data.authType) { ctx.addIssue({ code: "custom", - message: "authType is required for custom providers", + message: "authType is required", path: ["authType"] }); } @@ -83,7 +85,9 @@ export type AiProviderFormValues = z.infer; export const aiProviderCreateFormSchema = aiProviderFormSchema.superRefine( (data, ctx) => { - if (!data.apiKey?.trim()) { + const authType: AiProviderAuthType = data.authType ?? "bearer"; + + if (authTypeRequiresApiKey(authType) && !data.apiKey?.trim()) { ctx.addIssue({ code: "custom", message: "API key is required", @@ -93,6 +97,15 @@ export const aiProviderCreateFormSchema = aiProviderFormSchema.superRefine( } ); +export function defaultAuthTypeForProvider( + type: AiProviderType +): AiProviderAuthType { + if (type === "custom") { + return "bearer"; + } + return AI_PROVIDER_DEFAULTS[type].authType; +} + export function emptyUpstreamForType(type: AiProviderType): string { if (type === "custom") { return ""; @@ -136,10 +149,7 @@ export function toAiProviderCreatePayload(values: AiProviderFormValues) { routingMode: values.type === "custom" ? routingMode : undefined, upstreamUrl, apiKey: values.apiKey?.trim() ? values.apiKey.trim() : undefined, - authType: - values.type === "custom" - ? (values.authType ?? "bearer") - : undefined, + authType: values.authType ?? "bearer", skipTlsVerification: values.skipTlsVerification, enabled: values.enabled ?? true }; @@ -160,14 +170,11 @@ export function toAiProviderUpdatePayload(values: AiProviderFormValues) { name: values.name.trim(), routingMode: values.type === "custom" ? routingMode : "url", upstreamUrl, + authType: values.authType ?? "bearer", skipTlsVerification: values.skipTlsVerification ?? false, enabled: values.enabled ?? true }; - if (values.type === "custom") { - payload.authType = values.authType ?? "bearer"; - } - if (values.apiKey?.trim()) { payload.apiKey = values.apiKey.trim(); } @@ -185,13 +192,10 @@ export function toAiProviderNetworkPayload(values: AiProviderFormValues) { } export function toAiProviderAuthPayload(values: AiProviderFormValues) { - const payload: Record = { + return { + authType: values.authType ?? "bearer", ...(values.apiKey !== undefined ? { apiKey: values.apiKey.trim() } : {}) }; - if (values.type === "custom") { - payload.authType = values.authType ?? "bearer"; - } - return payload; } export function toAiProviderConfigurationPayload(values: AiProviderFormValues) { From 790daba79633435c229fbd529f9e35ea19818a97 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 5 Aug 2026 15:43:42 -0400 Subject: [PATCH 052/296] allow no provider on a resource --- messages/en-US.json | 1 - server/routers/resource/createResource.ts | 2 +- .../resource/removeAiProviderFromResource.ts | 20 ++++-------- .../resource/setResourceAiProviders.ts | 4 +-- .../siteResource/createSiteResource.ts | 2 +- .../removeAiProviderFromSiteResource.ts | 20 ++++-------- .../setSiteResourceAiProviders.ts | 4 +-- .../private/[niceId]/providers/page.tsx | 6 ++-- .../public/[niceId]/providers/page.tsx | 6 ++-- .../settings/resources/public/create/page.tsx | 32 +------------------ src/lib/privateResourceForm.ts | 14 -------- 11 files changed, 25 insertions(+), 86 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index bf498421a..5528c0ee1 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1739,7 +1739,6 @@ "aiResourceProvidersHelp": "Models must be defined on each provider. Model names cannot overlap across selected providers.", "aiResourceProvidersSelect": "Select providers", "aiResourceProvidersEmpty": "No AI providers found", - "aiResourceProvidersRequired": "Select at least one AI provider", "aiResourceProvidersUpdated": "Providers updated", "aiResourceProvidersErrorUpdate": "Failed to update providers", "aiResourceAliasRequired": "Alias is required for inference resources", diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index 4d546468f..47e70140f 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -394,7 +394,7 @@ async function createHttpResource( const resolved = await resolveProviderAttachments({ orgId, attachments: aiProviderInputs ?? [], - requireAtLeastOne: true + requireAtLeastOne: false }); if (isInferenceFieldsError(resolved)) { return next(createHttpError(HttpCode.BAD_REQUEST, resolved.error)); diff --git a/server/routers/resource/removeAiProviderFromResource.ts b/server/routers/resource/removeAiProviderFromResource.ts index d3a4294ad..e74b0c886 100644 --- a/server/routers/resource/removeAiProviderFromResource.ts +++ b/server/routers/resource/removeAiProviderFromResource.ts @@ -77,8 +77,9 @@ export async function removeAiProviderFromResource( const { providerId } = parsedBody.data; - const parsedParams = - removeAiProviderFromResourceParamsSchema.safeParse(req.params); + const parsedParams = removeAiProviderFromResourceParamsSchema.safeParse( + req.params + ); if (!parsedParams.success) { return next( createHttpError( @@ -129,22 +130,15 @@ export async function removeAiProviderFromResource( modelAccessMode: a.modelAccessMode })); - if (remaining.length === 0) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "At least one AI provider is required for inference-mode resources" - ) - ); - } - const attachments = await resolveProviderAttachments({ orgId: resource.orgId, attachments: remaining, - requireAtLeastOne: true + requireAtLeastOne: false }); if (isInferenceFieldsError(attachments)) { - return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + return next( + createHttpError(HttpCode.BAD_REQUEST, attachments.error) + ); } await setPublicResourceAiProviders(resourceId, attachments); diff --git a/server/routers/resource/setResourceAiProviders.ts b/server/routers/resource/setResourceAiProviders.ts index 0cd0d5d84..88552954b 100644 --- a/server/routers/resource/setResourceAiProviders.ts +++ b/server/routers/resource/setResourceAiProviders.ts @@ -27,7 +27,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-providers", description: - "Replace the AI providers attached to an inference resource. At least one provider is required. Model keys must be unique across attached catalog providers.", + "Replace the AI providers attached to an inference resource. An empty list clears all providers. Model keys must be unique across attached catalog providers.", tags: [OpenAPITags.PublicResource], request: { params: setResourceAiProvidersParamsSchema, @@ -113,7 +113,7 @@ export async function setResourceAiProviders( const attachments = await resolveProviderAttachments({ orgId: resource.orgId, attachments: providers, - requireAtLeastOne: true + requireAtLeastOne: false }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 277c853ab..feeca0938 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -353,7 +353,7 @@ export async function createSiteResource( const resolved = await resolveProviderAttachments({ orgId, attachments: aiProviderInputs ?? [], - requireAtLeastOne: true + requireAtLeastOne: false }); if (isInferenceFieldsError(resolved)) { return next( diff --git a/server/routers/siteResource/removeAiProviderFromSiteResource.ts b/server/routers/siteResource/removeAiProviderFromSiteResource.ts index 65781859c..3d4392820 100644 --- a/server/routers/siteResource/removeAiProviderFromSiteResource.ts +++ b/server/routers/siteResource/removeAiProviderFromSiteResource.ts @@ -63,8 +63,9 @@ export async function removeAiProviderFromSiteResource( next: NextFunction ): Promise { try { - const parsedBody = - removeAiProviderFromSiteResourceBodySchema.safeParse(req.body); + const parsedBody = removeAiProviderFromSiteResourceBodySchema.safeParse( + req.body + ); if (!parsedBody.success) { return next( createHttpError( @@ -128,22 +129,15 @@ export async function removeAiProviderFromSiteResource( modelAccessMode: a.modelAccessMode })); - if (remaining.length === 0) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "At least one AI provider is required for inference-mode resources" - ) - ); - } - const attachments = await resolveProviderAttachments({ orgId: siteResource.orgId, attachments: remaining, - requireAtLeastOne: true + requireAtLeastOne: false }); if (isInferenceFieldsError(attachments)) { - return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + return next( + createHttpError(HttpCode.BAD_REQUEST, attachments.error) + ); } await setSiteResourceAiProviders(siteResourceId, attachments); diff --git a/server/routers/siteResource/setSiteResourceAiProviders.ts b/server/routers/siteResource/setSiteResourceAiProviders.ts index c6e708064..fd685bb20 100644 --- a/server/routers/siteResource/setSiteResourceAiProviders.ts +++ b/server/routers/siteResource/setSiteResourceAiProviders.ts @@ -27,7 +27,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-providers", description: - "Replace the AI providers attached to an inference site resource. At least one provider is required. Model keys must be unique across attached catalog providers.", + "Replace the AI providers attached to an inference site resource. An empty list clears all providers. Model keys must be unique across attached catalog providers.", tags: [OpenAPITags.PrivateResource], request: { params: setSiteResourceAiProvidersParamsSchema, @@ -115,7 +115,7 @@ export async function setSiteResourceAiProviders( const attachments = await resolveProviderAttachments({ orgId: siteResource.orgId, attachments: providers, - requireAtLeastOne: true + requireAtLeastOne: false }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx index ead7ab24b..dcbc4ea29 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx @@ -58,11 +58,9 @@ export default function PrivateResourceProvidersPage() { const formSchema = useMemo( () => z.object({ - providerIds: z - .array(z.number().int().positive()) - .min(1, t("aiResourceProvidersRequired")) + providerIds: z.array(z.number().int().positive()) }), - [t] + [] ); type FormValues = z.infer; diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx index 67ef0fa9e..51182c2a8 100644 --- a/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx +++ b/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx @@ -58,11 +58,9 @@ export default function PublicResourceProvidersPage() { const formSchema = useMemo( () => z.object({ - providerIds: z - .array(z.number().int().positive()) - .min(1, t("aiResourceProvidersRequired")) + providerIds: z.array(z.number().int().positive()) }), - [t] + [] ); type FormValues = z.infer; diff --git a/src/app/[orgId]/settings/resources/public/create/page.tsx b/src/app/[orgId]/settings/resources/public/create/page.tsx index c0e0a0f8e..a5088d84e 100644 --- a/src/app/[orgId]/settings/resources/public/create/page.tsx +++ b/src/app/[orgId]/settings/resources/public/create/page.tsx @@ -252,7 +252,6 @@ export default function Page() { const [selectedProviders, setSelectedProviders] = useState< SelectedAiProvider[] >([]); - const [showProvidersError, setShowProvidersError] = useState(false); // SSH-specific state const [sshServerMode, setSshServerMode] = useState<"standard" | "native">( @@ -355,9 +354,6 @@ export default function Page() { if (!availableTypes.includes(resourceType)) { setResourceType("http"); } - if (resourceType !== "inference") { - setShowProvidersError(false); - } }, [availableTypes, resourceType]); const baseResourceFormSchema = useMemo( @@ -1440,14 +1436,6 @@ export default function Page() { setSelectedProviders( providers ); - if ( - providers.length > - 0 - ) { - setShowProvidersError( - false - ); - } }} />

@@ -1455,13 +1443,6 @@ export default function Page() { "aiResourceProvidersHelp" )}

- {showProvidersError && ( -

- {t( - "aiResourceProvidersRequired" - )} -

- )}
@@ -1493,16 +1474,6 @@ export default function Page() { const tcpValid = !isHttpResource ? await tcpUdpForm.trigger() : true; - const providersValid = - resourceType !== "inference" || - selectedProviders.length > 0; - - if ( - resourceType === "inference" && - !providersValid - ) { - setShowProvidersError(true); - } if ( resourceType === "ssh" && @@ -1527,8 +1498,7 @@ export default function Page() { baseValid && domainValid && tcpValid && - bgValid && - providersValid + bgValid ) { onSubmit(); } diff --git a/src/lib/privateResourceForm.ts b/src/lib/privateResourceForm.ts index 4c60f1cef..e336a45a1 100644 --- a/src/lib/privateResourceForm.ts +++ b/src/lib/privateResourceForm.ts @@ -467,13 +467,6 @@ export function createCreateFormSchema(t: TranslateFn) { path: ["alias"] }); } - if (!data.providerIds || data.providerIds.length < 1) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: t("aiResourceProvidersRequired"), - path: ["providerIds"] - }); - } } if ( data.mode !== "ssh" && @@ -639,13 +632,6 @@ export function createInferenceFormSchema(t: TranslateFn) { path: ["alias"] }); } - if (!data.providerIds || data.providerIds.length < 1) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: t("aiResourceProvidersRequired"), - path: ["providerIds"] - }); - } }); } From 2e8bd7a8c7c253d03bec186f94557b60c54d944d Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 5 Aug 2026 15:56:12 -0400 Subject: [PATCH 053/296] improve form error codes --- messages/en-US.json | 5 + .../[providerId]/authentication/page.tsx | 8 +- .../[providerId]/general/page.tsx | 23 ++-- .../[providerId]/network/page.tsx | 8 +- .../settings/ai-providers/create/page.tsx | 8 +- src/lib/aiProviderFormSchema.ts | 128 ++++++++++-------- 6 files changed, 103 insertions(+), 77 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 5528c0ee1..39a3b54e6 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1721,6 +1721,11 @@ "aiProviderErrorUpdate": "Failed to update AI provider", "aiProviderErrorDelete": "Failed to delete AI provider", "aiProviderErrorLoad": "Failed to load AI provider", + "aiProviderErrorUpstreamUrlInvalid": "Enter a valid upstream URL", + "aiProviderErrorUpstreamUrlRequired": "Upstream URL is required for this provider", + "aiProviderErrorAuthTypeRequired": "Auth type is required", + "aiProviderErrorApiKeyRequired": "API key is required", + "aiProviderErrorRoutingModeTarget": "Site targets routing is only available for custom providers", "aiProviderCreated": "AI provider created", "aiProviderUpdated": "AI provider updated", "aiProviderDeleted": "AI provider deleted", diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx index a8f827e93..5a7b068d8 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx @@ -29,7 +29,7 @@ import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { - aiProviderFormSchema, + createAiProviderFormSchema, toAiProviderAuthPayload, type AiProviderFormValues } from "@app/lib/aiProviderFormSchema"; @@ -43,7 +43,7 @@ import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/ import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; import { useRouter } from "next/navigation"; -import { useState } from "react"; +import { useMemo, useState } from "react"; import { useForm } from "react-hook-form"; export default function AiProviderAuthenticationPage() { @@ -54,8 +54,10 @@ export default function AiProviderAuthenticationPage() { const t = useTranslations(); const [saveLoading, setSaveLoading] = useState(false); + const formSchema = useMemo(() => createAiProviderFormSchema(t), [t]); + const form = useForm({ - resolver: zodResolver(aiProviderFormSchema), + resolver: zodResolver(formSchema), defaultValues: { name: provider.name, type: provider.type as AiProviderType, diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx index 99db2eda8..1018f8e63 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx @@ -32,17 +32,10 @@ import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/ import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; import { useRouter } from "next/navigation"; -import { useState } from "react"; +import { useMemo, useState } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod"; -const generalSchema = z.object({ - name: z.string().trim().min(1), - enabled: z.boolean() -}); - -type GeneralFormValues = z.infer; - export default function AiProviderGeneralPage() { const { provider, updateProvider } = useAiProviderContext(); const { env } = useEnvContext(); @@ -51,6 +44,20 @@ export default function AiProviderGeneralPage() { const t = useTranslations(); const [saveLoading, setSaveLoading] = useState(false); + const generalSchema = useMemo( + () => + z.object({ + name: z + .string() + .trim() + .min(1, { message: t("nameRequired") }), + enabled: z.boolean() + }), + [t] + ); + + type GeneralFormValues = z.infer; + const form = useForm({ resolver: zodResolver(generalSchema), defaultValues: { diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx index 270ba3c68..49f251cfa 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx @@ -37,7 +37,7 @@ import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { - aiProviderFormSchema, + createAiProviderFormSchema, showsUpstreamUrlField, toAiProviderNetworkPayload, upstreamUrlRequired, @@ -54,7 +54,7 @@ import { useQuery } from "@tanstack/react-query"; import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; import { useParams, useRouter } from "next/navigation"; -import { useRef, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { useForm } from "react-hook-form"; export default function AiProviderNetworkPage() { @@ -68,8 +68,10 @@ export default function AiProviderNetworkPage() { const [saveLoading, setSaveLoading] = useState(false); const targetsFormRef = useRef(null); + const formSchema = useMemo(() => createAiProviderFormSchema(t), [t]); + const form = useForm({ - resolver: zodResolver(aiProviderFormSchema), + resolver: zodResolver(formSchema), defaultValues: { name: provider.name, type: provider.type as AiProviderType, diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 1de013a2e..1947c3f85 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -38,7 +38,7 @@ import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { - aiProviderCreateFormSchema, + createAiProviderCreateFormSchema, defaultAuthTypeForProvider, emptyUpstreamForType, showsUpstreamUrlField, @@ -52,7 +52,7 @@ import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/ import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; import { useParams, useRouter } from "next/navigation"; -import { useRef, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { useForm } from "react-hook-form"; export default function CreateAiProviderPage() { @@ -65,8 +65,10 @@ export default function CreateAiProviderPage() { const [loading, setLoading] = useState(false); const targetsRef = useRef([]); + const formSchema = useMemo(() => createAiProviderCreateFormSchema(t), [t]); + const form = useForm({ - resolver: zodResolver(aiProviderCreateFormSchema), + resolver: zodResolver(formSchema), defaultValues: { name: "", type: "openai", diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index 67b540305..f484d1f05 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -8,6 +8,8 @@ import { type AiProviderType } from "@server/lib/aiProviderDefaults"; +type TranslateFn = (key: string) => string; + export const aiProviderTypeValues = [ "openai", "anthropic", @@ -20,82 +22,88 @@ export const aiProviderTypeValues = [ "custom" ] as const satisfies readonly AiProviderType[]; -export const aiProviderFormSchema = z - .object({ - name: z.string().trim().min(1), - type: z.enum(aiProviderTypeValues), - upstreamUrl: z.string().optional().nullable(), - apiKey: z.string().optional(), - authType: z.enum(AI_PROVIDER_AUTH_TYPES).optional().nullable(), - routingMode: z.enum(["url", "target"]).optional(), - skipTlsVerification: z.boolean().optional(), - enabled: z.boolean().optional() - }) - .superRefine((data, ctx) => { - const routingMode = - data.type === "custom" ? (data.routingMode ?? "url") : "url"; +export function createAiProviderFormSchema(t: TranslateFn) { + return z + .object({ + name: z + .string() + .trim() + .min(1, { message: t("nameRequired") }), + type: z.enum(aiProviderTypeValues), + upstreamUrl: z.string().optional().nullable(), + apiKey: z.string().optional(), + authType: z.enum(AI_PROVIDER_AUTH_TYPES).optional().nullable(), + routingMode: z.enum(["url", "target"]).optional(), + skipTlsVerification: z.boolean().optional(), + enabled: z.boolean().optional() + }) + .superRefine((data, ctx) => { + const routingMode = + data.type === "custom" ? (data.routingMode ?? "url") : "url"; - if (data.type !== "custom" && data.routingMode === "target") { - ctx.addIssue({ - code: "custom", - message: - "routingMode target is only allowed for custom providers", - path: ["routingMode"] - }); - } - - const upstreamUrl = - data.upstreamUrl && data.upstreamUrl.trim().length > 0 - ? data.upstreamUrl.trim() - : null; - - if (upstreamUrl) { - try { - new URL(upstreamUrl); - } catch { + if (data.type !== "custom" && data.routingMode === "target") { ctx.addIssue({ code: "custom", - message: "Invalid URL", + message: t("aiProviderErrorRoutingModeTarget"), + path: ["routingMode"] + }); + } + + const upstreamUrl = + data.upstreamUrl && data.upstreamUrl.trim().length > 0 + ? data.upstreamUrl.trim() + : null; + + if (upstreamUrl) { + try { + new URL(upstreamUrl); + } catch { + ctx.addIssue({ + code: "custom", + message: t("aiProviderErrorUpstreamUrlInvalid"), + path: ["upstreamUrl"] + }); + } + } + + if ( + providerRequiresUpstreamUrl(data.type, routingMode) && + !upstreamUrl + ) { + ctx.addIssue({ + code: "custom", + message: t("aiProviderErrorUpstreamUrlRequired"), path: ["upstreamUrl"] }); } - } - if ( - providerRequiresUpstreamUrl(data.type, routingMode) && - !upstreamUrl - ) { - ctx.addIssue({ - code: "custom", - message: `upstreamUrl is required for ${data.type} providers`, - path: ["upstreamUrl"] - }); - } + if (!data.authType) { + ctx.addIssue({ + code: "custom", + message: t("aiProviderErrorAuthTypeRequired"), + path: ["authType"] + }); + } + }); +} - if (!data.authType) { - ctx.addIssue({ - code: "custom", - message: "authType is required", - path: ["authType"] - }); - } - }); - -export type AiProviderFormValues = z.infer; - -export const aiProviderCreateFormSchema = aiProviderFormSchema.superRefine( - (data, ctx) => { +export function createAiProviderCreateFormSchema(t: TranslateFn) { + return createAiProviderFormSchema(t).superRefine((data, ctx) => { const authType: AiProviderAuthType = data.authType ?? "bearer"; if (authTypeRequiresApiKey(authType) && !data.apiKey?.trim()) { ctx.addIssue({ code: "custom", - message: "API key is required", + message: t("aiProviderErrorApiKeyRequired"), path: ["apiKey"] }); } - } -); + }); +} + +export type AiProviderFormValues = z.infer< + ReturnType +>; export function defaultAuthTypeForProvider( type: AiProviderType From c5b0e1e87640bf6ce9a7838e04260af558a667d6 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 5 Aug 2026 13:41:57 -0400 Subject: [PATCH 054/296] add QUERY_LOGGING env to sqlite driver for verbose logging --- server/db/sqlite/driver.ts | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/server/db/sqlite/driver.ts b/server/db/sqlite/driver.ts index a7eee52b7..6b115e4ee 100644 --- a/server/db/sqlite/driver.ts +++ b/server/db/sqlite/driver.ts @@ -6,13 +6,48 @@ import fs from "fs"; import { APP_PATH } from "@server/lib/consts"; import { existsSync, mkdirSync } from "fs"; +// Temporary diagnostic trace for the random better-sqlite3 native aborts +// (Statement::~Statement -> RemoveEnvironmentCleanupHook assertion). That +// abort is a hard SIGABRT from native code, so it bypasses uncaughtException/ +// unhandledRejection and can outrun winston's async file transport. This +// writes every statement text synchronously (fsync'd via appendFileSync) so +// the statements immediately preceding a crash survive it. better-sqlite3's +// `verbose` hook fires for BEGIN/SAVEPOINT/RELEASE/COMMIT/ROLLBACK too, since +// those are just prepared statements under the hood - so this also lets us +// see if two "transactions" ever overlap on the shared connection. +// Enable with SQL_TRACE=true; remove once the crash is root-caused. +function sqlTraceVerbose(): + | ((message: unknown, ...args: unknown[]) => void) + | undefined { + if (process.env.SQL_TRACE !== "true") { + return undefined; + } + const traceLogDir = path.join(APP_PATH, "logs"); + if (!existsSync(traceLogDir)) { + mkdirSync(traceLogDir, { recursive: true }); + } + const traceLogPath = path.join(traceLogDir, "sql-trace.log"); + let seq = 0; + return (message: unknown) => { + seq += 1; + const line = `${new Date().toISOString()} pid=${process.pid} #${seq} ${String(message).replace(/\s+/g, " ").trim()}\n`; + try { + fs.appendFileSync(traceLogPath, line); + } catch { + // best-effort diagnostic logging only + } + }; +} + export const location = path.join(APP_PATH, "db", "db.sqlite"); export const exists = checkFileExists(location); bootstrapVolume(); function createDb() { - const sqlite = new Database(location); + const verbose = + process.env.QUERY_LOGGING == "true" ? sqlTraceVerbose() : undefined; + const sqlite = new Database(location, { verbose }); if (process.env.ENABLE_SQLITE_WAL_MODE == "true") { // Enable WAL mode — allows concurrent readers + single writer, preventing From 796bf3714183700a401f80ca15c35e1d75d55183 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 5 Aug 2026 16:24:01 -0400 Subject: [PATCH 055/296] ai gateway override only required sometimes --- .../private/lib/traefik/getTraefikConfig.ts | 67 +++++++++++-------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 67c037550..47d6b69c8 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -1542,6 +1542,12 @@ export async function getTraefikConfig( aiGatewayHost = undefined; } + // The p-host smuggling above is only necessary when the AI gateway + // is overridden to a different host than the resource's own. In the + // default case, leave the Host header untouched so it's visible on + // the other end. + const aiGatewayOverride = config.getRawConfig().server.ai_gateway_override; + // Public inference resources: same TLS/cert-resolver handling as // plain http-mode resources, but the service points at the AI // gateway instead of any real backend targets. @@ -1607,23 +1613,24 @@ export async function getTraefikConfig( } } - const irHeadersMiddlewareName = `${irKey}-headers-middleware`; - config_output.http.middlewares[irHeadersMiddlewareName] = { - headers: { - customRequestHeaders: { - ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": fullDomain - } - } - }; - const additionalMiddlewares = config.getRawConfig().traefik.additional_middlewares || []; - const routerMiddlewares = [ - badgerMiddlewareName, - irHeadersMiddlewareName, - ...additionalMiddlewares - ]; + const routerMiddlewares = [badgerMiddlewareName]; + + if (aiGatewayOverride) { + const irHeadersMiddlewareName = `${irKey}-headers-middleware`; + config_output.http.middlewares[irHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), + "p-host": fullDomain + } + } + }; + routerMiddlewares.push(irHeadersMiddlewareName); + } + + routerMiddlewares.push(...additionalMiddlewares); if (ir.ssl) { config_output.http.routers[routerName + "-redirect"] = { @@ -1705,22 +1712,24 @@ export async function getTraefikConfig( } } - const srHeadersMiddlewareName = `${srKey}-headers-middleware`; - config_output.http.middlewares[srHeadersMiddlewareName] = { - headers: { - customRequestHeaders: { - ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": alias - } - } - }; - const additionalMiddlewares = config.getRawConfig().traefik.additional_middlewares || []; - const routerMiddlewares = [ - srHeadersMiddlewareName, - ...additionalMiddlewares - ]; + const routerMiddlewares: string[] = []; + + if (aiGatewayOverride) { + const srHeadersMiddlewareName = `${srKey}-headers-middleware`; + config_output.http.middlewares[srHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), + "p-host": alias + } + } + }; + routerMiddlewares.push(srHeadersMiddlewareName); + } + + routerMiddlewares.push(...additionalMiddlewares); if (sr.ssl) { config_output.http.routers[routerName + "-redirect"] = { From b775c5b6748c50aee2adecb9db66cfe35ebbab6a Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 5 Aug 2026 16:24:09 -0400 Subject: [PATCH 056/296] include clients in the peer config --- server/routers/gerbil/getConfig.ts | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/server/routers/gerbil/getConfig.ts b/server/routers/gerbil/getConfig.ts index 77921c39f..ffc3c6120 100644 --- a/server/routers/gerbil/getConfig.ts +++ b/server/routers/gerbil/getConfig.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { sites, exitNodes, ExitNode } from "@server/db"; +import { sites, exitNodes, ExitNode, clients } from "@server/db"; import { db } from "@server/db"; import { eq, isNotNull, and } from "drizzle-orm"; import HttpCode from "@server/types/HttpCode"; @@ -93,7 +93,23 @@ export async function generateGerbilConfig(exitNode: ExitNode) { ) ); - const peers = await Promise.all( + const clientsRes = await db + .select() + .from(clients) + .where( + and( + eq(clients.exitNodeId, exitNode.exitNodeId), + isNotNull(clients.pubKey), + isNotNull(clients.exitNodeSubnet) + ) + ); + + let peers: { + publicKey: string | null; + allowedIps: string[]; + }[] = []; + + const sitePeers = await Promise.all( sitesRes.map(async (site) => { if (site.type === "wireguard") { return { @@ -113,6 +129,15 @@ export async function generateGerbilConfig(exitNode: ExitNode) { }) ); + const clientPeers = clientsRes.map((client) => { + return { + publicKey: client.pubKey, + allowedIps: [client.exitNodeSubnet!] + }; + }); + + peers = [...sitePeers, ...clientPeers]; + const configResponse: GetConfigResponse = { listenPort: exitNode.listenPort || 51820, ipAddress: exitNode.address, From 39e06f2b6dddde344e33510d503800bd12118ec2 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 5 Aug 2026 16:55:48 -0400 Subject: [PATCH 057/296] add api capabilities --- messages/en-US.json | 23 ++ server/aiGatewayServer.ts | 4 +- server/db/pg/schema/schema.ts | 1 + server/db/sqlite/schema/schema.ts | 1 + server/lib/aiCapabilities.ts | 257 ++++++++++++++++++ .../aiGateway/createAiGatewayRouter.ts | 21 ++ server/routers/aiGateway/index.ts | 3 +- .../{chatCompletions.ts => pipeline.ts} | 73 ++--- server/routers/aiProvider/createAiProvider.ts | 12 + server/routers/aiProvider/types.ts | 14 +- server/routers/aiProvider/updateAiProvider.ts | 31 ++- server/routers/aiProvider/validation.ts | 18 ++ .../[providerId]/general/page.tsx | 119 +++++++- .../settings/ai-providers/create/page.tsx | 75 +++++ .../AiProviderCapabilitiesSelect.tsx | 89 ++++++ src/lib/aiProviderFormSchema.ts | 31 +++ 16 files changed, 715 insertions(+), 57 deletions(-) create mode 100644 server/lib/aiCapabilities.ts create mode 100644 server/routers/aiGateway/createAiGatewayRouter.ts rename server/routers/aiGateway/{chatCompletions.ts => pipeline.ts} (88%) create mode 100644 src/components/AiProviderCapabilitiesSelect.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 39a3b54e6..ac6131abc 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1726,6 +1726,29 @@ "aiProviderErrorAuthTypeRequired": "Auth type is required", "aiProviderErrorApiKeyRequired": "API key is required", "aiProviderErrorRoutingModeTarget": "Site targets routing is only available for custom providers", + "aiProviderErrorCapabilitiesRequired": "Select at least one API capability", + "aiProviderCapabilities": "API Capabilities", + "aiProviderCapabilitiesDescription": "Which API formats this provider accepts. Built-in providers use fixed capabilities.", + "aiProviderCapabilitiesCustomDescription": "Select which API formats this custom provider can handle", + "aiProviderCapabilitiesSelect": "Select capabilities", + "aiProviderCapabilitiesEmpty": "No capabilities found", + "aiProviderCapabilitiesSearch": "Search capabilities...", + "aiCapabilityOpenaiChat": "OpenAI Chat Completions", + "aiCapabilityOpenaiChatDescription": "Supports /v1/chat/completions", + "aiCapabilityOpenaiResponses": "OpenAI Responses", + "aiCapabilityOpenaiResponsesDescription": "Supports /v1/responses", + "aiCapabilityAnthropicMessages": "Anthropic Messages", + "aiCapabilityAnthropicMessagesDescription": "Supports /v1/messages", + "aiCapabilityGeminiGenerateContent": "Gemini Generate Content", + "aiCapabilityGeminiGenerateContentDescription": "Supports the direct Gemini API", + "aiCapabilityBedrockModelInvoke": "Bedrock Model Invoke", + "aiCapabilityBedrockModelInvokeDescription": "Supports Amazon Bedrock InvokeModel", + "aiCapabilityGoogleGenerateContent": "Vertex Generate Content", + "aiCapabilityGoogleGenerateContentDescription": "Supports Vertex AI Gemini format", + "aiCapabilityGoogleRawPredict": "Vertex Raw Predict", + "aiCapabilityGoogleRawPredictDescription": "Supports Vertex AI rawPredict for Anthropic models", + "aiCapabilityBedrockConverse": "Bedrock Converse", + "aiCapabilityBedrockConverseDescription": "Supports Amazon Bedrock Converse API", "aiProviderCreated": "AI provider created", "aiProviderUpdated": "AI provider updated", "aiProviderDeleted": "AI provider deleted", diff --git a/server/aiGatewayServer.ts b/server/aiGatewayServer.ts index db8b12324..87d57b86b 100644 --- a/server/aiGatewayServer.ts +++ b/server/aiGatewayServer.ts @@ -7,7 +7,7 @@ import { errorHandlerMiddleware, notFoundMiddleware } from "@server/middlewares"; -import * as aiGateway from "@server/routers/aiGateway"; +import { createAiGatewayRouter } from "@server/routers/aiGateway"; const aiGatewayPort = config.getRawConfig().server.ai_gateway_port; @@ -23,7 +23,7 @@ export function createAiGatewayServer() { aiGatewayServer.use(cors()); aiGatewayServer.use(express.json()); - aiGatewayServer.post("/chat/completions", aiGateway.chatCompletions); + aiGatewayServer.use(createAiGatewayRouter()); aiGatewayServer.use(notFoundMiddleware); aiGatewayServer.use(errorHandlerMiddleware); diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 9bf11a21d..3091ed618 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1659,6 +1659,7 @@ export const aiProviders = pgTable("aiProviders", { .$type<"url" | "target">() .notNull() .default("url"), + capabilities: text("capabilities").notNull().default("[]"), skipTlsVerification: boolean("skipTlsVerification") .notNull() .default(false), diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 35028e815..5c115cb2e 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1641,6 +1641,7 @@ export const aiProviders = sqliteTable("aiProviders", { .$type<"url" | "target">() .notNull() .default("url"), + capabilities: text("capabilities").notNull().default("[]"), skipTlsVerification: integer("skipTlsVerification", { mode: "boolean" }) .notNull() .default(false), diff --git a/server/lib/aiCapabilities.ts b/server/lib/aiCapabilities.ts new file mode 100644 index 000000000..b6d2f7984 --- /dev/null +++ b/server/lib/aiCapabilities.ts @@ -0,0 +1,257 @@ +import type { Request } from "express"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; + +export const AI_CAPABILITIES = [ + "openai_chat", + "openai_responses", + "anthropic_messages", + "gemini_generate_content", + "bedrock_model_invoke", + "google_generate_content", + "google_raw_predict", + "bedrock_converse" +] as const; + +export type AiCapability = (typeof AI_CAPABILITIES)[number]; + +export type AiCapabilityRoute = { + method: "POST"; + path: string; +}; + +export type AiCapabilityDefinition = { + id: AiCapability; + routes: AiCapabilityRoute[]; + extractModel: (req: Request) => string | undefined; + resolveUpstreamUrl: ( + baseUrl: string, + req: Request, + model: string + ) => string; +}; + +function bodyModel(req: Request): string | undefined { + return typeof req.body?.model === "string" ? req.body.model : undefined; +} + +function paramModel(req: Request): string | undefined { + const model = req.params?.model; + return typeof model === "string" && model.length > 0 ? model : undefined; +} + +/** + * Join base URL with a path, avoiding double slashes and a duplicated trailing + * /v1 when the inbound path already starts with /v1 and the base ends with /v1. + */ +export function joinUpstreamUrl(baseUrl: string, path: string): string { + const base = baseUrl.replace(/\/+$/, ""); + let suffix = path.startsWith("/") ? path : `/${path}`; + + if ( + base.endsWith("/v1") && + (suffix === "/v1" || suffix.startsWith("/v1/")) + ) { + suffix = suffix.slice("/v1".length) || "/"; + } + + if (suffix === "/") { + return base; + } + + return `${base}${suffix}`; +} + +function pathFromRequest(req: Request): string { + // Prefer originalUrl path (includes mounted path) over req.path when available. + const raw = + req.originalUrl?.split("?")[0] || req.url?.split("?")[0] || req.path; + return raw.startsWith("/") ? raw : `/${raw}`; +} + +export const AI_CAPABILITY_DEFS: Record = + { + openai_chat: { + id: "openai_chat", + routes: [ + { method: "POST", path: "/v1/chat/completions" }, + { method: "POST", path: "/chat/completions" } + ], + extractModel: bodyModel, + resolveUpstreamUrl: (base, req) => + joinUpstreamUrl(base, pathFromRequest(req)) + }, + openai_responses: { + id: "openai_responses", + routes: [{ method: "POST", path: "/v1/responses" }], + extractModel: bodyModel, + resolveUpstreamUrl: (base, req) => + joinUpstreamUrl(base, pathFromRequest(req)) + }, + anthropic_messages: { + id: "anthropic_messages", + routes: [{ method: "POST", path: "/v1/messages" }], + extractModel: bodyModel, + resolveUpstreamUrl: (base, req) => + joinUpstreamUrl(base, pathFromRequest(req)) + }, + gemini_generate_content: { + id: "gemini_generate_content", + routes: [ + { + method: "POST", + path: "/v1beta/models/:model\\:generateContent" + }, + { + method: "POST", + path: "/v1beta/models/:model\\:streamGenerateContent" + } + ], + extractModel: paramModel, + resolveUpstreamUrl: (base, req) => + joinUpstreamUrl(base, pathFromRequest(req)) + }, + google_generate_content: { + id: "google_generate_content", + routes: [ + { + method: "POST", + // Vertex publisher model generateContent + path: "/v1/projects/:project/locations/:location/publishers/:publisher/models/:model\\:generateContent" + }, + { + method: "POST", + path: "/v1/projects/:project/locations/:location/publishers/:publisher/models/:model\\:streamGenerateContent" + } + ], + extractModel: paramModel, + resolveUpstreamUrl: (base, req) => + joinUpstreamUrl(base, pathFromRequest(req)) + }, + google_raw_predict: { + id: "google_raw_predict", + routes: [ + { + method: "POST", + path: "/v1/projects/:project/locations/:location/publishers/:publisher/models/:model\\:rawPredict" + }, + { + method: "POST", + path: "/v1/projects/:project/locations/:location/publishers/:publisher/models/:model\\:streamRawPredict" + } + ], + extractModel: paramModel, + resolveUpstreamUrl: (base, req) => + joinUpstreamUrl(base, pathFromRequest(req)) + }, + bedrock_model_invoke: { + id: "bedrock_model_invoke", + routes: [ + { method: "POST", path: "/model/:model/invoke" }, + { + method: "POST", + path: "/model/:model/invoke-with-response-stream" + } + ], + extractModel: paramModel, + resolveUpstreamUrl: (base, req) => + joinUpstreamUrl(base, pathFromRequest(req)) + }, + bedrock_converse: { + id: "bedrock_converse", + routes: [ + { method: "POST", path: "/model/:model/converse" }, + { method: "POST", path: "/model/:model/converse-stream" } + ], + extractModel: paramModel, + resolveUpstreamUrl: (base, req) => + joinUpstreamUrl(base, pathFromRequest(req)) + } + }; + +export const AI_PROVIDER_CAPABILITY_DEFAULTS: Record< + Exclude, + readonly AiCapability[] +> = { + openai: ["openai_chat"], + anthropic: ["anthropic_messages"], + googleGemini: ["openai_chat"], + vertexAi: ["google_generate_content"], + bedrock: ["bedrock_converse"], + microsoftFoundry: ["openai_chat"], + openRouter: ["openai_chat"], + vercelAiGateway: ["openai_chat"] +}; + +export function isAiCapability(value: unknown): value is AiCapability { + return ( + typeof value === "string" && + (AI_CAPABILITIES as readonly string[]).includes(value) + ); +} + +export function parseCapabilities(raw: unknown): AiCapability[] { + if (raw == null) { + return []; + } + + let parsed: unknown = raw; + if (typeof raw === "string") { + const trimmed = raw.trim(); + if (!trimmed) { + return []; + } + try { + parsed = JSON.parse(trimmed); + } catch { + return []; + } + } + + if (!Array.isArray(parsed)) { + return []; + } + + const out: AiCapability[] = []; + const seen = new Set(); + for (const item of parsed) { + if (isAiCapability(item) && !seen.has(item)) { + seen.add(item); + out.push(item); + } + } + return out; +} + +export function serializeCapabilities(capabilities: AiCapability[]): string { + return JSON.stringify(capabilities); +} + +export function providerHasCapability( + capabilities: AiCapability[] | string | null | undefined, + capability: AiCapability +): boolean { + const list = + typeof capabilities === "string" || capabilities == null + ? parseCapabilities(capabilities) + : capabilities; + return list.includes(capability); +} + +export function resolveCapabilitiesForCreate(input: { + type: AiProviderType; + capabilities?: AiCapability[] | null; +}): AiCapability[] { + if (input.type === "custom") { + return parseCapabilities(input.capabilities ?? []); + } + return [...AI_PROVIDER_CAPABILITY_DEFAULTS[input.type]]; +} + +export function defaultsForProviderType( + type: AiProviderType +): readonly AiCapability[] { + if (type === "custom") { + return []; + } + return AI_PROVIDER_CAPABILITY_DEFAULTS[type]; +} diff --git a/server/routers/aiGateway/createAiGatewayRouter.ts b/server/routers/aiGateway/createAiGatewayRouter.ts new file mode 100644 index 000000000..cc62ade28 --- /dev/null +++ b/server/routers/aiGateway/createAiGatewayRouter.ts @@ -0,0 +1,21 @@ +import { Router } from "express"; +import { + AI_CAPABILITY_DEFS, + type AiCapability +} from "@server/lib/aiCapabilities"; +import { handleAiGatewayProxy } from "@server/routers/aiGateway/pipeline"; + +export function createAiGatewayRouter() { + const router = Router(); + + for (const def of Object.values(AI_CAPABILITY_DEFS)) { + const capability = def.id as AiCapability; + for (const route of def.routes) { + router.post(route.path, (req, res) => + handleAiGatewayProxy(req, res, capability) + ); + } + } + + return router; +} diff --git a/server/routers/aiGateway/index.ts b/server/routers/aiGateway/index.ts index 691ebebe8..6eea36d60 100644 --- a/server/routers/aiGateway/index.ts +++ b/server/routers/aiGateway/index.ts @@ -1 +1,2 @@ -export * from "./chatCompletions"; +export { handleAiGatewayProxy } from "./pipeline"; +export { createAiGatewayRouter } from "./createAiGatewayRouter"; diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/pipeline.ts similarity index 88% rename from server/routers/aiGateway/chatCompletions.ts rename to server/routers/aiGateway/pipeline.ts index d2671aa63..cf031c381 100644 --- a/server/routers/aiGateway/chatCompletions.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -22,6 +22,11 @@ import { applyAiProviderAuthHeaders, authTypeRequiresApiKey } from "@server/lib/aiProviderDefaults"; +import { + AI_CAPABILITY_DEFS, + providerHasCapability, + type AiCapability +} from "@server/lib/aiCapabilities"; import { SESSION_COOKIE_NAME, validateSessionToken @@ -45,10 +50,6 @@ const REQUEST_USER_TTL_SEC = 30; type CachedClient = { clientId: number; userId: string | null } | null; -// The set of CIDRs an exit node manages; client exitNodeSubnets are always -// /32s carved out of one of these ranges. Checking against this small, -// cacheable list lets us skip the (much more frequent) per-IP client lookup -// entirely for traffic that could never match a client anyway. async function getExitNodeRanges(): Promise { const cached = localCache.get(EXIT_NODE_RANGES_CACHE_KEY); if (cached) { @@ -96,8 +97,6 @@ type ResolvedTarget = { siteResourceId: number | null; orgId: string | null; attachments: ProviderAttachment[]; - // Model IDs on the resource allowlist that belong to allowlist-mode - // providers. Empty when no attached provider uses allowlist mode. allowlistedModelIds: Set; }; @@ -150,13 +149,9 @@ async function buildRequestUser( async function resolveRequestUser( req: Request, - resourceId: number | null, + _resourceId: number | null, orgId: string | null ): Promise { - // Public resources behind badger: badger passes the resource session - // cookie through to the backend (same mechanism the browser gateway, - // e.g. the SSH page, relies on), so we can validate it exactly like - // verifySessionUserMiddleware does for the dashboard. const sessionToken = req.cookies?.[SESSION_COOKIE_NAME]; if (sessionToken) { const { session, user } = await validateSessionToken(sessionToken); @@ -167,10 +162,6 @@ async function resolveRequestUser( // TODO: MAKE SURE THIS CAN NOT BE SPOOFED AND CAN BE TRUSTED AS AN INTERNAL ADDRESS FROM A NODE - // No session cookie - fall back to identifying the caller by source IP. - // A client's exitNodeSubnet is a /32 handed out from one of our exit - // node's address ranges, so an IP that isn't inside any of those ranges - // can never belong to a client and we can skip the DB entirely. const ip = req.ip; if (!ip) { return null; @@ -193,9 +184,6 @@ async function resolveRequestUser( } async function resolveTarget(host: string): Promise { - // TODO: eventually we need to know if it's a private or public resource - // and not just simply check the fullDomain in case there is a private resource with the same fullDomain - const [resourceRow] = await db .select({ resourceId: resources.resourceId, @@ -378,7 +366,6 @@ async function selectProvider( }; } - // One lookup for the requested model key across all attached providers. const matchingModels = await db .select({ modelId: aiModels.modelId, @@ -407,7 +394,6 @@ async function selectProvider( continue; } - // allowlist: only models explicitly attached to the resource if (allowlistedModelIds.has(model.modelId)) { candidates.push(attachment.provider); } @@ -432,11 +418,14 @@ async function selectProvider( }; } -export async function chatCompletions( +export async function handleAiGatewayProxy( req: Request, - res: Response + res: Response, + capability: AiCapability ): Promise { try { + const def = AI_CAPABILITY_DEFS[capability]; + const host = ( (req.headers["p-host"] as string | undefined) || req.headers.host || @@ -448,7 +437,7 @@ export async function chatCompletions( .json({ error: { message: "Missing Host header" } }); } - logger.info(`AI gateway request for host: ${host}`); + logger.info(`AI gateway ${capability} request for host: ${host}`); const target = await resolveTarget(host); if (!target) { @@ -461,11 +450,6 @@ export async function chatCompletions( const { attachments, allowlistedModelIds, resourceId, orgId } = target; - logger.debug("+++++ gateway target: ", target); - - // Best-effort identity resolution - not yet enforced, but lets us - // start making per-user access decisions (e.g. model/role-based - // restrictions) without another round of plumbing later. const requestUser = await resolveRequestUser(req, resourceId, orgId); if (requestUser) { logger.debug( @@ -473,11 +457,22 @@ export async function chatCompletions( ); } - const requestedModel = - typeof req.body?.model === "string" ? req.body.model : undefined; + const capableAttachments = attachments.filter((a) => + providerHasCapability(a.provider.capabilities, capability) + ); + + if (capableAttachments.length === 0) { + return res.status(HttpCode.FORBIDDEN).json({ + error: { + message: `No AI provider on this resource supports ${capability}` + } + }); + } + + const requestedModel = def.extractModel(req); const selection = await selectProvider( - attachments, + capableAttachments, allowlistedModelIds, requestedModel ); @@ -513,11 +508,12 @@ export async function chatCompletions( apiKey = decrypt(provider.apiKey, secret); } - const targetUrl = `${upstreamUrl.replace(/\/$/, "")}`; + const targetUrl = def.resolveUpstreamUrl( + upstreamUrl, + req, + requestedModel! + ); - // Drop hop-by-hop / proxy-only headers. Forwarding Host especially - // breaks Node fetch (TLS/SNI targets the upstream URL while Host - // still says localhost). const skipHeaders = new Set([ "p-host", "host", @@ -554,6 +550,7 @@ export async function chatCompletions( const body = JSON.stringify(req.body); logger.debug("AI gateway upstream request", { + capability, url: targetUrl, method: "POST", headers, @@ -591,7 +588,11 @@ export async function chatCompletions( const contentType = upstreamRes.headers.get("content-type") || ""; const isStream = req.body?.stream === true || - contentType.includes("text/event-stream"); + contentType.includes("text/event-stream") || + req.path.includes("streamGenerateContent") || + req.path.includes("streamRawPredict") || + req.path.includes("converse-stream") || + req.path.includes("invoke-with-response-stream"); res.status(upstreamRes.status); res.setHeader("Content-Type", contentType || "application/json"); diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index 6188601a3..2fb61bf88 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -14,10 +14,15 @@ import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/ import { toPublicAiProvider } from "@server/routers/aiProvider/types"; import { aiAuthTypeSchema, + aiCapabilitiesSchema, aiProviderTypeSchema, aiRoutingModeSchema, refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; +import { + resolveCapabilitiesForCreate, + serializeCapabilities +} from "@server/lib/aiCapabilities"; const paramsSchema = z.strictObject({ orgId: z.string().nonempty() @@ -31,6 +36,7 @@ const bodySchema = z apiKey: z.string().optional(), authType: aiAuthTypeSchema.optional(), routingMode: aiRoutingModeSchema.optional(), + capabilities: aiCapabilitiesSchema.optional(), skipTlsVerification: z.boolean().optional(), enabled: z.boolean().optional() }) @@ -94,6 +100,7 @@ export async function createAiProvider( apiKey, authType, routingMode, + capabilities, skipTlsVerification, enabled } = parsedBody.data; @@ -108,6 +115,10 @@ export async function createAiProvider( authType, routingMode }); + const resolvedCapabilities = resolveCapabilitiesForCreate({ + type, + capabilities + }); const [provider] = await db .insert(aiProviders) @@ -120,6 +131,7 @@ export async function createAiProvider( apiKeyLastChars, authType: resolved.authType, routingMode: resolved.routingMode, + capabilities: serializeCapabilities(resolvedCapabilities), skipTlsVerification: skipTlsVerification ?? false, enabled: enabled ?? true, createdAt: now, diff --git a/server/routers/aiProvider/types.ts b/server/routers/aiProvider/types.ts index 7ac492056..5a1a41249 100644 --- a/server/routers/aiProvider/types.ts +++ b/server/routers/aiProvider/types.ts @@ -1,12 +1,17 @@ import type { AiModel, AiProvider } from "@server/db"; import type { PaginatedResponse } from "@server/types/Pagination"; import type { AiProviderAuthType } from "@server/lib/aiProviderDefaults"; +import { + parseCapabilities, + type AiCapability +} from "@server/lib/aiCapabilities"; import { decrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; -export type AiProviderPublic = Omit & { +export type AiProviderPublic = Omit & { /** Decrypted API key. Only included on get/create/update of a single provider. */ apiKey?: string | null; + capabilities: AiCapability[]; effectiveUpstreamUrl: string | null; effectiveAuthType: AiProviderAuthType; }; @@ -39,7 +44,11 @@ export function toPublicAiProvider( provider: AiProvider, options?: { includeApiKey?: boolean } ): AiProviderPublic { - const { apiKey: encryptedApiKey, ...rest } = provider; + const { + apiKey: encryptedApiKey, + capabilities: rawCapabilities, + ...rest + } = provider; let apiKey: string | null | undefined; if (options?.includeApiKey) { @@ -56,6 +65,7 @@ export function toPublicAiProvider( return { ...rest, ...(options?.includeApiKey ? { apiKey } : {}), + capabilities: parseCapabilities(rawCapabilities), effectiveUpstreamUrl: provider.upstreamUrl, effectiveAuthType: provider.authType as AiProviderAuthType }; diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts index 82224c28b..9dc28be60 100644 --- a/server/routers/aiProvider/updateAiProvider.ts +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -14,6 +14,7 @@ import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/ import { toPublicAiProvider } from "@server/routers/aiProvider/types"; import { aiAuthTypeSchema, + aiCapabilitiesSchema, aiProviderTypeSchema, aiRoutingModeSchema, refineProviderUpstreamFields @@ -23,6 +24,10 @@ import type { AiProviderRoutingMode, AiProviderType } from "@server/lib/aiProviderDefaults"; +import { + parseCapabilities, + serializeCapabilities +} from "@server/lib/aiCapabilities"; const paramsSchema = z.strictObject({ providerId: z.coerce.number().int().positive() @@ -34,6 +39,7 @@ const bodySchema = z.strictObject({ apiKey: z.string().optional(), authType: aiAuthTypeSchema.optional(), routingMode: aiRoutingModeSchema.optional(), + capabilities: aiCapabilitiesSchema.optional(), skipTlsVerification: z.boolean().optional(), enabled: z.boolean().optional() }); @@ -122,19 +128,37 @@ export async function updateAiProvider( ? body.authType : (existing.authType as AiProviderAuthType); + if (body.capabilities !== undefined && providerType !== "custom") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Capabilities can only be updated for custom providers" + ) + ); + } + + const nextCapabilities = + providerType === "custom" + ? body.capabilities !== undefined + ? body.capabilities + : parseCapabilities(existing.capabilities) + : parseCapabilities(existing.capabilities); + const validation = z .object({ type: aiProviderTypeSchema, upstreamUrl: z.string().nullable().optional(), authType: aiAuthTypeSchema, - routingMode: aiRoutingModeSchema.optional() + routingMode: aiRoutingModeSchema.optional(), + capabilities: aiCapabilitiesSchema.optional() }) .superRefine((data, ctx) => refineProviderUpstreamFields(data, ctx)) .safeParse({ type: providerType, upstreamUrl: nextUpstreamUrl, authType: nextAuthType, - routingMode: nextRoutingMode + routingMode: nextRoutingMode, + capabilities: nextCapabilities }); if (!validation.success) { @@ -168,6 +192,9 @@ export async function updateAiProvider( if (body.authType !== undefined) { updateData.authType = body.authType; } + if (providerType === "custom" && body.capabilities !== undefined) { + updateData.capabilities = serializeCapabilities(body.capabilities); + } if (body.apiKey !== undefined) { const key = config.getRawConfig().server.secret!; diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts index b4ab104aa..957a3914f 100644 --- a/server/routers/aiProvider/validation.ts +++ b/server/routers/aiProvider/validation.ts @@ -6,6 +6,7 @@ import { type AiProviderRoutingMode, type AiProviderType } from "@server/lib/aiProviderDefaults"; +import { AI_CAPABILITIES } from "@server/lib/aiCapabilities"; export const aiProviderTypeSchema = z.enum([ "openai", @@ -23,12 +24,17 @@ export const aiAuthTypeSchema = z.enum(AI_PROVIDER_AUTH_TYPES); export const aiRoutingModeSchema = z.enum(["url", "target"]); +export const aiCapabilitySchema = z.enum(AI_CAPABILITIES); + +export const aiCapabilitiesSchema = z.array(aiCapabilitySchema); + export function refineProviderUpstreamFields( data: { type: AiProviderType; upstreamUrl?: string | null; authType?: AiProviderAuthType | null; routingMode?: AiProviderRoutingMode | null; + capabilities?: z.infer | null; }, ctx: z.RefinementCtx ) { @@ -52,4 +58,16 @@ export function refineProviderUpstreamFields( path: ["upstreamUrl"] }); } + + if (data.type === "custom") { + const caps = data.capabilities; + if (!caps || caps.length === 0) { + ctx.addIssue({ + code: "custom", + message: + "At least one capability is required for custom providers", + path: ["capabilities"] + }); + } + } } diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx index 1018f8e63..62ffa6eb6 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx @@ -12,11 +12,16 @@ import { SettingsSectionHeader, SettingsSectionTitle } from "@app/components/Settings"; +import { + AiProviderCapabilitiesSelect, + capabilityLabelKey +} from "@app/components/AiProviderCapabilitiesSelect"; import { SwitchInput } from "@app/components/SwitchInput"; import { Button } from "@app/components/ui/button"; import { Form, FormControl, + FormDescription, FormField, FormItem, FormLabel, @@ -28,6 +33,7 @@ import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { zodResolver } from "@hookform/resolvers/zod"; +import { AI_CAPABILITIES, type AiCapability } from "@server/lib/aiCapabilities"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; @@ -43,17 +49,32 @@ export default function AiProviderGeneralPage() { const router = useRouter(); const t = useTranslations(); const [saveLoading, setSaveLoading] = useState(false); + const isCustom = provider.type === "custom"; const generalSchema = useMemo( () => - z.object({ - name: z - .string() - .trim() - .min(1, { message: t("nameRequired") }), - enabled: z.boolean() - }), - [t] + z + .object({ + name: z + .string() + .trim() + .min(1, { message: t("nameRequired") }), + enabled: z.boolean(), + capabilities: z.array(z.enum(AI_CAPABILITIES)).optional() + }) + .superRefine((data, ctx) => { + if ( + isCustom && + (!data.capabilities || data.capabilities.length === 0) + ) { + ctx.addIssue({ + code: "custom", + message: t("aiProviderErrorCapabilitiesRequired"), + path: ["capabilities"] + }); + } + }), + [t, isCustom] ); type GeneralFormValues = z.infer; @@ -62,24 +83,35 @@ export default function AiProviderGeneralPage() { resolver: zodResolver(generalSchema), defaultValues: { name: provider.name, - enabled: provider.enabled + enabled: provider.enabled, + capabilities: provider.capabilities ?? [] } }); async function onSubmit(values: GeneralFormValues) { setSaveLoading(true); try { - const res = await api.post< - AxiosResponse - >(`/ai-provider/${provider.providerId}`, { + const body: { + name: string; + enabled: boolean; + capabilities?: AiCapability[]; + } = { name: values.name.trim(), enabled: values.enabled - }); + }; + if (isCustom) { + body.capabilities = values.capabilities ?? []; + } + + const res = await api.post< + AxiosResponse + >(`/ai-provider/${provider.providerId}`, body); const updated = res.data.data.provider; updateProvider(updated); form.reset({ name: updated.name, - enabled: updated.enabled + enabled: updated.enabled, + capabilities: updated.capabilities ?? [] }); toast({ title: t("success"), @@ -166,6 +198,65 @@ export default function AiProviderGeneralPage() { )} /> + + + ( + + + {t( + "aiProviderCapabilities" + )} + + + {isCustom ? ( + + ) : ( +
+ {( + provider.capabilities ?? + [] + ).map((cap) => ( + + {t( + capabilityLabelKey( + cap + ) + )} + + ))} +
+ )} + + + {isCustom + ? t( + "aiProviderCapabilitiesCustomDescription" + ) + : t( + "aiProviderCapabilitiesDescription" + )} + + + + )} + /> + diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 1947c3f85..34fac7b55 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -20,6 +20,10 @@ import { } from "@app/components/Settings"; import HeaderTitle from "@app/components/SettingsSectionTitle"; import { AiProviderAuthTypeSelect } from "@app/components/AiProviderAuthTypeSelect"; +import { + AiProviderCapabilitiesSelect, + capabilityLabelKey +} from "@app/components/AiProviderCapabilitiesSelect"; import { AiProviderTypeSelect } from "@app/components/AiProviderTypeSelect"; import { StrategySelect } from "@app/components/StrategySelect"; import { SwitchInput } from "@app/components/SwitchInput"; @@ -40,6 +44,7 @@ import { createApiClient, formatAxiosError } from "@app/lib/api"; import { createAiProviderCreateFormSchema, defaultAuthTypeForProvider, + defaultCapabilitiesForProvider, emptyUpstreamForType, showsUpstreamUrlField, toAiProviderCreatePayload, @@ -76,6 +81,7 @@ export default function CreateAiProviderPage() { apiKey: "", authType: defaultAuthTypeForProvider("openai"), routingMode: "url", + capabilities: defaultCapabilitiesForProvider("openai"), skipTlsVerification: false, enabled: true } @@ -84,12 +90,14 @@ export default function CreateAiProviderPage() { const providerType = form.watch("type"); const routingMode = form.watch("routingMode"); const authType = form.watch("authType"); + const capabilities = form.watch("capabilities"); const showUpstream = showsUpstreamUrlField(providerType, routingMode); const requireUpstream = upstreamUrlRequired(providerType, routingMode); const showRoutingMode = providerType === "custom"; const showTargets = providerType === "custom" && routingMode === "target"; const showApiKey = authTypeRequiresApiKey(authType ?? "bearer"); + const showCapabilitiesSelect = providerType === "custom"; async function createTargets( providerId: number, @@ -277,6 +285,12 @@ export default function CreateAiProviderPage() { value ) ); + form.setValue( + "capabilities", + defaultCapabilitiesForProvider( + value + ) + ); if ( value !== "custom" @@ -296,6 +310,67 @@ export default function CreateAiProviderPage() { )} /> + + + ( + + + {t( + "aiProviderCapabilities" + )} + + + {showCapabilitiesSelect ? ( + + ) : ( +
+ {( + capabilities ?? + defaultCapabilitiesForProvider( + providerType + ) + ).map((cap) => ( + + {t( + capabilityLabelKey( + cap + ) + )} + + ))} +
+ )} + + + {showCapabilitiesSelect + ? t( + "aiProviderCapabilitiesCustomDescription" + ) + : t( + "aiProviderCapabilitiesDescription" + )} + + + + )} + /> + diff --git a/src/components/AiProviderCapabilitiesSelect.tsx b/src/components/AiProviderCapabilitiesSelect.tsx new file mode 100644 index 000000000..086b6f03e --- /dev/null +++ b/src/components/AiProviderCapabilitiesSelect.tsx @@ -0,0 +1,89 @@ +"use client"; + +import { MultiSelectTagInput } from "@app/components/multi-select/multi-select-tag-input"; +import { AI_CAPABILITIES, type AiCapability } from "@server/lib/aiCapabilities"; +import { useTranslations } from "next-intl"; +import { useMemo, useState } from "react"; + +export type CapabilityOption = { + id: string; + text: string; +}; + +export type AiProviderCapabilitiesSelectProps = { + value: AiCapability[]; + onChange: (capabilities: AiCapability[]) => void; + disabled?: boolean; +}; + +const CAPABILITY_LABEL_KEYS: Record = { + openai_chat: "aiCapabilityOpenaiChat", + openai_responses: "aiCapabilityOpenaiResponses", + anthropic_messages: "aiCapabilityAnthropicMessages", + gemini_generate_content: "aiCapabilityGeminiGenerateContent", + bedrock_model_invoke: "aiCapabilityBedrockModelInvoke", + google_generate_content: "aiCapabilityGoogleGenerateContent", + google_raw_predict: "aiCapabilityGoogleRawPredict", + bedrock_converse: "aiCapabilityBedrockConverse" +}; + +export function capabilityLabelKey(capability: AiCapability): string { + return CAPABILITY_LABEL_KEYS[capability]; +} + +export function AiProviderCapabilitiesSelect({ + value, + onChange, + disabled +}: AiProviderCapabilitiesSelectProps) { + const t = useTranslations(); + const [searchQuery, setSearchQuery] = useState(""); + + const options: CapabilityOption[] = useMemo( + () => + AI_CAPABILITIES.map((id) => ({ + id, + text: t(CAPABILITY_LABEL_KEYS[id]) + })), + [t] + ); + + const filtered = useMemo(() => { + const q = searchQuery.trim().toLowerCase(); + if (!q) { + return options; + } + return options.filter( + (o) => + o.text.toLowerCase().includes(q) || + o.id.toLowerCase().includes(q) + ); + }, [options, searchQuery]); + + const selected: CapabilityOption[] = value.map((id) => ({ + id, + text: t(CAPABILITY_LABEL_KEYS[id]) + })); + + return ( + + onChange( + next + .map((item) => item.id) + .filter((id): id is AiCapability => + (AI_CAPABILITIES as readonly string[]).includes(id) + ) + ) + } + onSearch={setSearchQuery} + disabled={disabled} + /> + ); +} diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index f484d1f05..b01728478 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -7,6 +7,11 @@ import { type AiProviderAuthType, type AiProviderType } from "@server/lib/aiProviderDefaults"; +import { + AI_CAPABILITIES, + defaultsForProviderType, + type AiCapability +} from "@server/lib/aiCapabilities"; type TranslateFn = (key: string) => string; @@ -22,6 +27,8 @@ export const aiProviderTypeValues = [ "custom" ] as const satisfies readonly AiProviderType[]; +export const aiCapabilityValues = AI_CAPABILITIES; + export function createAiProviderFormSchema(t: TranslateFn) { return z .object({ @@ -34,6 +41,7 @@ export function createAiProviderFormSchema(t: TranslateFn) { apiKey: z.string().optional(), authType: z.enum(AI_PROVIDER_AUTH_TYPES).optional().nullable(), routingMode: z.enum(["url", "target"]).optional(), + capabilities: z.array(z.enum(AI_CAPABILITIES)).optional(), skipTlsVerification: z.boolean().optional(), enabled: z.boolean().optional() }) @@ -84,6 +92,17 @@ export function createAiProviderFormSchema(t: TranslateFn) { path: ["authType"] }); } + + if ( + data.type === "custom" && + (!data.capabilities || data.capabilities.length === 0) + ) { + ctx.addIssue({ + code: "custom", + message: t("aiProviderErrorCapabilitiesRequired"), + path: ["capabilities"] + }); + } }); } @@ -114,6 +133,12 @@ export function defaultAuthTypeForProvider( return AI_PROVIDER_DEFAULTS[type].authType; } +export function defaultCapabilitiesForProvider( + type: AiProviderType +): AiCapability[] { + return [...defaultsForProviderType(type)]; +} + export function emptyUpstreamForType(type: AiProviderType): string { if (type === "custom") { return ""; @@ -158,6 +183,8 @@ export function toAiProviderCreatePayload(values: AiProviderFormValues) { upstreamUrl, apiKey: values.apiKey?.trim() ? values.apiKey.trim() : undefined, authType: values.authType ?? "bearer", + capabilities: + values.type === "custom" ? (values.capabilities ?? []) : undefined, skipTlsVerification: values.skipTlsVerification, enabled: values.enabled ?? true }; @@ -183,6 +210,10 @@ export function toAiProviderUpdatePayload(values: AiProviderFormValues) { enabled: values.enabled ?? true }; + if (values.type === "custom" && values.capabilities) { + payload.capabilities = values.capabilities; + } + if (values.apiKey?.trim()) { payload.apiKey = values.apiKey.trim(); } From 3d7e322bf97cd5771c515a8b8ef9815cce7c9daf Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 5 Aug 2026 17:42:58 -0400 Subject: [PATCH 058/296] add headers to provider --- cli/commands/rotateServerSecret.ts | 55 ++++++++++++++++++- messages/en-US.json | 1 + server/db/pg/schema/schema.ts | 1 + server/db/sqlite/schema/schema.ts | 1 + server/lib/aiProviderDefaults.ts | 49 +++++++++++++++++ server/routers/aiGateway/pipeline.ts | 6 ++ server/routers/aiProvider/createAiProvider.ts | 5 ++ server/routers/aiProvider/types.ts | 20 ++++++- server/routers/aiProvider/updateAiProvider.ts | 16 ++++-- server/routers/aiProvider/validation.ts | 43 +++++++++++++++ .../[providerId]/network/page.tsx | 38 ++++++++++++- .../settings/ai-providers/create/page.tsx | 37 ++++++++++++- src/lib/aiProviderFormSchema.ts | 10 +++- 13 files changed, 271 insertions(+), 11 deletions(-) diff --git a/cli/commands/rotateServerSecret.ts b/cli/commands/rotateServerSecret.ts index afac262b2..2edb7592d 100644 --- a/cli/commands/rotateServerSecret.ts +++ b/cli/commands/rotateServerSecret.ts @@ -1,5 +1,5 @@ import { CommandModule } from "yargs"; -import { db, idpOidcConfig, licenseKey, certificates, eventStreamingDestinations, alertWebhookActions } from "@server/db"; +import { db, idpOidcConfig, licenseKey, certificates, eventStreamingDestinations, alertWebhookActions, aiProviders } from "@server/db"; import { encrypt, decrypt } from "@server/lib/crypto"; import { configFilePath1, configFilePath2 } from "@server/lib/consts"; import { eq } from "drizzle-orm"; @@ -132,12 +132,14 @@ export const rotateServerSecret: CommandModule< const certs = await db.select().from(certificates); const streamingDestinations = await db.select().from(eventStreamingDestinations); const webhookActions = await db.select().from(alertWebhookActions); + const providers = await db.select().from(aiProviders); console.log(`Found ${idpConfigs.length} OIDC IdP configuration(s)`); console.log(`Found ${licenseKeys.length} license key(s)`); console.log(`Found ${certs.length} certificate(s)`); console.log(`Found ${streamingDestinations.length} event streaming destination(s)`); console.log(`Found ${webhookActions.length} alert webhook action(s)`); + console.log(`Found ${providers.length} AI provider(s)`); // Prepare all decrypted and re-encrypted values console.log("\nDecrypting and re-encrypting values..."); @@ -171,11 +173,18 @@ export const rotateServerSecret: CommandModule< encryptedConfig: string; }; + type AiProviderUpdate = { + providerId: number; + encryptedApiKey: string | null; + encryptedHeaders: string | null; + }; + const idpUpdates: IdpUpdate[] = []; const licenseKeyUpdates: LicenseKeyUpdate[] = []; const certUpdates: CertUpdate[] = []; const streamingDestinationUpdates: StreamingDestinationUpdate[] = []; const webhookActionUpdates: WebhookActionUpdate[] = []; + const aiProviderUpdates: AiProviderUpdate[] = []; // Process idpOidcConfig entries for (const idpConfig of idpConfigs) { @@ -306,6 +315,37 @@ export const rotateServerSecret: CommandModule< } } + // Process aiProviders entries (apiKey + headers) + for (const provider of providers) { + try { + if (!provider.apiKey && !provider.headers) { + continue; + } + + const encryptedApiKey = provider.apiKey + ? encrypt(decrypt(provider.apiKey, oldSecret), newSecret) + : null; + const encryptedHeaders = provider.headers + ? encrypt( + decrypt(provider.headers, oldSecret), + newSecret + ) + : null; + + aiProviderUpdates.push({ + providerId: provider.providerId, + encryptedApiKey, + encryptedHeaders + }); + } catch (error) { + console.error( + `Error processing AI provider ${provider.providerId}:`, + error + ); + throw error; + } + } + // Perform all database updates in a single transaction console.log("\nUpdating database in transaction..."); await db.transaction(async (trx) => { @@ -376,6 +416,17 @@ export const rotateServerSecret: CommandModule< ) ); } + + // Update AI provider entries + for (const update of aiProviderUpdates) { + await trx + .update(aiProviders) + .set({ + apiKey: update.encryptedApiKey, + headers: update.encryptedHeaders + }) + .where(eq(aiProviders.providerId, update.providerId)); + } }); console.log(`Rotated ${idpUpdates.length} OIDC IdP configuration(s)`); @@ -383,6 +434,7 @@ export const rotateServerSecret: CommandModule< console.log(`Rotated ${certUpdates.length} certificate(s)`); console.log(`Rotated ${streamingDestinationUpdates.length} event streaming destination(s)`); console.log(`Rotated ${webhookActionUpdates.length} alert webhook action(s)`); + console.log(`Rotated ${aiProviderUpdates.length} AI provider(s)`); // Update config file with new secret console.log("\nUpdating config file..."); @@ -402,6 +454,7 @@ export const rotateServerSecret: CommandModule< console.log(` - Certificates: ${certUpdates.length}`); console.log(` - Event streaming destinations: ${streamingDestinationUpdates.length}`); console.log(` - Alert webhook actions: ${webhookActionUpdates.length}`); + console.log(` - AI providers: ${aiProviderUpdates.length}`); console.log( `\n IMPORTANT: Restart the server for the new secret to take effect.` ); diff --git a/messages/en-US.json b/messages/en-US.json index ac6131abc..a10a45421 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1680,6 +1680,7 @@ "aiProviderEffectiveUpstreamUrl": "Effective Upstream URL", "aiProviderApiKey": "API Key", "aiProviderApiKeyDescription": "API key used to authenticate requests to this provider", + "aiProviderCustomHeadersDescription": "Headers sent on every request to this provider. Newline separated: Header-Name: value", "aiProviderApiKeyLastChars": "API Key", "aiProviderAuthType": "Auth Type", "aiProviderAuthTypeSearch": "Search auth types...", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 3091ed618..6bc038660 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1660,6 +1660,7 @@ export const aiProviders = pgTable("aiProviders", { .notNull() .default("url"), capabilities: text("capabilities").notNull().default("[]"), + headers: text("headers"), // JSON array of { name, value } skipTlsVerification: boolean("skipTlsVerification") .notNull() .default(false), diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 5c115cb2e..8130af9ea 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1642,6 +1642,7 @@ export const aiProviders = sqliteTable("aiProviders", { .notNull() .default("url"), capabilities: text("capabilities").notNull().default("[]"), + headers: text("headers"), // JSON array of { name, value } skipTlsVerification: integer("skipTlsVerification", { mode: "boolean" }) .notNull() .default(false), diff --git a/server/lib/aiProviderDefaults.ts b/server/lib/aiProviderDefaults.ts index 70b666954..e48c6f877 100644 --- a/server/lib/aiProviderDefaults.ts +++ b/server/lib/aiProviderDefaults.ts @@ -1,3 +1,5 @@ +import { decrypt, encrypt } from "@server/lib/crypto"; + export type AiProviderType = | "openai" | "anthropic" @@ -127,6 +129,53 @@ export function resolveAiProviderCreateFields(input: { }; } +export type AiProviderHeader = { name: string; value: string }; + +export function serializeAiProviderHeaders( + headers: AiProviderHeader[] | null | undefined, + secret: string +): string | null { + if (!headers || headers.length === 0) { + return null; + } + return encrypt(JSON.stringify(headers), secret); +} + +export function parseAiProviderHeaders( + raw: string | null | undefined, + secret: string +): AiProviderHeader[] { + if (!raw) { + return []; + } + try { + const decrypted = decrypt(raw, secret); + const parsed = JSON.parse(decrypted); + if (!Array.isArray(parsed)) { + return []; + } + return parsed.filter( + (h): h is AiProviderHeader => + h != null && + typeof h === "object" && + typeof h.name === "string" && + typeof h.value === "string" + ); + } catch { + return []; + } +} + +export function applyAiProviderCustomHeaders( + headers: Record, + raw: string | null | undefined, + secret: string +): void { + for (const { name, value } of parseAiProviderHeaders(raw, secret)) { + headers[name] = value; + } +} + /** * Apply provider auth to upstream headers. * - Injected modes: strip client auth headers, then set the provider key. diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index cf031c381..44e2aef66 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -20,6 +20,7 @@ import { decrypt } from "@server/lib/crypto"; import { AiProviderAuthType, applyAiProviderAuthHeaders, + applyAiProviderCustomHeaders, authTypeRequiresApiKey } from "@server/lib/aiProviderDefaults"; import { @@ -536,6 +537,11 @@ export async function handleAiGatewayProxy( } headers[key] = Array.isArray(value) ? value.join(", ") : value; } + applyAiProviderCustomHeaders( + headers, + provider.headers, + config.getRawConfig().server.secret! + ); applyAiProviderAuthHeaders(headers, authType, apiKey); // No dedicated per-request TLS agent is wired up (no extra deps for diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index 2fb61bf88..995e9eaa0 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -15,10 +15,12 @@ import { toPublicAiProvider } from "@server/routers/aiProvider/types"; import { aiAuthTypeSchema, aiCapabilitiesSchema, + aiProviderHeadersSchema, aiProviderTypeSchema, aiRoutingModeSchema, refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; +import { serializeAiProviderHeaders } from "@server/lib/aiProviderDefaults"; import { resolveCapabilitiesForCreate, serializeCapabilities @@ -37,6 +39,7 @@ const bodySchema = z authType: aiAuthTypeSchema.optional(), routingMode: aiRoutingModeSchema.optional(), capabilities: aiCapabilitiesSchema.optional(), + headers: aiProviderHeadersSchema, skipTlsVerification: z.boolean().optional(), enabled: z.boolean().optional() }) @@ -101,6 +104,7 @@ export async function createAiProvider( authType, routingMode, capabilities, + headers, skipTlsVerification, enabled } = parsedBody.data; @@ -132,6 +136,7 @@ export async function createAiProvider( authType: resolved.authType, routingMode: resolved.routingMode, capabilities: serializeCapabilities(resolvedCapabilities), + headers: serializeAiProviderHeaders(headers, key), skipTlsVerification: skipTlsVerification ?? false, enabled: enabled ?? true, createdAt: now, diff --git a/server/routers/aiProvider/types.ts b/server/routers/aiProvider/types.ts index 5a1a41249..d2f314b30 100644 --- a/server/routers/aiProvider/types.ts +++ b/server/routers/aiProvider/types.ts @@ -1,6 +1,10 @@ import type { AiModel, AiProvider } from "@server/db"; import type { PaginatedResponse } from "@server/types/Pagination"; -import type { AiProviderAuthType } from "@server/lib/aiProviderDefaults"; +import { + parseAiProviderHeaders, + type AiProviderAuthType, + type AiProviderHeader +} from "@server/lib/aiProviderDefaults"; import { parseCapabilities, type AiCapability @@ -8,10 +12,13 @@ import { import { decrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; -export type AiProviderPublic = Omit & { - /** Decrypted API key. Only included on get/create/update of a single provider. */ +export type AiProviderPublic = Omit< + AiProvider, + "apiKey" | "capabilities" | "headers" +> & { apiKey?: string | null; capabilities: AiCapability[]; + headers: AiProviderHeader[] | null; effectiveUpstreamUrl: string | null; effectiveAuthType: AiProviderAuthType; }; @@ -47,6 +54,7 @@ export function toPublicAiProvider( const { apiKey: encryptedApiKey, capabilities: rawCapabilities, + headers: rawHeaders, ...rest } = provider; @@ -62,10 +70,16 @@ export function toPublicAiProvider( } } + const parsedHeaders = parseAiProviderHeaders( + rawHeaders, + config.getRawConfig().server.secret! + ); + return { ...rest, ...(options?.includeApiKey ? { apiKey } : {}), capabilities: parseCapabilities(rawCapabilities), + headers: parsedHeaders.length > 0 ? parsedHeaders : null, effectiveUpstreamUrl: provider.upstreamUrl, effectiveAuthType: provider.authType as AiProviderAuthType }; diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts index 9dc28be60..7cbf1145a 100644 --- a/server/routers/aiProvider/updateAiProvider.ts +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -15,14 +15,16 @@ import { toPublicAiProvider } from "@server/routers/aiProvider/types"; import { aiAuthTypeSchema, aiCapabilitiesSchema, + aiProviderHeadersSchema, aiProviderTypeSchema, aiRoutingModeSchema, refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; -import type { - AiProviderAuthType, - AiProviderRoutingMode, - AiProviderType +import { + serializeAiProviderHeaders, + type AiProviderAuthType, + type AiProviderRoutingMode, + type AiProviderType } from "@server/lib/aiProviderDefaults"; import { parseCapabilities, @@ -40,6 +42,7 @@ const bodySchema = z.strictObject({ authType: aiAuthTypeSchema.optional(), routingMode: aiRoutingModeSchema.optional(), capabilities: aiCapabilitiesSchema.optional(), + headers: aiProviderHeadersSchema, skipTlsVerification: z.boolean().optional(), enabled: z.boolean().optional() }); @@ -202,6 +205,11 @@ export async function updateAiProvider( updateData.apiKeyLastChars = body.apiKey.slice(-4); } + if (body.headers !== undefined) { + const key = config.getRawConfig().server.secret!; + updateData.headers = serializeAiProviderHeaders(body.headers, key); + } + const [provider] = await db .update(aiProviders) .set(updateData) diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts index 957a3914f..a56ccad1c 100644 --- a/server/routers/aiProvider/validation.ts +++ b/server/routers/aiProvider/validation.ts @@ -28,6 +28,49 @@ export const aiCapabilitySchema = z.enum(AI_CAPABILITIES); export const aiCapabilitiesSchema = z.array(aiCapabilitySchema); +const validHeaderName = /^[a-zA-Z0-9!#$%&'*+\-.^_`|~]+$/; +const validHeaderValue = /^[\t\x20-\x7E]*$/; +const templatePattern = /\{\{[^}]+\}\}/; + +export const aiProviderHeadersSchema = z + .array(z.strictObject({ name: z.string(), value: z.string() })) + .nullable() + .optional() + .superRefine((headers, ctx) => { + if (!headers) { + return; + } + for (const [index, header] of headers.entries()) { + if (!validHeaderName.test(header.name)) { + ctx.addIssue({ + code: "custom", + message: + "Header names may only contain valid HTTP token characters (letters, digits, and !#$%&'*+-.^_`|~).", + path: [index, "name"] + }); + } + if (!validHeaderValue.test(header.value)) { + ctx.addIssue({ + code: "custom", + message: + "Header values may only contain printable ASCII characters and horizontal whitespace.", + path: [index, "value"] + }); + } + if ( + templatePattern.test(header.name) || + templatePattern.test(header.value) + ) { + ctx.addIssue({ + code: "custom", + message: + "Header names and values must not contain template expressions such as {{value}}.", + path: [index] + }); + } + } + }); + export function refineProviderUpstreamFields( data: { type: AiProviderType; diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx index 49f251cfa..8b8814303 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx @@ -21,6 +21,7 @@ import { } from "@app/components/Settings"; import { StrategySelect } from "@app/components/StrategySelect"; import { SwitchInput } from "@app/components/SwitchInput"; +import { HeadersInput } from "@app/components/HeadersInput"; import { Button } from "@app/components/ui/button"; import { Form, @@ -66,6 +67,7 @@ export default function AiProviderNetworkPage() { const router = useRouter(); const t = useTranslations(); const [saveLoading, setSaveLoading] = useState(false); + const [headersValid, setHeadersValid] = useState(true); const targetsFormRef = useRef(null); const formSchema = useMemo(() => createAiProviderFormSchema(t), [t]); @@ -79,6 +81,7 @@ export default function AiProviderNetworkPage() { apiKey: "", authType: (provider.authType as AiProviderAuthType) ?? "bearer", routingMode: (provider.routingMode as "url" | "target") ?? "url", + headers: provider.headers ?? [], skipTlsVerification: provider.skipTlsVerification, enabled: provider.enabled } @@ -122,6 +125,7 @@ export default function AiProviderNetworkPage() { apiKey: "", authType: (updated.authType as AiProviderAuthType) ?? "bearer", routingMode: (updated.routingMode as "url" | "target") ?? "url", + headers: updated.headers ?? [], skipTlsVerification: updated.skipTlsVerification, enabled: updated.enabled }); @@ -310,6 +314,38 @@ export default function AiProviderNetworkPage() { /> )} + + + ( + + + {t("customHeaders")} + + + + + + {t( + "aiProviderCustomHeadersDescription" + )} + + + + )} + /> + @@ -346,7 +382,7 @@ export default function AiProviderNetworkPage() {
)} diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 457702c38..0d9541e54 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -591,6 +591,7 @@ export default function CreateAiProviderPage() { )} embedded hideSaveButton + disableAdvancedMode />
)} diff --git a/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx b/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx index bc2f985e5..1440e6c02 100644 --- a/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx +++ b/src/app/[orgId]/settings/resources/public/ProxyResourceTargetsForm.tsx @@ -104,6 +104,8 @@ type ProxyResourceTargetsFormProps = { embedded?: boolean; /** Hide the built-in save button (use ref.save from parent) */ hideSaveButton?: boolean; + /** Hide the advanced mode toggle and always use non-advanced mode (e.g. AI providers) */ + disableAdvancedMode?: boolean; }; export const ProxyResourceTargetsForm = forwardRef< @@ -121,7 +123,8 @@ export const ProxyResourceTargetsForm = forwardRef< allowedMethods = ["http", "https", "h2c"], emptyMessage, embedded = false, - hideSaveButton = false + hideSaveButton = false, + disableAdvancedMode = false }, ref ) { @@ -221,6 +224,9 @@ export const ProxyResourceTargetsForm = forwardRef< ); const [isAdvancedMode, setIsAdvancedMode] = useState(() => { + if (disableAdvancedMode) { + return false; + } if (typeof window !== "undefined") { const saved = localStorage.getItem("proxy-advanced-mode"); return saved === "true"; @@ -709,13 +715,14 @@ export const ProxyResourceTargetsForm = forwardRef< }, [sites]); useEffect(() => { + if (disableAdvancedMode) return; if (typeof window !== "undefined") { localStorage.setItem( "proxy-advanced-mode", isAdvancedMode.toString() ); } - }, [isAdvancedMode]); + }, [isAdvancedMode, disableAdvancedMode]); const [, formAction, isSubmitting] = useActionState( async () => { @@ -940,19 +947,21 @@ export const ProxyResourceTargetsForm = forwardRef<
{addTargetButton} -
- - -
+ {!disableAdvancedMode && ( +
+ + +
+ )}
)} From fb896d6c0f0a9262c87c58d9bfefb053515723f5 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 12:03:40 -0400 Subject: [PATCH 063/296] Add domain to the resource selection --- messages/en-US.json | 2 + .../siteResource/createSiteResource.ts | 20 +- .../siteResource/updateSiteResource.ts | 18 +- .../private/[niceId]/general/page.tsx | 30 +- .../private/[niceId]/inference/page.tsx | 325 +++++++++++++++++- .../resources/private/[niceId]/layout.tsx | 45 +-- .../private/[niceId]/providers/page.tsx | 230 ------------- .../public/[niceId]/inference/page.tsx | 237 ++++++++++++- .../resources/public/[niceId]/layout.tsx | 15 +- .../public/[niceId]/providers/page.tsx | 230 ------------- src/lib/privateResourceForm.ts | 8 +- 11 files changed, 590 insertions(+), 570 deletions(-) delete mode 100644 src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx delete mode 100644 src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx diff --git a/messages/en-US.json b/messages/en-US.json index a10a45421..f5de37878 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1771,6 +1771,8 @@ "aiResourceProvidersUpdated": "Providers updated", "aiResourceProvidersErrorUpdate": "Failed to update providers", "aiResourceAliasRequired": "Alias is required for inference resources", + "aiResourceDomainConfiguration": "Domain configuration", + "aiResourceDomainConfigurationDescription": "Choose the domain clients will use to reach this inference resource.", "sidebarApiKeys": "API Keys", "sidebarProvisioning": "Provisioning", "sidebarSettings": "Settings", diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index feeca0938..815d7adb8 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -168,7 +168,7 @@ const createSiteResourceSchema = z ) .refine( (data) => { - // destination is only optional for ssh mode with native authDaemonMode + // destination is only optional for ssh mode with native authDaemonMode or inference if ( (data.mode === "ssh" && data.authDaemonMode === "native") || data.mode == "inference" @@ -182,7 +182,7 @@ const createSiteResourceSchema = z }, { message: - "Destination is required unless mode is ssh with authDaemonMode native" + "Destination is required unless mode is ssh with authDaemonMode native or inference" } ) .refine( @@ -509,7 +509,15 @@ export async function createSiteResource( const existingResource = await db .select() .from(siteResources) - .where(eq(siteResources.fullDomain, fullDomain)); + .where( + and( + eq(siteResources.fullDomain, fullDomain), + ne( + siteResources.requiresExitNodeConnection, + mode == "inference" + ) + ) + ); // exclude looking at the ones on exit nodes if this is an inference resource if (existingResource.length > 0) { return next( @@ -529,11 +537,7 @@ export async function createSiteResource( .where( and( eq(siteResources.orgId, orgId), - eq(siteResources.alias, alias.trim()), - ne( - siteResources.requiresExitNodeConnection, - mode == "inference" - ) // exclude looking at the ones on exit nodes if this is an inference resource + eq(siteResources.alias, alias.trim()) ) ) .limit(1); diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index 457c401d4..754c82a65 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -172,7 +172,7 @@ const updateSiteResourceSchema = z }, { message: - "Destination is required unless mode is ssh with authDaemonMode native" + "Destination is required unless mode is ssh with authDaemonMode native or inference" } ) .refine( @@ -481,7 +481,15 @@ export async function updateSiteResource( const [existingDomain] = await db .select() .from(siteResources) - .where(eq(siteResources.fullDomain, fullDomain)); + .where( + and( + eq(siteResources.fullDomain, fullDomain), + ne( + siteResources.requiresExitNodeConnection, + mode == "inference" + ) + ) + ); // exclude looking at the ones on exit nodes if this is an inference resource if ( existingDomain && @@ -511,11 +519,7 @@ export async function updateSiteResource( and( eq(siteResources.orgId, existingSiteResource.orgId), eq(siteResources.alias, alias.trim()), - ne(siteResources.siteResourceId, siteResourceId), // exclude self - ne( - siteResources.requiresExitNodeConnection, - mode == "inference" - ) // exclude looking at the ones on exit nodes if this is an inference resource + ne(siteResources.siteResourceId, siteResourceId) // exclude self ) ) .limit(1); diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/general/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/general/page.tsx index b0af75ec0..8dbdb3809 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/general/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/general/page.tsx @@ -24,9 +24,7 @@ import { } from "@app/components/ui/form"; import { Input } from "@app/components/ui/input"; import { SwitchInput } from "@app/components/SwitchInput"; -import { PrivateResourceAliasField } from "@app/components/PrivateResourceDestinationFields"; import { createGeneralFormSchema } from "@app/lib/privateResourceForm"; -import { asAnyControl, asAnyWatch } from "@app/lib/formControlUtils"; import { zodResolver } from "@hookform/resolvers/zod"; import { useTranslations } from "next-intl"; import { useActionState, useMemo } from "react"; @@ -37,12 +35,8 @@ import { useSaveSiteResource } from "@app/hooks/useSaveSiteResource"; export default function PrivateResourceGeneralPage() { const t = useTranslations(); const { save, siteResource } = useSaveSiteResource(); - const isInference = siteResource.mode === "inference"; - const formSchema = useMemo( - () => createGeneralFormSchema(t, { requireAlias: isInference }), - [t, isInference] - ); + const formSchema = useMemo(() => createGeneralFormSchema(t), [t]); type FormValues = z.infer; const form = useForm({ @@ -50,8 +44,7 @@ export default function PrivateResourceGeneralPage() { defaultValues: { name: siteResource.name, niceId: siteResource.niceId, - enabled: siteResource.enabled, - alias: siteResource.alias ?? null + enabled: siteResource.enabled } }); @@ -63,13 +56,7 @@ export default function PrivateResourceGeneralPage() { await save({ name: data.name, niceId: data.niceId, - enabled: data.enabled, - ...(isInference - ? { - mode: "inference" as const, - alias: data.alias - } - : {}) + enabled: data.enabled }); }, null); @@ -165,17 +152,6 @@ export default function PrivateResourceGeneralPage() { )} /> - {isInference && ( - - - - )} diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx index 34534fdd4..78f13370c 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx @@ -1,15 +1,318 @@ -import type { Metadata } from "next"; -import { redirect } from "next/navigation"; +"use client"; -export const metadata: Metadata = { - title: "Private Resource" -}; +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle, + SettingsSubsectionDescription, + SettingsSubsectionHeader, + SettingsSubsectionTitle +} from "@app/components/Settings"; +import { + AiProvidersSelector, + type SelectedAiProvider +} from "@app/components/AiProvidersSelector"; +import DomainPicker from "@app/components/DomainPicker"; +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { useSaveSiteResource } from "@app/hooks/useSaveSiteResource"; +import { useSiteResourceContext } from "@app/hooks/useSiteResourceContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { resourceQueries } from "@app/lib/queries"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useTranslations } from "next-intl"; +import { useRouter } from "next/navigation"; +import { useActionState, useEffect, useMemo, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; -export default async function PrivateResourceInferencePage(props: { - params: Promise<{ niceId: string; orgId: string }>; -}) { - const params = await props.params; - redirect( - `/${params.orgId}/settings/resources/private/${params.niceId}/providers` +export default function PrivateResourceInferencePage() { + const t = useTranslations(); + const router = useRouter(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const queryClient = useQueryClient(); + const { siteResource } = useSiteResourceContext(); + const { save } = useSaveSiteResource(); + + useEffect(() => { + if (siteResource.mode !== "inference") { + router.replace( + `/${siteResource.orgId}/settings/resources/private/${siteResource.niceId}/general` + ); + } + }, [router, siteResource.mode, siteResource.niceId, siteResource.orgId]); + + const formSchema = useMemo( + () => + z.object({ + providerIds: z.array(z.number().int().positive()), + httpConfigSubdomain: z.string().nullish(), + httpConfigDomainId: z.string().nullish(), + httpConfigFullDomain: z.string().nullish() + }), + [] + ); + type FormValues = z.infer; + + const [selectedProviders, setSelectedProviders] = useState< + SelectedAiProvider[] + >([]); + + const attachedQuery = useQuery({ + ...resourceQueries.siteResourceAiProviders({ + siteResourceId: siteResource.id + }), + enabled: siteResource.mode === "inference" + }); + + const form = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + providerIds: [], + httpConfigSubdomain: siteResource.subdomain ?? null, + httpConfigDomainId: siteResource.domainId ?? null, + httpConfigFullDomain: siteResource.fullDomain ?? null + } + }); + + const httpConfigSubdomain = form.watch("httpConfigSubdomain"); + const httpConfigDomainId = form.watch("httpConfigDomainId"); + const httpConfigFullDomain = form.watch("httpConfigFullDomain"); + + useEffect(() => { + if (!attachedQuery.data) return; + const providers = attachedQuery.data.map((provider) => ({ + id: String(provider.providerId), + text: provider.name + })); + setSelectedProviders(providers); + form.setValue( + "providerIds", + attachedQuery.data.map((p) => p.providerId) + ); + }, [attachedQuery.data, form]); + + const [, formAction, saveLoading] = useActionState(async () => { + const isValid = await form.trigger(); + if (!isValid) return; + + const data = form.getValues(); + try { + await save({ + mode: "inference", + httpConfigSubdomain: data.httpConfigSubdomain, + httpConfigDomainId: data.httpConfigDomainId, + httpConfigFullDomain: data.httpConfigFullDomain + }); + + await api.post(`/site-resource/${siteResource.id}/ai-providers`, { + providers: data.providerIds.map((providerId) => ({ + providerId, + modelAccessMode: "catalog" + })) + }); + + await queryClient.invalidateQueries( + resourceQueries.siteResourceAiProviders({ + siteResourceId: siteResource.id + }) + ); + + toast({ + title: t("success"), + description: t("aiResourceProvidersUpdated") + }); + } catch (error) { + toast({ + variant: "destructive", + title: t("aiResourceProvidersErrorUpdate"), + description: formatAxiosError( + error, + t("aiResourceProvidersErrorUpdate") + ) + }); + } + }, null); + + if (siteResource.mode !== "inference") { + return null; + } + + return ( + + + + + {t("aiResourceProviders")} + + + {t("aiResourceProvidersDescription")} + + + + + +
+ + + + ( + + + {t( + "aiResourceProviders" + )} + + + { + setSelectedProviders( + providers + ); + form.setValue( + "providerIds", + providers.map( + (p) => + parseInt( + p.id, + 10 + ) + ), + { + shouldValidate: true + } + ); + }} + /> + + + {t( + "aiResourceProvidersHelp" + )} + + + + )} + /> + + + + + + {t( + "aiResourceDomainConfiguration" + )} + + + {t( + "aiResourceDomainConfigurationDescription" + )} + + + + + { + if (res === null) { + form.setValue( + "httpConfigSubdomain", + null + ); + form.setValue( + "httpConfigDomainId", + null + ); + form.setValue( + "httpConfigFullDomain", + null + ); + return; + } + form.setValue( + "httpConfigSubdomain", + res.subdomain ?? null + ); + form.setValue( + "httpConfigDomainId", + res.domainId + ); + form.setValue( + "httpConfigFullDomain", + res.fullDomain + ); + }} + /> + + +
+ + + + + + + + + ); } diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx index 57064e545..f07b92673 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx @@ -55,37 +55,20 @@ export default async function PrivateResourceLayout( | "sshSettings" | "inferenceSettings"; - const isInference = siteResource.mode === "inference"; - - const navItems = isInference - ? [ - { - title: t("general"), - href: `/{orgId}/settings/resources/private/{niceId}/general` - }, - { - title: t("aiResourceProviders"), - href: `/{orgId}/settings/resources/private/{niceId}/providers` - }, - { - title: t("authentication"), - href: `/{orgId}/settings/resources/private/{niceId}/access` - } - ] - : [ - { - title: t("general"), - href: `/{orgId}/settings/resources/private/{niceId}/general` - }, - { - title: t(modeSettingsKey), - href: `/{orgId}/settings/resources/private/{niceId}/${siteResource.mode}` - }, - { - title: t("authentication"), - href: `/{orgId}/settings/resources/private/{niceId}/access` - } - ]; + const navItems = [ + { + title: t("general"), + href: `/{orgId}/settings/resources/private/{niceId}/general` + }, + { + title: t(modeSettingsKey), + href: `/{orgId}/settings/resources/private/{niceId}/${siteResource.mode}` + }, + { + title: t("authentication"), + href: `/{orgId}/settings/resources/private/{niceId}/access` + } + ]; return ( <> diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx deleted file mode 100644 index dcbc4ea29..000000000 --- a/src/app/[orgId]/settings/resources/private/[niceId]/providers/page.tsx +++ /dev/null @@ -1,230 +0,0 @@ -"use client"; - -import { - SettingsContainer, - SettingsFormCell, - SettingsFormGrid, - SettingsSection, - SettingsSectionBody, - SettingsSectionDescription, - SettingsSectionFooter, - SettingsSectionForm, - SettingsSectionHeader, - SettingsSectionTitle -} from "@app/components/Settings"; -import { - AiProvidersSelector, - type SelectedAiProvider -} from "@app/components/AiProvidersSelector"; -import { Button } from "@app/components/ui/button"; -import { - Form, - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage -} from "@app/components/ui/form"; -import { useEnvContext } from "@app/hooks/useEnvContext"; -import { useSiteResourceContext } from "@app/hooks/useSiteResourceContext"; -import { toast } from "@app/hooks/useToast"; -import { createApiClient, formatAxiosError } from "@app/lib/api"; -import { resourceQueries } from "@app/lib/queries"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { useTranslations } from "next-intl"; -import { useRouter } from "next/navigation"; -import { useActionState, useEffect, useMemo, useState } from "react"; -import { useForm } from "react-hook-form"; -import { z } from "zod"; - -export default function PrivateResourceProvidersPage() { - const t = useTranslations(); - const router = useRouter(); - const { env } = useEnvContext(); - const api = createApiClient({ env }); - const queryClient = useQueryClient(); - const { siteResource } = useSiteResourceContext(); - - useEffect(() => { - if (siteResource.mode !== "inference") { - router.replace( - `/${siteResource.orgId}/settings/resources/private/${siteResource.niceId}/general` - ); - } - }, [router, siteResource.mode, siteResource.niceId, siteResource.orgId]); - - const formSchema = useMemo( - () => - z.object({ - providerIds: z.array(z.number().int().positive()) - }), - [] - ); - type FormValues = z.infer; - - const [selectedProviders, setSelectedProviders] = useState< - SelectedAiProvider[] - >([]); - - const attachedQuery = useQuery({ - ...resourceQueries.siteResourceAiProviders({ - siteResourceId: siteResource.id - }), - enabled: siteResource.mode === "inference" - }); - - const form = useForm({ - resolver: zodResolver(formSchema), - defaultValues: { - providerIds: [] - } - }); - - useEffect(() => { - if (!attachedQuery.data) return; - const providers = attachedQuery.data.map((provider) => ({ - id: String(provider.providerId), - text: provider.name - })); - setSelectedProviders(providers); - form.reset({ - providerIds: attachedQuery.data.map((p) => p.providerId) - }); - }, [attachedQuery.data, form]); - - const [, formAction, saveLoading] = useActionState(async () => { - const isValid = await form.trigger(); - if (!isValid) return; - - const data = form.getValues(); - try { - await api.post(`/site-resource/${siteResource.id}/ai-providers`, { - providers: data.providerIds.map((providerId) => ({ - providerId, - modelAccessMode: "catalog" - })) - }); - - await queryClient.invalidateQueries( - resourceQueries.siteResourceAiProviders({ - siteResourceId: siteResource.id - }) - ); - - toast({ - title: t("success"), - description: t("aiResourceProvidersUpdated") - }); - } catch (error) { - toast({ - variant: "destructive", - title: t("aiResourceProvidersErrorUpdate"), - description: formatAxiosError( - error, - t("aiResourceProvidersErrorUpdate") - ) - }); - } - }, null); - - if (siteResource.mode !== "inference") { - return null; - } - - return ( - - - - - {t("aiResourceProviders")} - - - {t("aiResourceProvidersDescription")} - - - - - -
- - - - ( - - - {t( - "aiResourceProviders" - )} - - - { - setSelectedProviders( - providers - ); - form.setValue( - "providerIds", - providers.map( - (p) => - parseInt( - p.id, - 10 - ) - ), - { - shouldValidate: true - } - ); - }} - /> - - - {t( - "aiResourceProvidersHelp" - )} - - - - )} - /> - - -
- - - - - - - - - - ); -} diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx index 8ea8d2dd5..8741159b4 100644 --- a/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx +++ b/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx @@ -1,15 +1,230 @@ -import type { Metadata } from "next"; -import { redirect } from "next/navigation"; +"use client"; -export const metadata: Metadata = { - title: "Public Resource" -}; +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionForm, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { + AiProvidersSelector, + type SelectedAiProvider +} from "@app/components/AiProvidersSelector"; +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { useResourceContext } from "@app/hooks/useResourceContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { resourceQueries } from "@app/lib/queries"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useTranslations } from "next-intl"; +import { useRouter } from "next/navigation"; +import { useActionState, useEffect, useMemo, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; -export default async function PublicResourceInferencePage(props: { - params: Promise<{ niceId: string; orgId: string }>; -}) { - const params = await props.params; - redirect( - `/${params.orgId}/settings/resources/public/${params.niceId}/providers` +export default function PublicResourceInferencePage() { + const t = useTranslations(); + const router = useRouter(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const queryClient = useQueryClient(); + const { resource } = useResourceContext(); + + useEffect(() => { + if (resource.mode !== "inference") { + router.replace( + `/${resource.orgId}/settings/resources/public/${resource.niceId}/general` + ); + } + }, [router, resource.mode, resource.niceId, resource.orgId]); + + const formSchema = useMemo( + () => + z.object({ + providerIds: z.array(z.number().int().positive()) + }), + [] + ); + type FormValues = z.infer; + + const [selectedProviders, setSelectedProviders] = useState< + SelectedAiProvider[] + >([]); + + const attachedQuery = useQuery({ + ...resourceQueries.resourceAiProviders({ + resourceId: resource.resourceId + }), + enabled: resource.mode === "inference" + }); + + const form = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + providerIds: [] + } + }); + + useEffect(() => { + if (!attachedQuery.data) return; + const providers = attachedQuery.data.map((provider) => ({ + id: String(provider.providerId), + text: provider.name + })); + setSelectedProviders(providers); + form.reset({ + providerIds: attachedQuery.data.map((p) => p.providerId) + }); + }, [attachedQuery.data, form]); + + const [, formAction, saveLoading] = useActionState(async () => { + const isValid = await form.trigger(); + if (!isValid) return; + + const data = form.getValues(); + try { + await api.post(`/resource/${resource.resourceId}/ai-providers`, { + providers: data.providerIds.map((providerId) => ({ + providerId, + modelAccessMode: "catalog" + })) + }); + + await queryClient.invalidateQueries( + resourceQueries.resourceAiProviders({ + resourceId: resource.resourceId + }) + ); + + toast({ + title: t("success"), + description: t("aiResourceProvidersUpdated") + }); + } catch (error) { + toast({ + variant: "destructive", + title: t("aiResourceProvidersErrorUpdate"), + description: formatAxiosError( + error, + t("aiResourceProvidersErrorUpdate") + ) + }); + } + }, null); + + if (resource.mode !== "inference") { + return null; + } + + return ( + + + + + {t("aiResourceProviders")} + + + {t("aiResourceProvidersDescription")} + + + + + +
+ + + + ( + + + {t( + "aiResourceProviders" + )} + + + { + setSelectedProviders( + providers + ); + form.setValue( + "providerIds", + providers.map( + (p) => + parseInt( + p.id, + 10 + ) + ), + { + shouldValidate: true + } + ); + }} + /> + + + {t( + "aiResourceProvidersHelp" + )} + + + + )} + /> + + +
+ + + + + + + + + ); } diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx index 302650bf2..7eb664b5a 100644 --- a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx @@ -82,20 +82,7 @@ export default async function ResourceLayout(props: ResourceLayoutProps) { redirect(`/${params.orgId}/settings/resources`); } - const isInference = resource.mode === "inference"; - - const navItems = isInference - ? [ - { - title: t("general"), - href: `/{orgId}/settings/resources/public/{niceId}/general` - }, - { - title: t("aiResourceProviders"), - href: `/{orgId}/settings/resources/public/{niceId}/providers` - } - ] - : [ + const navItems = [ { title: t("general"), href: `/{orgId}/settings/resources/public/{niceId}/general` diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx deleted file mode 100644 index 51182c2a8..000000000 --- a/src/app/[orgId]/settings/resources/public/[niceId]/providers/page.tsx +++ /dev/null @@ -1,230 +0,0 @@ -"use client"; - -import { - SettingsContainer, - SettingsFormCell, - SettingsFormGrid, - SettingsSection, - SettingsSectionBody, - SettingsSectionDescription, - SettingsSectionFooter, - SettingsSectionForm, - SettingsSectionHeader, - SettingsSectionTitle -} from "@app/components/Settings"; -import { - AiProvidersSelector, - type SelectedAiProvider -} from "@app/components/AiProvidersSelector"; -import { Button } from "@app/components/ui/button"; -import { - Form, - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage -} from "@app/components/ui/form"; -import { useEnvContext } from "@app/hooks/useEnvContext"; -import { useResourceContext } from "@app/hooks/useResourceContext"; -import { toast } from "@app/hooks/useToast"; -import { createApiClient, formatAxiosError } from "@app/lib/api"; -import { resourceQueries } from "@app/lib/queries"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { useTranslations } from "next-intl"; -import { useRouter } from "next/navigation"; -import { useActionState, useEffect, useMemo, useState } from "react"; -import { useForm } from "react-hook-form"; -import { z } from "zod"; - -export default function PublicResourceProvidersPage() { - const t = useTranslations(); - const router = useRouter(); - const { env } = useEnvContext(); - const api = createApiClient({ env }); - const queryClient = useQueryClient(); - const { resource } = useResourceContext(); - - useEffect(() => { - if (resource.mode !== "inference") { - router.replace( - `/${resource.orgId}/settings/resources/public/${resource.niceId}/general` - ); - } - }, [router, resource.mode, resource.niceId, resource.orgId]); - - const formSchema = useMemo( - () => - z.object({ - providerIds: z.array(z.number().int().positive()) - }), - [] - ); - type FormValues = z.infer; - - const [selectedProviders, setSelectedProviders] = useState< - SelectedAiProvider[] - >([]); - - const attachedQuery = useQuery({ - ...resourceQueries.resourceAiProviders({ - resourceId: resource.resourceId - }), - enabled: resource.mode === "inference" - }); - - const form = useForm({ - resolver: zodResolver(formSchema), - defaultValues: { - providerIds: [] - } - }); - - useEffect(() => { - if (!attachedQuery.data) return; - const providers = attachedQuery.data.map((provider) => ({ - id: String(provider.providerId), - text: provider.name - })); - setSelectedProviders(providers); - form.reset({ - providerIds: attachedQuery.data.map((p) => p.providerId) - }); - }, [attachedQuery.data, form]); - - const [, formAction, saveLoading] = useActionState(async () => { - const isValid = await form.trigger(); - if (!isValid) return; - - const data = form.getValues(); - try { - await api.post(`/resource/${resource.resourceId}/ai-providers`, { - providers: data.providerIds.map((providerId) => ({ - providerId, - modelAccessMode: "catalog" - })) - }); - - await queryClient.invalidateQueries( - resourceQueries.resourceAiProviders({ - resourceId: resource.resourceId - }) - ); - - toast({ - title: t("success"), - description: t("aiResourceProvidersUpdated") - }); - } catch (error) { - toast({ - variant: "destructive", - title: t("aiResourceProvidersErrorUpdate"), - description: formatAxiosError( - error, - t("aiResourceProvidersErrorUpdate") - ) - }); - } - }, null); - - if (resource.mode !== "inference") { - return null; - } - - return ( - - - - - {t("aiResourceProviders")} - - - {t("aiResourceProvidersDescription")} - - - - - -
- - - - ( - - - {t( - "aiResourceProviders" - )} - - - { - setSelectedProviders( - providers - ); - form.setValue( - "providerIds", - providers.map( - (p) => - parseInt( - p.id, - 10 - ) - ), - { - shouldValidate: true - } - ); - }} - /> - - - {t( - "aiResourceProvidersHelp" - )} - - - - )} - /> - - -
- - - - - - - - - - ); -} diff --git a/src/lib/privateResourceForm.ts b/src/lib/privateResourceForm.ts index e336a45a1..3eb83cce3 100644 --- a/src/lib/privateResourceForm.ts +++ b/src/lib/privateResourceForm.ts @@ -305,7 +305,13 @@ export function buildUpdateSiteResourcePayload( typeof data.alias === "string" && data.alias.trim() ? data.alias - : null + : null, + domainId: data.httpConfigDomainId + ? data.httpConfigDomainId + : undefined, + subdomain: data.httpConfigSubdomain + ? data.httpConfigSubdomain + : undefined }), ...((data.mode === "host" || data.mode === "cidr") && { tcpPortRangeString: data.tcpPortRangeString, From b4d2974e19472306ff79b0dd473a91d80c4125b5 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 12:20:56 -0400 Subject: [PATCH 064/296] Properly configure ssl and domain when creating and editing --- .../private/[niceId]/inference/page.tsx | 37 ++++++++- .../resources/private/create/page.tsx | 82 ++++++++++--------- src/components/PrivateResourcesTable.tsx | 10 ++- src/components/PublicResourcesTable.tsx | 5 +- src/lib/privateResourceForm.ts | 26 ++---- 5 files changed, 95 insertions(+), 65 deletions(-) diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx index 78f13370c..48304d350 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx @@ -20,6 +20,7 @@ import { type SelectedAiProvider } from "@app/components/AiProvidersSelector"; import DomainPicker from "@app/components/DomainPicker"; +import { SwitchInput } from "@app/components/SwitchInput"; import { Button } from "@app/components/ui/button"; import { Form, @@ -67,7 +68,8 @@ export default function PrivateResourceInferencePage() { providerIds: z.array(z.number().int().positive()), httpConfigSubdomain: z.string().nullish(), httpConfigDomainId: z.string().nullish(), - httpConfigFullDomain: z.string().nullish() + httpConfigFullDomain: z.string().nullish(), + ssl: z.boolean().optional() }), [] ); @@ -90,7 +92,8 @@ export default function PrivateResourceInferencePage() { providerIds: [], httpConfigSubdomain: siteResource.subdomain ?? null, httpConfigDomainId: siteResource.domainId ?? null, - httpConfigFullDomain: siteResource.fullDomain ?? null + httpConfigFullDomain: siteResource.fullDomain ?? null, + ssl: siteResource.ssl ?? false } }); @@ -121,7 +124,8 @@ export default function PrivateResourceInferencePage() { mode: "inference", httpConfigSubdomain: data.httpConfigSubdomain, httpConfigDomainId: data.httpConfigDomainId, - httpConfigFullDomain: data.httpConfigFullDomain + httpConfigFullDomain: data.httpConfigFullDomain, + ssl: data.ssl }); await api.post(`/site-resource/${siteResource.id}/ai-providers`, { @@ -296,6 +300,33 @@ export default function PrivateResourceInferencePage() { }} /> + + ( + + + + + + )} + /> + diff --git a/src/app/[orgId]/settings/resources/private/create/page.tsx b/src/app/[orgId]/settings/resources/private/create/page.tsx index 9420da17b..da3c968b6 100644 --- a/src/app/[orgId]/settings/resources/private/create/page.tsx +++ b/src/app/[orgId]/settings/resources/private/create/page.tsx @@ -368,56 +368,58 @@ export default function CreatePrivateResourcePage() { /> - {mode === "http" && ( - - - { - if (!res) { + {mode === "http" || + (mode === "inference" && ( + + + { + if (!res) { + form.setValue( + "httpConfigSubdomain", + null + ); + form.setValue( + "httpConfigDomainId", + null + ); + form.setValue( + "httpConfigFullDomain", + null + ); + return; + } form.setValue( "httpConfigSubdomain", - null + res.subdomain ?? + null ); form.setValue( "httpConfigDomainId", - null + res.domainId ); form.setValue( "httpConfigFullDomain", - null + res.fullDomain ); - return; - } - form.setValue( - "httpConfigSubdomain", - res.subdomain ?? - null - ); - form.setValue( - "httpConfigDomainId", - res.domainId - ); - form.setValue( - "httpConfigFullDomain", - res.fullDomain - ); - }} - /> - - - {t( - "resourceDomainDescription" - )} - - - - )} + }} + /> + + + {t( + "resourceDomainDescription" + )} + + + + ))} {(mode === "host" || - mode === "inference" || (mode === "ssh" && !isNativeSsh)) && ( - {t("sshServer")} + {t("sshSettings")} {t("sshServerDescription")} diff --git a/src/components/PrivateResourcesTable.tsx b/src/components/PrivateResourcesTable.tsx index 87eddb921..cb7d2645c 100644 --- a/src/components/PrivateResourcesTable.tsx +++ b/src/components/PrivateResourcesTable.tsx @@ -332,7 +332,7 @@ export default function PrivateResourcesTable({ }, { accessorKey: "mode", - friendlyName: t("editInternalResourceDialogMode"), + friendlyName: t("type"), header: () => ( ), diff --git a/src/components/PublicResourcesTable.tsx b/src/components/PublicResourcesTable.tsx index 48d7b4486..af89b878b 100644 --- a/src/components/PublicResourcesTable.tsx +++ b/src/components/PublicResourcesTable.tsx @@ -280,7 +280,7 @@ export default function PublicResourcesTable({ }, { accessorKey: "protocol", - friendlyName: t("protocol"), + friendlyName: t("type"), enableHiding: true, header: () => ( ), @@ -747,7 +747,6 @@ export default function PublicResourcesTable({ enableColumnVisibility columnVisibility={{ niceId: false, - protocol: false, labels: true }} stickyLeftColumn="name" diff --git a/src/lib/privateResourceForm.ts b/src/lib/privateResourceForm.ts index 3eb83cce3..f186ffb9d 100644 --- a/src/lib/privateResourceForm.ts +++ b/src/lib/privateResourceForm.ts @@ -216,16 +216,17 @@ export function buildCreateSiteResourcePayload( }) }), ...(data.mode === "inference" && { - alias: - data.alias && - typeof data.alias === "string" && - data.alias.trim() - ? data.alias - : undefined, aiProviders: (data.providerIds ?? []).map((providerId) => ({ providerId, modelAccessMode: "catalog" as const - })) + })), + ssl: data.ssl ?? false, + domainId: data.httpConfigDomainId + ? data.httpConfigDomainId + : undefined, + subdomain: data.httpConfigSubdomain + ? data.httpConfigSubdomain + : undefined }), ...((data.mode === "host" || data.mode === "cidr") && { tcpPortRangeString: data.tcpPortRangeString, @@ -306,6 +307,7 @@ export function buildUpdateSiteResourcePayload( data.alias.trim() ? data.alias : null, + ssl: data.ssl ?? false, domainId: data.httpConfigDomainId ? data.httpConfigDomainId : undefined, @@ -464,16 +466,6 @@ export function createCreateFormSchema(t: TranslateFn) { path: ["siteIds"] }); } - if (data.mode === "inference") { - const trimmedAlias = data.alias?.trim(); - if (!trimmedAlias) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: t("aiResourceAliasRequired"), - path: ["alias"] - }); - } - } if ( data.mode !== "ssh" && data.mode !== "inference" && From 6cca5e047228573fb1e9b4798d409443851fd1ed Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 13:45:51 -0400 Subject: [PATCH 065/296] Use fulldoman instead of the alias for the site resources --- server/lib/rebuildClientAssociations.ts | 18 +++++++------- server/lib/traefik/getTraefikConfig.ts | 14 +++++------ .../private/lib/traefik/getTraefikConfig.ts | 24 +++++++++---------- server/routers/olm/buildConfiguration.ts | 2 +- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/server/lib/rebuildClientAssociations.ts b/server/lib/rebuildClientAssociations.ts index fe1cdea6f..045ca3927 100644 --- a/server/lib/rebuildClientAssociations.ts +++ b/server/lib/rebuildClientAssociations.ts @@ -1117,7 +1117,8 @@ async function syncClientExitNodeConnections( const requiresExitNodeRows = await trx .select({ clientId: clientSiteResourcesAssociationsCache.clientId, - alias: siteResources.alias + alias: siteResources.alias, + fullDomain: siteResources.fullDomain }) .from(clientSiteResourcesAssociationsCache) .innerJoin( @@ -1145,14 +1146,16 @@ async function syncClientExitNodeConnections( // Aliases for every exit-node-backed resource this client can reach, so // the live connect push carries the same alias list the register/reconnect // path (buildSiteConfigurationForOlmClient) would compute. - const exitNodeAliasesByClientId = new Map(); + const exitNodeAliasesByClientId = new Map(); for (const row of requiresExitNodeRows) { - if (row.alias == null) continue; + if (row.alias == null && row.fullDomain == null) continue; const existing = exitNodeAliasesByClientId.get(row.clientId); if (existing) { - existing.push(row.alias); + existing.push(row.fullDomain || row.alias); // accept both for now in case we have other resource types that dont use the full domain } else { - exitNodeAliasesByClientId.set(row.clientId, [row.alias]); + exitNodeAliasesByClientId.set(row.clientId, [ + row.fullDomain || row.alias + ]); } } @@ -1231,9 +1234,8 @@ async function syncClientExitNodeConnections( publicKey: exitNode.publicKey, serverIP: exitNode.address.split("/")[0], tunnelIP: client.exitNodeSubnet.split("/")[0], - aliases: exitNodeAliasesByClientId.get( - client.clientId - ) ?? [] + aliases: + exitNodeAliasesByClientId.get(client.clientId) ?? [] } }, options: { diff --git a/server/lib/traefik/getTraefikConfig.ts b/server/lib/traefik/getTraefikConfig.ts index 4df621684..a6535c766 100644 --- a/server/lib/traefik/getTraefikConfig.ts +++ b/server/lib/traefik/getTraefikConfig.ts @@ -860,7 +860,7 @@ export async function getTraefikConfig( const siteResourcesInference = await db .selectDistinct({ siteResourceId: siteResources.siteResourceId, - alias: siteResources.alias, + fullDomain: siteResources.fullDomain, ssl: siteResources.ssl, enabled: siteResources.enabled }) @@ -869,23 +869,23 @@ export async function getTraefikConfig( and( eq(siteResources.mode, "inference"), eq(siteResources.enabled, true), - isNotNull(siteResources.alias) + isNotNull(siteResources.fullDomain) ) ); for (const sr of siteResourcesInference) { - if (!sr.enabled || !sr.alias) continue; + if (!sr.enabled || !sr.fullDomain) continue; if (!config_output.http.routers) config_output.http.routers = {}; if (!config_output.http.services) config_output.http.services = {}; - const alias = sr.alias; + const fullDomain = sr.fullDomain; const srKey = `inference-sr${sr.siteResourceId}`; const routerName = `${srKey}-router`; const serviceName = `${srKey}-service`; - const rule = `Host(\`${alias}\`) && ClientIP(${exitNode.address})`; // restrict to coming from the exit node ip range that the client is connected to + const rule = `Host(\`${fullDomain}\`) && ClientIP(${exitNode.address})`; // restrict to coming from the exit node ip range that the client is connected to - const domainParts = alias.split("."); + const domainParts = fullDomain.split("."); const wildCard = domainParts.length <= 2 ? `*.${domainParts.join(".")}` @@ -911,7 +911,7 @@ export async function getTraefikConfig( headers: { customRequestHeaders: { ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": alias + "p-host": fullDomain } } }; diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 5cd5549c8..b00dbd7c8 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -407,7 +407,7 @@ export async function getTraefikConfig( and( eq(siteResources.enabled, true), isNotNull(siteResources.fullDomain), - eq(siteResources.mode, "http"), + eq(siteResources.mode, "http"), // important so we dont double get the inference siteResources below eq(siteResources.ssl, true), eq(sites.exitNodeId, exitNodeId), inArray(sites.type, siteTypes) @@ -452,7 +452,7 @@ export async function getTraefikConfig( const siteResourcesInference = await db .selectDistinct({ siteResourceId: siteResources.siteResourceId, - alias: siteResources.alias, + fullDomain: siteResources.fullDomain, ssl: siteResources.ssl, enabled: siteResources.enabled }) @@ -461,7 +461,7 @@ export async function getTraefikConfig( and( eq(siteResources.mode, "inference"), eq(siteResources.enabled, true), - isNotNull(siteResources.alias) + isNotNull(siteResources.fullDomain) ) ); @@ -493,8 +493,8 @@ export async function getTraefikConfig( } } for (const sr of siteResourcesInference) { - if (sr.enabled && sr.ssl && sr.alias) { - domains.add(sr.alias); + if (sr.enabled && sr.ssl && sr.fullDomain) { + domains.add(sr.fullDomain); } } // get the valid certs for these domains @@ -1687,20 +1687,20 @@ export async function getTraefikConfig( // siteResources today (see plan doc), so gating here is // reachability-only for now. for (const sr of siteResourcesInference) { - if (!sr.enabled || !sr.alias) continue; + if (!sr.enabled || !sr.fullDomain) continue; if (!config_output.http.routers) config_output.http.routers = {}; if (!config_output.http.services) config_output.http.services = {}; - const alias = sr.alias; + const fullDomain = sr.fullDomain; const srKey = `inference-sr${sr.siteResourceId}`; const routerName = `${srKey}-router`; const serviceName = `${srKey}-service`; - const rule = `Host(\`${alias}\`) && ClientIP(${exitNode.address})`; // restrict to coming from the exit node ip range that the client is connected to + const rule = `Host(\`${fullDomain}\`) && ClientIP(${exitNode.address})`; // restrict to coming from the exit node ip range that the client is connected to let tls: any = {}; if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { - const domainParts = alias.split("."); + const domainParts = fullDomain.split("."); const wildCard = domainParts.length <= 2 ? `*.${domainParts.join(".")}` @@ -1719,11 +1719,11 @@ export async function getTraefikConfig( }; } else { const matchingCert = validCerts.find( - (cert) => cert.queriedDomain === alias + (cert) => cert.queriedDomain === fullDomain ); if (!matchingCert) { logger.debug( - `No matching certificate found for inference siteResource alias: ${alias}` + `No matching certificate found for inference siteResource fullDomain: ${fullDomain}` ); continue; } @@ -1739,7 +1739,7 @@ export async function getTraefikConfig( headers: { customRequestHeaders: { ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": alias + "p-host": fullDomain } } }; diff --git a/server/routers/olm/buildConfiguration.ts b/server/routers/olm/buildConfiguration.ts index af309f89a..1eb2cfa91 100644 --- a/server/routers/olm/buildConfiguration.ts +++ b/server/routers/olm/buildConfiguration.ts @@ -98,7 +98,7 @@ export async function buildSiteConfigurationForOlmClient( } exitNodeAliases = siteResourcesForExitNode - .map((sr) => sr.alias) + .map((sr) => sr.fullDomain || sr.alias) // take either in case we introduce different resource types that don't have a fullDomain .filter((a) => a != null); if (sitesData.length == 0) { From c5d68675c9e637d72f3fc73f5cb19afbcb0d300d Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Thu, 6 Aug 2026 14:27:08 -0400 Subject: [PATCH 066/296] support patterns in model key --- messages/en-US.json | 6 +- server/lib/aiInferenceResource.ts | 36 +++++++----- server/lib/aiModelKeyMatch.ts | 85 ++++++++++++++++++++++++++++ server/routers/aiGateway/pipeline.ts | 71 ++++++++++++++++------- 4 files changed, 162 insertions(+), 36 deletions(-) create mode 100644 server/lib/aiModelKeyMatch.ts diff --git a/messages/en-US.json b/messages/en-US.json index f5de37878..e994c823c 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1759,13 +1759,13 @@ "aiProviderMessageRemove": "This will permanently delete the provider and its models and targets. This cannot be undone.", "aiProviderErrorNoUpdate": "AI provider is not available to update", "aiProviderModels": "Models", - "aiProviderModelsDescription": "Define model names available on this provider. Requests must use one of these model keys.", - "aiProviderModelsPlaceholder": "Type a model name and press Enter", + "aiProviderModelsDescription": "Define model names available on this provider. Requests must match one of these keys. Use * and ? as wildcards (for example gpt-4* or claude-?).", + "aiProviderModelsPlaceholder": "Model name or pattern (e.g. gpt-4*)", "aiProviderModelsUpdated": "Models updated", "aiProviderModelsErrorUpdate": "Failed to update models", "aiResourceProviders": "Providers", "aiResourceProvidersDescription": "Choose which AI providers this inference resource can use", - "aiResourceProvidersHelp": "Models must be defined on each provider. Model names cannot overlap across selected providers.", + "aiResourceProvidersHelp": "Models must be defined on each provider. Exact names and patterns that conflict (identical keys, or an exact key matching another provider's pattern) are not allowed across selected providers.", "aiResourceProvidersSelect": "Select providers", "aiResourceProvidersEmpty": "No AI providers found", "aiResourceProvidersUpdated": "Providers updated", diff --git a/server/lib/aiInferenceResource.ts b/server/lib/aiInferenceResource.ts index 6e0f490d8..d01b30dd9 100644 --- a/server/lib/aiInferenceResource.ts +++ b/server/lib/aiInferenceResource.ts @@ -10,6 +10,7 @@ import { type Transaction } from "@server/db"; import { z } from "zod"; +import { modelKeysConflict } from "@server/lib/aiModelKeyMatch"; type DbOrTrx = Transaction | typeof db; @@ -55,10 +56,13 @@ function normalizeAttachments( } /** - * Ensure enabled catalog modelKeys are unique across attached providers. + * Ensure enabled catalog modelKeys do not conflict across attached providers. * Catalog attachments contribute all enabled models on the provider. * Allowlist attachments contribute nothing until models are allowlisted * (those are checked when the allowlist is set). + * + * Conflicts: identical keys, or an exact key that matches another provider's + * pattern. Full glob intersections are left to runtime ambiguity errors. */ export async function assertNoOverlappingModelKeys( attachments: ResourceAiProviderAttachment[], @@ -85,25 +89,31 @@ export async function assertNoOverlappingModelKeys( ) ); - const keyToProviders = new Map(); - for (const model of models) { - const existing = keyToProviders.get(model.modelKey) ?? []; - if (!existing.includes(model.providerId)) { - existing.push(model.providerId); + const conflictPairs: string[] = []; + for (let i = 0; i < models.length; i++) { + for (let j = i + 1; j < models.length; j++) { + const left = models[i]; + const right = models[j]; + if (left.providerId === right.providerId) { + continue; + } + if (!modelKeysConflict(left.modelKey, right.modelKey)) { + continue; + } + const pair = [left.modelKey, right.modelKey].sort().join(" vs "); + if (!conflictPairs.includes(pair)) { + conflictPairs.push(pair); + } } - keyToProviders.set(model.modelKey, existing); } - const overlaps = [...keyToProviders.entries()].filter( - ([, providerIds]) => providerIds.length > 1 - ); - if (overlaps.length === 0) { + if (conflictPairs.length === 0) { return null; } - const keys = overlaps.map(([key]) => key).sort(); + conflictPairs.sort(); return { - error: `Model keys must be unique across providers on a resource. Overlapping keys: ${keys.join(", ")}` + error: `Model keys must be unique across providers on a resource. Overlapping keys: ${conflictPairs.join(", ")}` }; } diff --git a/server/lib/aiModelKeyMatch.ts b/server/lib/aiModelKeyMatch.ts new file mode 100644 index 000000000..fa27be56f --- /dev/null +++ b/server/lib/aiModelKeyMatch.ts @@ -0,0 +1,85 @@ +const modelKeyRegexCache = new Map(); + +export function isModelKeyPattern(key: string): boolean { + return key.includes("*") || key.includes("?"); +} + +function getModelKeyRegex(pattern: string): RegExp { + let regex = modelKeyRegexCache.get(pattern); + if (!regex) { + const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&"); + regex = new RegExp( + `^${escaped.replace(/\*/g, ".*").replace(/\?/g, ".")}$` + ); + modelKeyRegexCache.set(pattern, regex); + } + return regex; +} + +export function modelKeyMatches( + pattern: string, + requestedModel: string +): boolean { + return getModelKeyRegex(pattern).test(requestedModel); +} + +function wildcardCharCount(key: string): number { + let count = 0; + for (const char of key) { + if (char === "*" || char === "?") { + count += 1; + } + } + return count; +} + +function literalLength(key: string): number { + return key.replace(/[*?]/g, "").length; +} + +/** + * Sort comparator: more specific patterns sort before less specific ones + * (negative when `a` is more specific than `b`). + * + * 1. Exact keys beat patterns + * 2. Fewer wildcard characters win + * 3. Longer literal length wins + */ +export function compareModelKeySpecificity(a: string, b: string): number { + const aIsPattern = isModelKeyPattern(a); + const bIsPattern = isModelKeyPattern(b); + + if (aIsPattern !== bIsPattern) { + return aIsPattern ? 1 : -1; + } + + const wildcardDiff = wildcardCharCount(a) - wildcardCharCount(b); + if (wildcardDiff !== 0) { + return wildcardDiff; + } + + return literalLength(b) - literalLength(a); +} + +/** + * Attach-time conflict check. Detects identical keys and exact-vs-pattern + * matches. Does not attempt full glob intersection. + */ +export function modelKeysConflict(a: string, b: string): boolean { + if (a === b) { + return true; + } + + const aIsPattern = isModelKeyPattern(a); + const bIsPattern = isModelKeyPattern(b); + + if (aIsPattern === bIsPattern) { + return false; + } + + if (aIsPattern) { + return modelKeyMatches(a, b); + } + + return modelKeyMatches(b, a); +} diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index bdc4b0eb5..247e55b4c 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -39,6 +39,10 @@ import { localCache } from "@server/lib/cache"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; import type { ModelAccessMode } from "@server/lib/aiInferenceResource"; +import { + compareModelKeySpecificity, + modelKeyMatches +} from "@server/lib/aiModelKeyMatch"; import { aiGatewayUpstreamFetch } from "@server/lib/aiGatewayUpstreamFetch"; // Short-lived local caches so a burst of requests from the same IP/user @@ -369,55 +373,82 @@ async function selectProvider( }; } - const matchingModels = await db + const providerModels = await db .select({ modelId: aiModels.modelId, providerId: aiModels.providerId, + modelKey: aiModels.modelKey, enabled: aiModels.enabled }) .from(aiModels) - .where( - and( - inArray(aiModels.providerId, providerIds), - eq(aiModels.modelKey, requestedModel) - ) - ); + .where(inArray(aiModels.providerId, providerIds)); + + type ModelCandidate = { + provider: AiProvider; + modelKey: string; + }; + + const candidates: ModelCandidate[] = []; + for (const model of providerModels) { + if (!model.enabled) { + continue; + } + + if (!modelKeyMatches(model.modelKey, requestedModel)) { + continue; + } - const candidates: AiProvider[] = []; - for (const model of matchingModels) { const attachment = providerById.get(model.providerId); if (!attachment) { continue; } if (attachment.modelAccessMode === "catalog") { - if (model.enabled) { - candidates.push(attachment.provider); - } + candidates.push({ + provider: attachment.provider, + modelKey: model.modelKey + }); continue; } if (allowlistedModelIds.has(model.modelId)) { - candidates.push(attachment.provider); + candidates.push({ + provider: attachment.provider, + modelKey: model.modelKey + }); } } - if (candidates.length === 1) { - return { ok: true, provider: candidates[0] }; - } - - if (candidates.length > 1) { + if (candidates.length === 0) { return { ok: false, status: HttpCode.FORBIDDEN, - message: `Model "${requestedModel}" is ambiguous across multiple AI providers on this resource` + message: `Model "${requestedModel}" is not permitted on this resource` }; } + candidates.sort((a, b) => + compareModelKeySpecificity(a.modelKey, b.modelKey) + ); + + const bestSpecificity = candidates[0].modelKey; + const topCandidates = candidates.filter( + (c) => compareModelKeySpecificity(c.modelKey, bestSpecificity) === 0 + ); + + const uniqueProviders = new Map(); + for (const candidate of topCandidates) { + uniqueProviders.set(candidate.provider.providerId, candidate.provider); + } + + if (uniqueProviders.size === 1) { + return { ok: true, provider: [...uniqueProviders.values()][0] }; + } + return { ok: false, status: HttpCode.FORBIDDEN, - message: `Model "${requestedModel}" is not permitted on this resource` + message: `Model "${requestedModel}" is ambiguous across multiple AI providers on this resource` }; } From f72252552f0c61616f7d55719325f5fdebfe9f81 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 14:28:05 -0400 Subject: [PATCH 067/296] Fix saving --- .../ai-providers/[providerId]/authentication/page.tsx | 6 ++++-- .../settings/ai-providers/[providerId]/network/page.tsx | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx index 5a7b068d8..f9e44cb09 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx @@ -66,7 +66,8 @@ export default function AiProviderAuthenticationPage() { authType: (provider.authType as AiProviderAuthType) ?? "bearer", routingMode: (provider.routingMode as "url" | "target") ?? "url", skipTlsVerification: provider.skipTlsVerification, - enabled: provider.enabled + enabled: provider.enabled, + capabilities: provider.capabilities ?? [] } }); @@ -97,7 +98,8 @@ export default function AiProviderAuthenticationPage() { authType: (updated.authType as AiProviderAuthType) ?? "bearer", routingMode: (updated.routingMode as "url" | "target") ?? "url", skipTlsVerification: updated.skipTlsVerification, - enabled: updated.enabled + enabled: updated.enabled, + capabilities: updated.capabilities ?? [] }); toast({ title: t("success"), diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx index 8d9f02744..1aa25e9e5 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx @@ -83,7 +83,8 @@ export default function AiProviderNetworkPage() { routingMode: (provider.routingMode as "url" | "target") ?? "url", headers: provider.headers ?? [], skipTlsVerification: provider.skipTlsVerification, - enabled: provider.enabled + enabled: provider.enabled, + capabilities: provider.capabilities ?? [] } }); @@ -127,7 +128,8 @@ export default function AiProviderNetworkPage() { routingMode: (updated.routingMode as "url" | "target") ?? "url", headers: updated.headers ?? [], skipTlsVerification: updated.skipTlsVerification, - enabled: updated.enabled + enabled: updated.enabled, + capabilities: updated.capabilities ?? [] }); if (values.routingMode === "target" && targetsFormRef.current) { From a4d77a4fd336d2aa4068bc16b6ed72d3a7471660 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 14:45:25 -0400 Subject: [PATCH 068/296] Fix info box ui --- server/routers/aiGateway/pipeline.ts | 2 +- .../resources/private/[niceId]/layout.tsx | 2 +- .../resources/public/[niceId]/layout.tsx | 2 +- ...eInfoBox.tsx => PrivateResourceInfoBox.tsx} | 18 +++++++++++------- .../LauncherResourcePanel.tsx | 4 ++-- src/lib/launcherResourceAccess.ts | 5 ++++- 6 files changed, 20 insertions(+), 13 deletions(-) rename src/components/{SiteResourceInfoBox.tsx => PrivateResourceInfoBox.tsx} (94%) diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 247e55b4c..32690b1e1 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -461,7 +461,7 @@ export async function handleAiGatewayProxy( const def = AI_CAPABILITY_DEFS[capability]; const host = ( - (req.headers["p-host"] as string | undefined) || + (req.headers["p-host"] as string | undefined) || // p-host is only used sometimes when overriding the host header for some middleware proxy req.headers.host || "" ).split(":")[0]; diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx index f07b92673..294e8f81a 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx @@ -4,7 +4,7 @@ import { fetchSiteResourceByNiceId } from "@app/lib/fetchSiteResourceByNiceId"; import { getCachedOrg } from "@app/lib/api/getCachedOrg"; import OrgProvider from "@app/providers/OrgProvider"; import SiteResourceProvider from "@app/providers/SiteResourceProvider"; -import SiteResourceInfoBox from "@app/components/SiteResourceInfoBox"; +import SiteResourceInfoBox from "@app/components/PrivateResourceInfoBox"; import type { Metadata } from "next"; import { getTranslations } from "next-intl/server"; import { redirect } from "next/navigation"; diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx index 7eb664b5a..43a3a62c0 100644 --- a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx @@ -105,7 +105,7 @@ export default async function ResourceLayout(props: ResourceLayoutProps) { } ); - if (!env.flags.disableEnterpriseFeatures) { + if (!env.flags.disableEnterpriseFeatures && resource.mode !== "inference") { navItems.push({ title: t("maintenanceMode"), href: `/{orgId}/settings/resources/public/{niceId}/maintenance` diff --git a/src/components/SiteResourceInfoBox.tsx b/src/components/PrivateResourceInfoBox.tsx similarity index 94% rename from src/components/SiteResourceInfoBox.tsx rename to src/components/PrivateResourceInfoBox.tsx index e735ff7ae..fda9a7596 100644 --- a/src/components/SiteResourceInfoBox.tsx +++ b/src/components/PrivateResourceInfoBox.tsx @@ -80,7 +80,7 @@ function AccessMethodContent({ ); } -export function SiteResourceInfoSections({ +export function PrivateResourceInfoSections({ siteResource, access, variant, @@ -109,12 +109,16 @@ export function SiteResourceInfoSections({ udpPortRangeString: siteResource.udpPortRangeString ?? "*" }); const showAlias = - siteResource.mode !== "cidr" && siteResource.mode !== "http"; - const showDestination = !( - siteResource.mode === "ssh" && siteResource.authDaemonMode === "native" - ); + siteResource.mode !== "cidr" && + siteResource.mode !== "http" && + siteResource.mode !== "inference"; + const showDestination = + !( + siteResource.mode === "ssh" && + siteResource.authDaemonMode === "native" + ) && siteResource.mode !== "inference"; const showCertificate = !!( - siteResource.mode === "http" && + (siteResource.mode === "http" || siteResource.mode === "inference") && siteResource.ssl && siteResource.domainId && siteResource.fullDomain && @@ -258,7 +262,7 @@ export default function SiteResourceInfoBox({ }); return ( - - Date: Thu, 6 Aug 2026 14:55:07 -0400 Subject: [PATCH 069/296] Add back port restrictions --- server/routers/siteResource/createSiteResource.ts | 8 +++++--- server/routers/siteResource/updateSiteResource.ts | 10 +++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 815d7adb8..31f631ce2 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -605,7 +605,7 @@ export async function createSiteResource( } let tcpPortRangeStringAdjusted = tcpPortRangeString; - if (mode === "http") { + if (mode === "http" || mode === "inference") { tcpPortRangeStringAdjusted = "443,80"; } else if (mode === "ssh") { tcpPortRangeStringAdjusted = destinationPort @@ -628,12 +628,14 @@ export async function createSiteResource( aliasAddress, tcpPortRangeString: tcpPortRangeStringAdjusted, udpPortRangeString: - mode == "http" || mode == "ssh" + mode == "http" || mode == "ssh" || mode == "inference" ? "" : udpPortRangeString, disableIcmp: disableIcmp || - (mode == "http" || mode == "ssh" ? true : false), // default to true for http resources, otherwise false + (mode == "http" || mode == "ssh" || mode == "inference" + ? true + : false), // default to true for http resources, otherwise false domainId, subdomain: finalSubdomain, fullDomain, diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index 754c82a65..3ec8f7f65 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -558,7 +558,7 @@ export async function updateSiteResource( : {}; let tcpPortRangeStringAdjusted = tcpPortRangeString; - if (mode === "http") { + if (mode === "http" || mode === "inference") { tcpPortRangeStringAdjusted = "443,80"; } else if (mode === "ssh") { tcpPortRangeStringAdjusted = destinationPort @@ -585,13 +585,17 @@ export async function updateSiteResource( : undefined, tcpPortRangeString: tcpPortRangeStringAdjusted, udpPortRangeString: - mode == "http" || mode == "ssh" + mode == "http" || mode == "ssh" || mode == "inference" ? "" : udpPortRangeString, disableIcmp: mode !== undefined ? disableIcmp || - (mode == "http" || mode == "ssh" ? true : false) + (mode == "http" || + mode == "ssh" || + mode == "inference" + ? true + : false) : disableIcmp, domainId, subdomain: finalSubdomain, From e9f9cf54f49587a02d6c8934d212a4f06d7c71d6 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 15:18:11 -0400 Subject: [PATCH 070/296] Include the sheme in the routing header --- server/routers/aiGateway/targetRouting.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/routers/aiGateway/targetRouting.ts b/server/routers/aiGateway/targetRouting.ts index ad9f08f79..fe2598222 100644 --- a/server/routers/aiGateway/targetRouting.ts +++ b/server/routers/aiGateway/targetRouting.ts @@ -10,7 +10,7 @@ import HttpCode from "@server/types/HttpCode"; // almost immediately without needing explicit cache invalidation. const PROVIDER_TARGETS_TTL_SEC = 7; -// Header gerbil reads to know which host:port (reachable over the +// Header gerbil reads to know which scheme://host:port (reachable over the // WireGuard network) to rewrite an incoming /router/* request to. Must // match gerbil's `pangolinDestHeader` constant. const PANGOLIN_DEST_HEADER = "p-dest-header"; @@ -32,8 +32,9 @@ const SKIP_HEADERS = new Set([ type ResolvedProviderTarget = { targetId: number; - // ":", passed to gerbil as the - // destination to proxy the request to over the WireGuard tunnel. + // "://:", passed to + // gerbil as the destination to proxy the request to over the WireGuard + // tunnel. destination: string; // The target's site's exit node HTTP API base URL (gerbil's /router/*). gerbilBaseUrl: string; @@ -47,6 +48,7 @@ async function fetchProviderTargets( targetId: targets.targetId, internalPort: targets.internalPort, port: targets.port, + method: targets.method, exitNodeSubnet: sites.exitNodeSubnet, reachableAt: exitNodes.reachableAt }) @@ -66,9 +68,10 @@ async function fetchProviderTargets( } const host = row.exitNodeSubnet.split("/")[0]; const port = row.internalPort ?? row.port; + const scheme = row.method?.toLowerCase() ?? "https"; resolved.push({ targetId: row.targetId, - destination: `${host}:${port}`, + destination: `${scheme}://${host}:${port}`, gerbilBaseUrl: row.reachableAt }); } From 6c28c5f3833ea641c4ab6e4ff88b2037e29dbf6e Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 15:48:13 -0400 Subject: [PATCH 071/296] Basic target routing --- server/routers/aiGateway/targetRouting.ts | 55 +++++++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/server/routers/aiGateway/targetRouting.ts b/server/routers/aiGateway/targetRouting.ts index fe2598222..2e415b819 100644 --- a/server/routers/aiGateway/targetRouting.ts +++ b/server/routers/aiGateway/targetRouting.ts @@ -1,7 +1,15 @@ import { Request, Response } from "express"; import { and, eq } from "drizzle-orm"; import { AiProvider, db, exitNodes, sites, targets } from "@server/db"; +import config from "@server/lib/config"; +import { decrypt } from "@server/lib/crypto"; import { localCache } from "@server/lib/cache"; +import { + AiProviderAuthType, + applyAiProviderAuthHeaders, + applyAiProviderCustomHeaders, + authTypeRequiresApiKey +} from "@server/lib/aiProviderDefaults"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; @@ -15,6 +23,12 @@ const PROVIDER_TARGETS_TTL_SEC = 7; // match gerbil's `pangolinDestHeader` constant. const PANGOLIN_DEST_HEADER = "p-dest-header"; +// Header gerbil reads for the Host header value to send to the destination, +// when it should differ from PANGOLIN_DEST_HEADER (the target's configured +// ip rather than the WireGuard routing address). Must match gerbil's +// `pangolinHostHeader` constant. +const PANGOLIN_HOST_HEADER = "p-dest-host-header"; + const SKIP_HEADERS = new Set([ "p-host", "host", @@ -36,6 +50,10 @@ type ResolvedProviderTarget = { // gerbil as the destination to proxy the request to over the WireGuard // tunnel. destination: string; + // The target's configured ip, passed to gerbil as the Host header to + // send to the destination (which may differ from the WireGuard routing + // address above, e.g. for vhost-based targets). + hostHeader: string; // The target's site's exit node HTTP API base URL (gerbil's /router/*). gerbilBaseUrl: string; }; @@ -46,6 +64,7 @@ async function fetchProviderTargets( const rows = await db .select({ targetId: targets.targetId, + ip: targets.ip, internalPort: targets.internalPort, port: targets.port, method: targets.method, @@ -72,6 +91,7 @@ async function fetchProviderTargets( resolved.push({ targetId: row.targetId, destination: `${scheme}://${host}:${port}`, + hostHeader: row.ip, gerbilBaseUrl: row.reachableAt }); } @@ -117,9 +137,10 @@ function pathFromRequest(req: Request): string { * Proxies an AI gateway request to one of a "custom" / "target" routing-mode * provider's site targets, via that site's gerbil sidecar. Gerbil's * /router/* endpoint forwards the request (untouched body, same path minus - * the /router prefix, and all headers besides PANGOLIN_DEST_HEADER) over the - * WireGuard tunnel to the destination named in that header. Always writes a - * response to `res`, including on failure. + * the /router prefix, and all headers besides PANGOLIN_DEST_HEADER and + * PANGOLIN_HOST_HEADER) over the WireGuard tunnel to the destination named + * in PANGOLIN_DEST_HEADER, sending PANGOLIN_HOST_HEADER as the Host header. + * Always writes a response to `res`, including on failure. */ export async function proxyAiGatewayToSiteTarget( req: Request, @@ -147,7 +168,30 @@ export async function proxyAiGatewayToSiteTarget( } headers[key] = Array.isArray(value) ? value.join(", ") : value; } + + const authType = provider.authType as AiProviderAuthType; + let apiKey: string | null = null; + if (authTypeRequiresApiKey(authType)) { + if (!provider.apiKey) { + res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ + error: { + message: "AI provider has no API key configured" + } + }); + return; + } + const secret = config.getRawConfig().server.secret!; + apiKey = decrypt(provider.apiKey, secret); + } + applyAiProviderCustomHeaders( + headers, + provider.headers, + config.getRawConfig().server.secret! + ); + applyAiProviderAuthHeaders(headers, authType, apiKey); + headers[PANGOLIN_DEST_HEADER] = target.destination; + headers[PANGOLIN_HOST_HEADER] = target.hostHeader; const body = JSON.stringify(req.body); @@ -155,7 +199,10 @@ export async function proxyAiGatewayToSiteTarget( providerId: provider.providerId, targetId: target.targetId, destination: target.destination, - url: gerbilUrl + hostHeader: target.hostHeader, + url: gerbilUrl, + headers, + body: req.body }); let upstreamRes: globalThis.Response; From 22f2990f56ee1f9a751a12f03f949d56dfee2668 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 17:15:11 -0400 Subject: [PATCH 072/296] support streaming and closes properly on site targets --- server/lib/aiCapabilities.ts | 7 ++-- server/lib/aiGatewayUpstreamFetch.ts | 14 +++++++ server/routers/aiGateway/pipeline.ts | 39 +++++++++++++++--- server/routers/aiGateway/targetRouting.ts | 50 +++++++++++++++++++---- 4 files changed, 92 insertions(+), 18 deletions(-) diff --git a/server/lib/aiCapabilities.ts b/server/lib/aiCapabilities.ts index b6d2f7984..e899f83c3 100644 --- a/server/lib/aiCapabilities.ts +++ b/server/lib/aiCapabilities.ts @@ -62,9 +62,10 @@ export function joinUpstreamUrl(baseUrl: string, path: string): string { } function pathFromRequest(req: Request): string { - // Prefer originalUrl path (includes mounted path) over req.path when available. - const raw = - req.originalUrl?.split("?")[0] || req.url?.split("?")[0] || req.path; + // Prefer originalUrl (includes mounted path) over req.url when available. + // Query string is preserved - some providers use it to select the + // streaming response format (e.g. Gemini's `?alt=sse`). + const raw = req.originalUrl || req.url || req.path; return raw.startsWith("/") ? raw : `/${raw}`; } diff --git a/server/lib/aiGatewayUpstreamFetch.ts b/server/lib/aiGatewayUpstreamFetch.ts index e41ec23e4..7691f1f95 100644 --- a/server/lib/aiGatewayUpstreamFetch.ts +++ b/server/lib/aiGatewayUpstreamFetch.ts @@ -7,6 +7,7 @@ type UpstreamFetchInit = { headers: Record; body?: string; skipTlsVerification?: boolean; + signal?: AbortSignal; }; const insecureHttpsAgent = new https.Agent({ @@ -25,6 +26,11 @@ export function aiGatewayUpstreamFetch( isHttps && init.skipTlsVerification ? insecureHttpsAgent : undefined; return new Promise((resolve, reject) => { + if (init.signal?.aborted) { + reject(init.signal.reason ?? new Error("Request aborted")); + return; + } + const req = lib.request( url, { @@ -60,6 +66,14 @@ export function aiGatewayUpstreamFetch( req.on("error", reject); + if (init.signal) { + const onAbort = () => req.destroy(init.signal!.reason); + init.signal.addEventListener("abort", onAbort, { once: true }); + req.on("close", () => + init.signal!.removeEventListener("abort", onAbort) + ); + } + if (init.body !== undefined) { req.write(init.body); } diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 32690b1e1..ff76a5317 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -592,15 +592,33 @@ export async function handleAiGatewayProxy( skipTlsVerification: provider.skipTlsVerification }); + // Cancel the upstream request (and, transitively, anything it fans + // out to) if the client goes away before we're done - otherwise a + // client-cancelled streaming chat completion keeps running upstream + // to completion, wasting the connection and any per-token billing. + const abortController = new AbortController(); + const onClientClose = () => { + if (!res.writableEnded) { + abortController.abort(); + } + }; + res.on("close", onClientClose); + let upstreamRes: globalThis.Response; try { upstreamRes = await aiGatewayUpstreamFetch(targetUrl, { method: "POST", headers, body, - skipTlsVerification: provider.skipTlsVerification + skipTlsVerification: provider.skipTlsVerification, + signal: abortController.signal }); } catch (fetchError) { + res.off("close", onClientClose); + if (abortController.signal.aborted) { + // Client already disconnected; nothing left to respond to. + return; + } logger.error({ message: "AI gateway upstream fetch failed", url: targetUrl, @@ -628,14 +646,23 @@ export async function handleAiGatewayProxy( if (isStream && upstreamRes.body) { res.flushHeaders(); const reader = upstreamRes.body.getReader(); - while (true) { - const { done, value } = await reader.read(); - if (done) break; - res.write(value); + try { + while (!abortController.signal.aborted) { + const { done, value } = await reader.read(); + if (done) break; + res.write(value); + } + } finally { + await reader.cancel().catch(() => {}); + res.off("close", onClientClose); } - return res.end(); + if (!res.writableEnded) { + res.end(); + } + return; } + res.off("close", onClientClose); const text = await upstreamRes.text(); return res.send(text); } catch (error) { diff --git a/server/routers/aiGateway/targetRouting.ts b/server/routers/aiGateway/targetRouting.ts index 2e415b819..b0b2456d4 100644 --- a/server/routers/aiGateway/targetRouting.ts +++ b/server/routers/aiGateway/targetRouting.ts @@ -128,8 +128,10 @@ function pickTarget( } function pathFromRequest(req: Request): string { - const raw = - req.originalUrl?.split("?")[0] || req.url?.split("?")[0] || req.path; + // Query string is preserved - some providers use it to select the + // streaming response format (e.g. Gemini's `?alt=sse`), and gerbil's + // /router/* forwards it through untouched. + const raw = req.originalUrl || req.url || req.path; return raw.startsWith("/") ? raw : `/${raw}`; } @@ -205,14 +207,32 @@ export async function proxyAiGatewayToSiteTarget( body: req.body }); + // Cancel the request to gerbil (which cascades to gerbil cancelling its + // proxied request to the actual site target, since gerbil's reverse + // proxy derives the outbound request's context from the inbound one) if + // the client goes away before we're done. + const abortController = new AbortController(); + const onClientClose = () => { + if (!res.writableEnded) { + abortController.abort(); + } + }; + res.on("close", onClientClose); + let upstreamRes: globalThis.Response; try { upstreamRes = await fetch(gerbilUrl, { method: "POST", headers, - body + body, + signal: abortController.signal }); } catch (fetchError) { + res.off("close", onClientClose); + if (abortController.signal.aborted) { + // Client already disconnected; nothing left to respond to. + return; + } logger.error({ message: "AI gateway target proxy request failed", url: gerbilUrl, @@ -232,7 +252,11 @@ export async function proxyAiGatewayToSiteTarget( const contentType = upstreamRes.headers.get("content-type") || ""; const isStream = req.body?.stream === true || - contentType.includes("text/event-stream"); + contentType.includes("text/event-stream") || + pathFromRequest(req).includes("streamGenerateContent") || + pathFromRequest(req).includes("streamRawPredict") || + pathFromRequest(req).includes("converse-stream") || + pathFromRequest(req).includes("invoke-with-response-stream"); res.status(upstreamRes.status); res.setHeader("Content-Type", contentType || "application/json"); @@ -240,15 +264,23 @@ export async function proxyAiGatewayToSiteTarget( if (isStream && upstreamRes.body) { res.flushHeaders(); const reader = upstreamRes.body.getReader(); - while (true) { - const { done, value } = await reader.read(); - if (done) break; - res.write(value); + try { + while (!abortController.signal.aborted) { + const { done, value } = await reader.read(); + if (done) break; + res.write(value); + } + } finally { + await reader.cancel().catch(() => {}); + res.off("close", onClientClose); + } + if (!res.writableEnded) { + res.end(); } - res.end(); return; } + res.off("close", onClientClose); const text = await upstreamRes.text(); res.send(text); } From 184e1425a41b57a3375221addf104ccaba7d407c Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 17:22:01 -0400 Subject: [PATCH 073/296] Private connection working with traefik --- server/private/lib/traefik/getTraefikConfig.ts | 2 +- server/routers/aiGateway/pipeline.ts | 2 +- src/components/PrivateResourcesTable.tsx | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index b00dbd7c8..f51dad24e 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -1696,7 +1696,7 @@ export async function getTraefikConfig( const srKey = `inference-sr${sr.siteResourceId}`; const routerName = `${srKey}-router`; const serviceName = `${srKey}-service`; - const rule = `Host(\`${fullDomain}\`) && ClientIP(${exitNode.address})`; // restrict to coming from the exit node ip range that the client is connected to + const rule = `Host(\`${fullDomain}\`) && ClientIP(\`${exitNode.address}\`)`; // restrict to coming from the exit node ip range that the client is connected to let tls: any = {}; if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index ff76a5317..08ea6d1bc 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -273,7 +273,7 @@ async function resolveTarget(host: string): Promise { .from(siteResources) .where( and( - eq(siteResources.alias, host), + eq(siteResources.fullDomain, host), eq(siteResources.mode, "inference"), eq(siteResources.enabled, true) ) diff --git a/src/components/PrivateResourcesTable.tsx b/src/components/PrivateResourcesTable.tsx index cb7d2645c..bc77fbc38 100644 --- a/src/components/PrivateResourcesTable.tsx +++ b/src/components/PrivateResourcesTable.tsx @@ -421,7 +421,10 @@ export default function PrivateResourcesTable({ /> ); } - if (resourceRow.mode === "http") { + if ( + resourceRow.mode === "http" || + resourceRow.mode === "inference" + ) { const domainId = resourceRow.domainId; const fullDomain = resourceRow.fullDomain; const url = `${resourceRow.ssl ? "https" : "http"}://${fullDomain}`; From 07f628b9287b4ea80e7506d9ff8be1023154c2c3 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 6 Aug 2026 17:53:51 -0400 Subject: [PATCH 074/296] add the pangolin header information --- server/routers/aiGateway/pipeline.ts | 30 ++++++++++++++++++++++- server/routers/aiGateway/targetRouting.ts | 8 +++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 08ea6d1bc..04d318004 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -119,6 +119,28 @@ export type RequestUser = { role: string | null; }; +// Identity headers forwarded to the upstream inference endpoint when the +// requesting user is known. Omitted entirely (not sent empty) when we +// couldn't resolve a user for the request. +export function applyRequestUserHeaders( + headers: Record, + requestUser: RequestUser | null +): void { + if (!requestUser) { + return; + } + headers["Remote-User"] = requestUser.username; + if (requestUser.email) { + headers["Remote-Email"] = requestUser.email; + } + if (requestUser.name) { + headers["Remote-Name"] = requestUser.name; + } + if (requestUser.role) { + headers["Remote-Role"] = requestUser.role; + } +} + async function buildRequestUser( userId: string, orgId: string | null @@ -519,7 +541,12 @@ export async function handleAiGatewayProxy( const { provider } = selection; if (provider.type === "custom" && provider.routingMode === "target") { - return await proxyAiGatewayToSiteTarget(req, res, provider); + return await proxyAiGatewayToSiteTarget( + req, + res, + provider, + requestUser + ); } const upstreamUrl = provider.upstreamUrl; @@ -580,6 +607,7 @@ export async function handleAiGatewayProxy( config.getRawConfig().server.secret! ); applyAiProviderAuthHeaders(headers, authType, apiKey); + applyRequestUserHeaders(headers, requestUser); const body = JSON.stringify(req.body); diff --git a/server/routers/aiGateway/targetRouting.ts b/server/routers/aiGateway/targetRouting.ts index b0b2456d4..b091bf8f5 100644 --- a/server/routers/aiGateway/targetRouting.ts +++ b/server/routers/aiGateway/targetRouting.ts @@ -12,6 +12,10 @@ import { } from "@server/lib/aiProviderDefaults"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; +import { + applyRequestUserHeaders, + type RequestUser +} from "@server/routers/aiGateway/pipeline"; // Short TTL: long enough to spare the DB on a burst of requests, short // enough that target/site changes (added, removed, exit node moved) show up @@ -147,7 +151,8 @@ function pathFromRequest(req: Request): string { export async function proxyAiGatewayToSiteTarget( req: Request, res: Response, - provider: AiProvider + provider: AiProvider, + requestUser: RequestUser | null ): Promise { const providerTargets = await getProviderTargets(provider.providerId); if (providerTargets.length === 0) { @@ -191,6 +196,7 @@ export async function proxyAiGatewayToSiteTarget( config.getRawConfig().server.secret! ); applyAiProviderAuthHeaders(headers, authType, apiKey); + applyRequestUserHeaders(headers, requestUser); headers[PANGOLIN_DEST_HEADER] = target.destination; headers[PANGOLIN_HOST_HEADER] = target.hostHeader; From 9e7b4afaec6aa7495f8acaf3d350d079842fc7a1 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 7 Aug 2026 10:59:35 -0400 Subject: [PATCH 075/296] support allow list and blocklist --- messages/en-US.json | 13 +- server/db/pg/schema/schema.ts | 28 +- server/db/sqlite/schema/schema.ts | 28 +- server/lib/aiCapabilities.ts | 35 +- server/lib/aiInferenceResource.ts | 576 +++++++++++------- server/lib/aiModelKeyMatch.ts | 39 ++ server/lib/aiProviderDefaults.ts | 2 +- server/routers/aiGateway/pipeline.ts | 203 +++--- server/routers/aiProvider/createAiModel.ts | 7 +- server/routers/aiProvider/updateAiModel.ts | 7 +- .../routers/resource/addAiModelToResource.ts | 23 +- .../resource/addAiProviderToResource.ts | 19 +- server/routers/resource/createResource.ts | 9 +- .../routers/resource/listResourceAiModels.ts | 6 +- .../resource/listResourceAiProviders.ts | 3 +- .../resource/removeAiModelFromResource.ts | 6 +- .../resource/removeAiProviderFromResource.ts | 5 +- .../routers/resource/setResourceAiModels.ts | 38 +- .../resource/setResourceAiProviders.ts | 5 +- .../siteResource/addAiModelToSiteResource.ts | 26 +- .../addAiProviderToSiteResource.ts | 19 +- .../siteResource/createSiteResource.ts | 9 +- .../siteResource/listSiteResourceAiModels.ts | 6 +- .../listSiteResourceAiProviders.ts | 3 +- .../removeAiModelFromSiteResource.ts | 6 +- .../removeAiProviderFromSiteResource.ts | 5 +- .../siteResource/setSiteResourceAiModels.ts | 31 +- .../setSiteResourceAiProviders.ts | 5 +- .../ai-providers/[providerId]/models/page.tsx | 177 ++++-- .../private/[niceId]/inference/page.tsx | 9 +- .../public/[niceId]/inference/page.tsx | 3 +- .../settings/resources/public/create/page.tsx | 3 +- src/lib/privateResourceForm.ts | 3 +- src/lib/queries.ts | 4 +- 34 files changed, 891 insertions(+), 470 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index e994c823c..8cc780829 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1667,7 +1667,7 @@ "aiProviderTypeCustom": "Custom", "aiProviderTypeOpenaiDescription": "OpenAI API with default upstream URL", "aiProviderTypeAnthropicDescription": "Anthropic API with default upstream URL", - "aiProviderTypeGoogleGeminiDescription": "Google Gemini OpenAI-compatible endpoint", + "aiProviderTypeGoogleGeminiDescription": "Google Gemini generateContent API", "aiProviderTypeVertexAiDescription": "Google Vertex AI; upstream URL required", "aiProviderTypeBedrockDescription": "Amazon Bedrock Runtime", "aiProviderTypeMicrosoftFoundryDescription": "Microsoft Foundry; upstream URL required", @@ -1759,13 +1759,20 @@ "aiProviderMessageRemove": "This will permanently delete the provider and its models and targets. This cannot be undone.", "aiProviderErrorNoUpdate": "AI provider is not available to update", "aiProviderModels": "Models", - "aiProviderModelsDescription": "Define model names available on this provider. Requests must match one of these keys. Use * and ? as wildcards (for example gpt-4* or claude-?).", + "aiProviderModelsDescription": "Define allow and block patterns for this provider. Requests must match an allow pattern and must not match a block pattern. Use * and ? as wildcards (for example gpt-4* or claude-?). An empty allow list denies all models.", "aiProviderModelsPlaceholder": "Model name or pattern (e.g. gpt-4*)", + "aiProviderModelsAllow": "Allow List", + "aiProviderModelsAllowDescription": "Models that may be used through this provider. Empty means deny all.", + "aiProviderModelsAllowPlaceholder": "Allowed model or pattern (e.g. gpt-4*)", + "aiProviderModelsBlock": "Block List", + "aiProviderModelsBlockDescription": "Models to deny even if they match an allow pattern.", + "aiProviderModelsBlockPlaceholder": "Blocked model or pattern (e.g. gpt-4o-mini)", + "aiProviderModelsOverlapError": "These patterns cannot be on both lists: {keys}", "aiProviderModelsUpdated": "Models updated", "aiProviderModelsErrorUpdate": "Failed to update models", "aiResourceProviders": "Providers", "aiResourceProvidersDescription": "Choose which AI providers this inference resource can use", - "aiResourceProvidersHelp": "Models must be defined on each provider. Exact names and patterns that conflict (identical keys, or an exact key matching another provider's pattern) are not allowed across selected providers.", + "aiResourceProvidersHelp": "Each attached provider uses its own allow and block lists. Allow patterns that conflict across attached providers are not allowed.", "aiResourceProvidersSelect": "Select providers", "aiResourceProvidersEmpty": "No AI providers found", "aiResourceProvidersUpdated": "Providers updated", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 6bc038660..90dd5abd9 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -231,10 +231,10 @@ export const resourceAiProviders = pgTable( providerId: integer("providerId") .notNull() .references(() => aiProviders.providerId, { onDelete: "cascade" }), - modelAccessMode: varchar("modelAccessMode") - .$type<"catalog" | "allowlist">() + accessMode: varchar("accessMode") + .$type<"inherit" | "select">() .notNull() - .default("catalog") + .default("inherit") }, (t) => [primaryKey({ columns: [t.resourceId, t.providerId] })] ); @@ -247,7 +247,11 @@ export const resourceAiModels = pgTable( .references(() => resources.resourceId, { onDelete: "cascade" }), modelId: integer("modelId") .notNull() - .references(() => aiModels.modelId, { onDelete: "cascade" }) + .references(() => aiModels.modelId, { onDelete: "cascade" }), + listType: varchar("listType") + .$type<"allow" | "block">() + .notNull() + .default("allow") }, (t) => [primaryKey({ columns: [t.resourceId, t.modelId] })] ); @@ -522,10 +526,10 @@ export const siteResourceAiProviders = pgTable( providerId: integer("providerId") .notNull() .references(() => aiProviders.providerId, { onDelete: "cascade" }), - modelAccessMode: varchar("modelAccessMode") - .$type<"catalog" | "allowlist">() + accessMode: varchar("accessMode") + .$type<"inherit" | "select">() .notNull() - .default("catalog") + .default("inherit") }, (t) => [primaryKey({ columns: [t.siteResourceId, t.providerId] })] ); @@ -540,7 +544,11 @@ export const siteResourceAiModels = pgTable( }), modelId: integer("modelId") .notNull() - .references(() => aiModels.modelId, { onDelete: "cascade" }) + .references(() => aiModels.modelId, { onDelete: "cascade" }), + listType: varchar("listType") + .$type<"allow" | "block">() + .notNull() + .default("allow") }, (t) => [primaryKey({ columns: [t.siteResourceId, t.modelId] })] ); @@ -1678,6 +1686,10 @@ export const aiModels = pgTable( .references(() => aiProviders.providerId, { onDelete: "cascade" }), modelKey: varchar("modelKey").notNull(), name: varchar("name").notNull(), + listType: varchar("listType") + .$type<"allow" | "block">() + .notNull() + .default("allow"), enabled: boolean("enabled").notNull().default(true), createdAt: bigint("createdAt", { mode: "number" }).notNull(), updatedAt: bigint("updatedAt", { mode: "number" }).notNull() diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 8130af9ea..dab445282 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -228,10 +228,10 @@ export const resourceAiProviders = sqliteTable( providerId: integer("providerId") .notNull() .references(() => aiProviders.providerId, { onDelete: "cascade" }), - modelAccessMode: text("modelAccessMode") - .$type<"catalog" | "allowlist">() + accessMode: text("accessMode") + .$type<"inherit" | "select">() .notNull() - .default("catalog") + .default("inherit") }, (t) => [primaryKey({ columns: [t.resourceId, t.providerId] })] ); @@ -244,7 +244,11 @@ export const resourceAiModels = sqliteTable( .references(() => resources.resourceId, { onDelete: "cascade" }), modelId: integer("modelId") .notNull() - .references(() => aiModels.modelId, { onDelete: "cascade" }) + .references(() => aiModels.modelId, { onDelete: "cascade" }), + listType: text("listType") + .$type<"allow" | "block">() + .notNull() + .default("allow") }, (t) => [primaryKey({ columns: [t.resourceId, t.modelId] })] ); @@ -507,10 +511,10 @@ export const siteResourceAiProviders = sqliteTable( providerId: integer("providerId") .notNull() .references(() => aiProviders.providerId, { onDelete: "cascade" }), - modelAccessMode: text("modelAccessMode") - .$type<"catalog" | "allowlist">() + accessMode: text("accessMode") + .$type<"inherit" | "select">() .notNull() - .default("catalog") + .default("inherit") }, (t) => [primaryKey({ columns: [t.siteResourceId, t.providerId] })] ); @@ -525,7 +529,11 @@ export const siteResourceAiModels = sqliteTable( }), modelId: integer("modelId") .notNull() - .references(() => aiModels.modelId, { onDelete: "cascade" }) + .references(() => aiModels.modelId, { onDelete: "cascade" }), + listType: text("listType") + .$type<"allow" | "block">() + .notNull() + .default("allow") }, (t) => [primaryKey({ columns: [t.siteResourceId, t.modelId] })] ); @@ -1660,6 +1668,10 @@ export const aiModels = sqliteTable( .references(() => aiProviders.providerId, { onDelete: "cascade" }), modelKey: text("modelKey").notNull(), name: text("name").notNull(), + listType: text("listType") + .$type<"allow" | "block">() + .notNull() + .default("allow"), enabled: integer("enabled", { mode: "boolean" }) .notNull() .default(true), diff --git a/server/lib/aiCapabilities.ts b/server/lib/aiCapabilities.ts index e899f83c3..f7b370691 100644 --- a/server/lib/aiCapabilities.ts +++ b/server/lib/aiCapabilities.ts @@ -40,18 +40,37 @@ function paramModel(req: Request): string | undefined { } /** - * Join base URL with a path, avoiding double slashes and a duplicated trailing - * /v1 when the inbound path already starts with /v1 and the base ends with /v1. + * Join a provider base URL with an inbound request path. */ export function joinUpstreamUrl(baseUrl: string, path: string): string { const base = baseUrl.replace(/\/+$/, ""); let suffix = path.startsWith("/") ? path : `/${path}`; - if ( - base.endsWith("/v1") && - (suffix === "/v1" || suffix.startsWith("/v1/")) - ) { - suffix = suffix.slice("/v1".length) || "/"; + let basePathname = "/"; + try { + basePathname = new URL(base).pathname.replace(/\/+$/, "") || "/"; + } catch { + // Fall through with "/" non-absolute bases are not expected in + // production, but keep joining usable for malformed input. + } + + if (basePathname !== "/") { + const baseSegs = basePathname.split("/").filter(Boolean); + const pathSegs = suffix.split("/").filter(Boolean); + const max = Math.min(baseSegs.length, pathSegs.length); + let overlap = 0; + for (let n = max; n >= 1; n--) { + const baseSuffix = baseSegs.slice(-n); + const pathPrefix = pathSegs.slice(0, n); + if (baseSuffix.every((seg, i) => seg === pathPrefix[i])) { + overlap = n; + break; + } + } + if (overlap > 0) { + const remaining = pathSegs.slice(overlap); + suffix = remaining.length > 0 ? `/${remaining.join("/")}` : "/"; + } } if (suffix === "/") { @@ -175,7 +194,7 @@ export const AI_PROVIDER_CAPABILITY_DEFAULTS: Record< > = { openai: ["openai_chat"], anthropic: ["anthropic_messages"], - googleGemini: ["openai_chat"], + googleGemini: ["gemini_generate_content"], vertexAi: ["google_generate_content"], bedrock: ["bedrock_converse"], microsoftFoundry: ["openai_chat"], diff --git a/server/lib/aiInferenceResource.ts b/server/lib/aiInferenceResource.ts index d01b30dd9..51333473a 100644 --- a/server/lib/aiInferenceResource.ts +++ b/server/lib/aiInferenceResource.ts @@ -14,13 +14,17 @@ import { modelKeysConflict } from "@server/lib/aiModelKeyMatch"; type DbOrTrx = Transaction | typeof db; -export const modelAccessModeSchema = z.enum(["catalog", "allowlist"]); +export const modelListTypeSchema = z.enum(["allow", "block"]); -export type ModelAccessMode = z.infer; +export type ModelListType = z.infer; + +export const accessModeSchema = z.enum(["inherit", "select"]); + +export type AccessMode = z.infer; export const resourceAiProviderAttachmentSchema = z.strictObject({ providerId: z.number().int().positive(), - modelAccessMode: modelAccessModeSchema.optional() + accessMode: accessModeSchema.optional().default("inherit") }); export type ResourceAiProviderInput = z.infer< @@ -29,9 +33,16 @@ export type ResourceAiProviderInput = z.infer< export type ResourceAiProviderAttachment = { providerId: number; - modelAccessMode: ModelAccessMode; + accessMode: AccessMode; }; +export const resourceAiModelEntrySchema = z.strictObject({ + modelId: z.number().int().positive(), + listType: modelListTypeSchema +}); + +export type ResourceAiModelEntry = z.infer; + export type InferenceFieldsError = { error: string; }; @@ -42,58 +53,153 @@ export function isInferenceFieldsError( return "error" in value; } +/** + * Resolve which allow/block patterns apply for an attachment. + * inherit → provider lists; select → resource-selected lists (replace). + */ +export function resolveEffectiveLists(input: { + accessMode: AccessMode; + providerAllows: string[]; + providerBlocks: string[]; + resourceAllows: string[]; + resourceBlocks: string[]; +}): { allows: string[]; blocks: string[] } { + if (input.accessMode === "select") { + return { + allows: input.resourceAllows, + blocks: input.resourceBlocks + }; + } + return { + allows: input.providerAllows, + blocks: input.providerBlocks + }; +} + function normalizeAttachments( inputs: ResourceAiProviderInput[] ): ResourceAiProviderAttachment[] { - const byProvider = new Map(); + const byProviderId = new Map(); for (const input of inputs) { - byProvider.set(input.providerId, input.modelAccessMode ?? "catalog"); + byProviderId.set(input.providerId, input.accessMode ?? "inherit"); } - return [...byProvider.entries()].map(([providerId, modelAccessMode]) => ({ + return [...byProviderId.entries()].map(([providerId, accessMode]) => ({ providerId, - modelAccessMode + accessMode })); } +type EffectiveAllowRow = { + providerId: number; + modelKey: string; +}; + /** - * Ensure enabled catalog modelKeys do not conflict across attached providers. - * Catalog attachments contribute all enabled models on the provider. - * Allowlist attachments contribute nothing until models are allowlisted - * (those are checked when the allowlist is set). - * - * Conflicts: identical keys, or an exact key that matches another provider's - * pattern. Full glob intersections are left to runtime ambiguity errors. + * Ensure effective allow modelKeys do not conflict across attached providers. + * inherit uses provider allows; select uses resource-selected allows (or the + * optional override map). Block patterns are ignored for overlap checks. */ export async function assertNoOverlappingModelKeys( attachments: ResourceAiProviderAttachment[], - trx: DbOrTrx = db + options: { + trx?: DbOrTrx; + resourceId?: number; + siteResourceId?: number; + selectedAllowsByProvider?: Map; + } = {} ): Promise { - const catalogProviderIds = attachments - .filter((a) => a.modelAccessMode === "catalog") - .map((a) => a.providerId); + const trx = options.trx ?? db; - if (catalogProviderIds.length < 2) { + if (attachments.length < 2) { return null; } - const models = await trx - .select({ - providerId: aiModels.providerId, - modelKey: aiModels.modelKey - }) - .from(aiModels) - .where( - and( - inArray(aiModels.providerId, catalogProviderIds), - eq(aiModels.enabled, true) - ) - ); + const inheritProviderIds = attachments + .filter((a) => a.accessMode === "inherit") + .map((a) => a.providerId); + const selectProviderIds = attachments + .filter((a) => a.accessMode === "select") + .map((a) => a.providerId); + + const effectiveAllows: EffectiveAllowRow[] = []; + + if (inheritProviderIds.length > 0) { + const providerAllows = await trx + .select({ + providerId: aiModels.providerId, + modelKey: aiModels.modelKey + }) + .from(aiModels) + .where( + and( + inArray(aiModels.providerId, inheritProviderIds), + eq(aiModels.enabled, true), + eq(aiModels.listType, "allow") + ) + ); + effectiveAllows.push(...providerAllows); + } + + if (selectProviderIds.length > 0) { + if (options.selectedAllowsByProvider) { + for (const providerId of selectProviderIds) { + const keys = + options.selectedAllowsByProvider.get(providerId) ?? []; + for (const modelKey of keys) { + effectiveAllows.push({ providerId, modelKey }); + } + } + } else if (options.resourceId !== undefined) { + const rows = await trx + .select({ + providerId: aiModels.providerId, + modelKey: aiModels.modelKey + }) + .from(resourceAiModels) + .innerJoin( + aiModels, + eq(resourceAiModels.modelId, aiModels.modelId) + ) + .where( + and( + eq(resourceAiModels.resourceId, options.resourceId), + inArray(aiModels.providerId, selectProviderIds), + eq(resourceAiModels.listType, "allow"), + eq(aiModels.enabled, true) + ) + ); + effectiveAllows.push(...rows); + } else if (options.siteResourceId !== undefined) { + const rows = await trx + .select({ + providerId: aiModels.providerId, + modelKey: aiModels.modelKey + }) + .from(siteResourceAiModels) + .innerJoin( + aiModels, + eq(siteResourceAiModels.modelId, aiModels.modelId) + ) + .where( + and( + eq( + siteResourceAiModels.siteResourceId, + options.siteResourceId + ), + inArray(aiModels.providerId, selectProviderIds), + eq(siteResourceAiModels.listType, "allow"), + eq(aiModels.enabled, true) + ) + ); + effectiveAllows.push(...rows); + } + } const conflictPairs: string[] = []; - for (let i = 0; i < models.length; i++) { - for (let j = i + 1; j < models.length; j++) { - const left = models[i]; - const right = models[j]; + for (let i = 0; i < effectiveAllows.length; i++) { + for (let j = i + 1; j < effectiveAllows.length; j++) { + const left = effectiveAllows[i]; + const right = effectiveAllows[j]; if (left.providerId === right.providerId) { continue; } @@ -124,6 +230,8 @@ export async function resolveProviderAttachments(input: { orgId: string; attachments: ResourceAiProviderInput[]; requireAtLeastOne: boolean; + resourceId?: number; + siteResourceId?: number; }): Promise { const attachments = normalizeAttachments(input.attachments); @@ -165,7 +273,10 @@ export async function resolveProviderAttachments(input: { }; } - const overlapError = await assertNoOverlappingModelKeys(attachments); + const overlapError = await assertNoOverlappingModelKeys(attachments, { + resourceId: input.resourceId, + siteResourceId: input.siteResourceId + }); if (overlapError) { return overlapError; } @@ -188,6 +299,11 @@ export async function assertInferenceModeAllowsProviderFields(input: { return null; } +/** + * Attach providers to a resource. Inherit attachments use the provider lists + * as-is (resource model rows for those providers are pruned). Select + * attachments keep resource-selected allow/block subsets. + */ export async function setPublicResourceAiProviders( resourceId: number, attachments: ResourceAiProviderAttachment[], @@ -202,12 +318,16 @@ export async function setPublicResourceAiProviders( attachments.map((a) => ({ resourceId, providerId: a.providerId, - modelAccessMode: a.modelAccessMode + accessMode: a.accessMode })) ); } - await prunePublicResourceAllowlistToAllowlistProviders(resourceId, trx); + await prunePublicResourceModelsToSelectProviders( + resourceId, + attachments, + trx + ); } export async function setSiteResourceAiProviders( @@ -224,12 +344,103 @@ export async function setSiteResourceAiProviders( attachments.map((a) => ({ siteResourceId, providerId: a.providerId, - modelAccessMode: a.modelAccessMode + accessMode: a.accessMode })) ); } - await pruneSiteResourceAllowlistToAllowlistProviders(siteResourceId, trx); + await pruneSiteResourceModelsToSelectProviders( + siteResourceId, + attachments, + trx + ); +} + +/** + * Keep resource model rows only for providers in select mode. + */ +async function prunePublicResourceModelsToSelectProviders( + resourceId: number, + attachments: ResourceAiProviderAttachment[], + trx: DbOrTrx +): Promise { + const selectProviderIds = attachments + .filter((a) => a.accessMode === "select") + .map((a) => a.providerId); + + if (selectProviderIds.length === 0) { + await trx + .delete(resourceAiModels) + .where(eq(resourceAiModels.resourceId, resourceId)); + return; + } + + const existing = await trx + .select({ + modelId: resourceAiModels.modelId, + providerId: aiModels.providerId + }) + .from(resourceAiModels) + .innerJoin(aiModels, eq(resourceAiModels.modelId, aiModels.modelId)) + .where(eq(resourceAiModels.resourceId, resourceId)); + + const allowed = new Set(selectProviderIds); + const toRemove = existing + .filter((row) => !allowed.has(row.providerId)) + .map((row) => row.modelId); + + if (toRemove.length > 0) { + await trx + .delete(resourceAiModels) + .where( + and( + eq(resourceAiModels.resourceId, resourceId), + inArray(resourceAiModels.modelId, toRemove) + ) + ); + } +} + +async function pruneSiteResourceModelsToSelectProviders( + siteResourceId: number, + attachments: ResourceAiProviderAttachment[], + trx: DbOrTrx +): Promise { + const selectProviderIds = attachments + .filter((a) => a.accessMode === "select") + .map((a) => a.providerId); + + if (selectProviderIds.length === 0) { + await trx + .delete(siteResourceAiModels) + .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); + return; + } + + const existing = await trx + .select({ + modelId: siteResourceAiModels.modelId, + providerId: aiModels.providerId + }) + .from(siteResourceAiModels) + .innerJoin(aiModels, eq(siteResourceAiModels.modelId, aiModels.modelId)) + .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); + + const allowed = new Set(selectProviderIds); + const toRemove = existing + .filter((row) => !allowed.has(row.providerId)) + .map((row) => row.modelId); + + if (toRemove.length > 0) { + await trx + .delete(siteResourceAiModels) + .where( + and( + eq(siteResourceAiModels.siteResourceId, siteResourceId), + inArray(siteResourceAiModels.modelId, toRemove) + ) + ); + } } export async function clearPublicResourceAiConfig( @@ -256,120 +467,14 @@ export async function clearSiteResourceAiConfig( .where(eq(siteResourceAiProviders.siteResourceId, siteResourceId)); } -async function prunePublicResourceAllowlistToAllowlistProviders( - resourceId: number, - trx: DbOrTrx = db -): Promise { - const allowlistProviders = await trx - .select({ providerId: resourceAiProviders.providerId }) - .from(resourceAiProviders) - .where( - and( - eq(resourceAiProviders.resourceId, resourceId), - eq(resourceAiProviders.modelAccessMode, "allowlist") - ) - ); - - if (allowlistProviders.length === 0) { - await trx - .delete(resourceAiModels) - .where(eq(resourceAiModels.resourceId, resourceId)); - return; - } - - const validModels = await trx - .select({ modelId: aiModels.modelId }) - .from(aiModels) - .where( - inArray( - aiModels.providerId, - allowlistProviders.map((p) => p.providerId) - ) - ); - const validIds = validModels.map((m) => m.modelId); - - const existing = await trx - .select({ modelId: resourceAiModels.modelId }) - .from(resourceAiModels) - .where(eq(resourceAiModels.resourceId, resourceId)); - - const toRemove = existing - .map((e) => e.modelId) - .filter((id) => !validIds.includes(id)); - - if (toRemove.length > 0) { - await trx - .delete(resourceAiModels) - .where( - and( - eq(resourceAiModels.resourceId, resourceId), - inArray(resourceAiModels.modelId, toRemove) - ) - ); - } -} - -async function pruneSiteResourceAllowlistToAllowlistProviders( - siteResourceId: number, - trx: DbOrTrx = db -): Promise { - const allowlistProviders = await trx - .select({ providerId: siteResourceAiProviders.providerId }) - .from(siteResourceAiProviders) - .where( - and( - eq(siteResourceAiProviders.siteResourceId, siteResourceId), - eq(siteResourceAiProviders.modelAccessMode, "allowlist") - ) - ); - - if (allowlistProviders.length === 0) { - await trx - .delete(siteResourceAiModels) - .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); - return; - } - - const validModels = await trx - .select({ modelId: aiModels.modelId }) - .from(aiModels) - .where( - inArray( - aiModels.providerId, - allowlistProviders.map((p) => p.providerId) - ) - ); - const validIds = validModels.map((m) => m.modelId); - - const existing = await trx - .select({ modelId: siteResourceAiModels.modelId }) - .from(siteResourceAiModels) - .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); - - const toRemove = existing - .map((e) => e.modelId) - .filter((id) => !validIds.includes(id)); - - if (toRemove.length > 0) { - await trx - .delete(siteResourceAiModels) - .where( - and( - eq(siteResourceAiModels.siteResourceId, siteResourceId), - inArray(siteResourceAiModels.modelId, toRemove) - ) - ); - } -} - export async function listPublicResourceAiProviders(resourceId: number) { return db .select({ providerId: resourceAiProviders.providerId, - modelAccessMode: resourceAiProviders.modelAccessMode, name: aiProviders.name, type: aiProviders.type, - enabled: aiProviders.enabled + enabled: aiProviders.enabled, + accessMode: resourceAiProviders.accessMode }) .from(resourceAiProviders) .innerJoin( @@ -383,10 +488,10 @@ export async function listSiteResourceAiProviders(siteResourceId: number) { return db .select({ providerId: siteResourceAiProviders.providerId, - modelAccessMode: siteResourceAiProviders.modelAccessMode, name: aiProviders.name, type: aiProviders.type, - enabled: aiProviders.enabled + enabled: aiProviders.enabled, + accessMode: siteResourceAiProviders.accessMode }) .from(siteResourceAiProviders) .innerJoin( @@ -397,15 +502,15 @@ export async function listSiteResourceAiProviders(siteResourceId: number) { } /** - * Allowlist APIs require an inference resource with at least one - * attached provider in allowlist mode. + * Model list APIs require an inference resource with at least one select-mode + * attached provider. */ -export async function assertPublicAllowlistApiEligible(resource: { +export async function assertPublicModelListApiEligible(resource: { resourceId: number; mode: string; }): Promise { if (resource.mode !== "inference") { - return "AI model allowlists are only supported on inference-mode resources"; + return "AI model lists are only supported on inference-mode resources"; } const [row] = await db @@ -414,23 +519,23 @@ export async function assertPublicAllowlistApiEligible(resource: { .where( and( eq(resourceAiProviders.resourceId, resource.resourceId), - eq(resourceAiProviders.modelAccessMode, "allowlist") + eq(resourceAiProviders.accessMode, "select") ) ) .limit(1); if (!row) { - return "Attach at least one AI provider with modelAccessMode=allowlist before managing allowed models"; + return "Set at least one attached AI provider to select mode before managing model lists"; } return null; } -export async function assertSiteAllowlistApiEligible(siteResource: { +export async function assertSiteModelListApiEligible(siteResource: { siteResourceId: number; mode: string; }): Promise { if (siteResource.mode !== "inference") { - return "AI model allowlists are only supported on inference-mode resources"; + return "AI model lists are only supported on inference-mode resources"; } const [row] = await db @@ -442,33 +547,36 @@ export async function assertSiteAllowlistApiEligible(siteResource: { siteResourceAiProviders.siteResourceId, siteResource.siteResourceId ), - eq(siteResourceAiProviders.modelAccessMode, "allowlist") + eq(siteResourceAiProviders.accessMode, "select") ) ) .limit(1); if (!row) { - return "Attach at least one AI provider with modelAccessMode=allowlist before managing allowed models"; + return "Set at least one attached AI provider to select mode before managing model lists"; } return null; } /** - * Models must belong to providers attached to this resource in allowlist mode, - * and those providers must belong to the resource's org. + * Resource model entries must belong to select-mode attached providers, and + * listType must match the provider catalog entry (allow→allow, block→block). */ -export async function assertModelsBelongToPublicAllowlistProviders(input: { +export async function assertPublicResourceModelEntriesValid(input: { orgId: string; resourceId: number; - modelIds: number[]; + models: ResourceAiModelEntry[]; }): Promise { - const uniqueIds = [...new Set(input.modelIds)]; - if (uniqueIds.length === 0) { + const uniqueModels = dedupeModelEntries(input.models); + if (uniqueModels.length === 0) { return null; } - const allowlistProviders = await db - .select({ providerId: resourceAiProviders.providerId }) + const attachments = await db + .select({ + providerId: resourceAiProviders.providerId, + accessMode: resourceAiProviders.accessMode + }) .from(resourceAiProviders) .innerJoin( aiProviders, @@ -477,49 +585,33 @@ export async function assertModelsBelongToPublicAllowlistProviders(input: { .where( and( eq(resourceAiProviders.resourceId, input.resourceId), - eq(resourceAiProviders.modelAccessMode, "allowlist"), eq(aiProviders.orgId, input.orgId) ) ); - if (allowlistProviders.length === 0) { - return "No allowlist AI providers are attached to this resource"; - } - - const validModels = await db - .select({ modelId: aiModels.modelId }) - .from(aiModels) - .innerJoin(aiProviders, eq(aiModels.providerId, aiProviders.providerId)) - .where( - and( - inArray(aiModels.modelId, uniqueIds), - inArray( - aiModels.providerId, - allowlistProviders.map((p) => p.providerId) - ), - eq(aiProviders.orgId, input.orgId) - ) - ); - - if (validModels.length !== uniqueIds.length) { - return "One or more model IDs do not exist or do not belong to an allowlist provider on this resource"; - } - - return null; + return assertModelEntriesValid({ + orgId: input.orgId, + modelEntries: uniqueModels, + attachments, + resourceLabel: "resource" + }); } -export async function assertModelsBelongToSiteAllowlistProviders(input: { +export async function assertSiteResourceModelEntriesValid(input: { orgId: string; siteResourceId: number; - modelIds: number[]; + models: ResourceAiModelEntry[]; }): Promise { - const uniqueIds = [...new Set(input.modelIds)]; - if (uniqueIds.length === 0) { + const uniqueModels = dedupeModelEntries(input.models); + if (uniqueModels.length === 0) { return null; } - const allowlistProviders = await db - .select({ providerId: siteResourceAiProviders.providerId }) + const attachments = await db + .select({ + providerId: siteResourceAiProviders.providerId, + accessMode: siteResourceAiProviders.accessMode + }) .from(siteResourceAiProviders) .innerJoin( aiProviders, @@ -531,32 +623,92 @@ export async function assertModelsBelongToSiteAllowlistProviders(input: { siteResourceAiProviders.siteResourceId, input.siteResourceId ), - eq(siteResourceAiProviders.modelAccessMode, "allowlist"), eq(aiProviders.orgId, input.orgId) ) ); - if (allowlistProviders.length === 0) { - return "No allowlist AI providers are attached to this site resource"; + return assertModelEntriesValid({ + orgId: input.orgId, + modelEntries: uniqueModels, + attachments, + resourceLabel: "site resource" + }); +} + +function dedupeModelEntries( + models: ResourceAiModelEntry[] +): ResourceAiModelEntry[] { + const byModelId = new Map( + models.map((m) => [m.modelId, m.listType] as const) + ); + return [...byModelId.entries()].map(([modelId, listType]) => ({ + modelId, + listType + })); +} + +async function assertModelEntriesValid(input: { + orgId: string; + modelEntries: ResourceAiModelEntry[]; + attachments: ResourceAiProviderAttachment[]; + resourceLabel: string; +}): Promise { + const selectProviderIds = input.attachments + .filter((a) => a.accessMode === "select") + .map((a) => a.providerId); + + if (selectProviderIds.length === 0) { + return "Set at least one attached AI provider to select mode before managing model lists"; } - const validModels = await db - .select({ modelId: aiModels.modelId }) + const modelIds = input.modelEntries.map((m) => m.modelId); + const catalogRows = await db + .select({ + modelId: aiModels.modelId, + modelKey: aiModels.modelKey, + listType: aiModels.listType, + providerId: aiModels.providerId, + enabled: aiModels.enabled + }) .from(aiModels) .innerJoin(aiProviders, eq(aiModels.providerId, aiProviders.providerId)) .where( and( - inArray(aiModels.modelId, uniqueIds), - inArray( - aiModels.providerId, - allowlistProviders.map((p) => p.providerId) - ), + inArray(aiModels.modelId, modelIds), + inArray(aiModels.providerId, selectProviderIds), eq(aiProviders.orgId, input.orgId) ) ); - if (validModels.length !== uniqueIds.length) { - return "One or more model IDs do not exist or do not belong to an allowlist provider on this site resource"; + if (catalogRows.length !== modelIds.length) { + return `One or more model IDs do not exist or do not belong to a select-mode provider on this ${input.resourceLabel}`; + } + + const catalogById = new Map(catalogRows.map((row) => [row.modelId, row])); + const selectedAllowsByProvider = new Map(); + for (const entry of input.modelEntries) { + const catalog = catalogById.get(entry.modelId); + if (!catalog) { + return `One or more model IDs do not exist or do not belong to a select-mode provider on this ${input.resourceLabel}`; + } + if (catalog.listType !== entry.listType) { + return `Model ${entry.modelId} must use listType "${catalog.listType}" to match the provider catalog entry`; + } + if (!catalog.enabled) { + return `Model ${entry.modelId} is disabled on its provider`; + } + if (entry.listType === "allow") { + const keys = selectedAllowsByProvider.get(catalog.providerId) ?? []; + keys.push(catalog.modelKey); + selectedAllowsByProvider.set(catalog.providerId, keys); + } + } + + const overlapError = await assertNoOverlappingModelKeys(input.attachments, { + selectedAllowsByProvider + }); + if (overlapError) { + return overlapError.error; } return null; diff --git a/server/lib/aiModelKeyMatch.ts b/server/lib/aiModelKeyMatch.ts index fa27be56f..38184c3b0 100644 --- a/server/lib/aiModelKeyMatch.ts +++ b/server/lib/aiModelKeyMatch.ts @@ -83,3 +83,42 @@ export function modelKeysConflict(a: string, b: string): boolean { return modelKeyMatches(b, a); } + +/** + * Provider-layer policy: empty allowlist denies all. Blocklist only applies + * after an allow match. + */ +export function isAllowedByLists( + requested: string, + allows: string[], + blocks: string[] +): boolean { + if (allows.length === 0) { + return false; + } + if (!allows.some((pattern) => modelKeyMatches(pattern, requested))) { + return false; + } + if (blocks.some((pattern) => modelKeyMatches(pattern, requested))) { + return false; + } + return true; +} + +/** + * Among allow patterns that match `requested`, return the most specific one, + * or null if none match. + */ +export function mostSpecificMatchingAllow( + requested: string, + allows: string[] +): string | null { + const matching = allows.filter((pattern) => + modelKeyMatches(pattern, requested) + ); + if (matching.length === 0) { + return null; + } + matching.sort(compareModelKeySpecificity); + return matching[0]; +} diff --git a/server/lib/aiProviderDefaults.ts b/server/lib/aiProviderDefaults.ts index e48c6f877..f24330ec8 100644 --- a/server/lib/aiProviderDefaults.ts +++ b/server/lib/aiProviderDefaults.ts @@ -43,7 +43,7 @@ export const AI_PROVIDER_DEFAULTS: Record< authType: "x-api-key" }, googleGemini: { - upstreamUrl: "https://generativelanguage.googleapis.com/v1beta/openai/", + upstreamUrl: "https://generativelanguage.googleapis.com", authType: "x-goog-api-key" }, vertexAi: { diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 04d318004..d881b27ca 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -38,10 +38,15 @@ import { isIpInCidr } from "@server/lib/ip"; import { localCache } from "@server/lib/cache"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; -import type { ModelAccessMode } from "@server/lib/aiInferenceResource"; +import { + resolveEffectiveLists, + type AccessMode, + type ModelListType +} from "@server/lib/aiInferenceResource"; import { compareModelKeySpecificity, - modelKeyMatches + isAllowedByLists, + mostSpecificMatchingAllow } from "@server/lib/aiModelKeyMatch"; import { aiGatewayUpstreamFetch } from "@server/lib/aiGatewayUpstreamFetch"; @@ -96,7 +101,19 @@ async function findClientByIp(ip: string): Promise { type ProviderAttachment = { provider: AiProvider; - modelAccessMode: ModelAccessMode; + accessMode: AccessMode; +}; + +type ResourceModelPattern = { + providerId: number; + modelKey: string; + listType: ModelListType; + enabled: boolean; +}; + +type ProviderPatternLists = { + allows: string[]; + blocks: string[]; }; type ResolvedTarget = { @@ -104,7 +121,7 @@ type ResolvedTarget = { siteResourceId: number | null; orgId: string | null; attachments: ProviderAttachment[]; - allowlistedModelIds: Set; + resourceListsByProvider: Map; }; type ProviderSelection = @@ -231,8 +248,8 @@ async function resolveTarget(host: string): Promise { if (resourceRow) { const attachmentRows = await db .select({ - modelAccessMode: resourceAiProviders.modelAccessMode, - provider: aiProviders + provider: aiProviders, + accessMode: resourceAiProviders.accessMode }) .from(resourceAiProviders) .innerJoin( @@ -252,38 +269,26 @@ async function resolveTarget(host: string): Promise { const attachments: ProviderAttachment[] = attachmentRows.map((a) => ({ provider: a.provider, - modelAccessMode: a.modelAccessMode as ModelAccessMode + accessMode: a.accessMode })); - const allowlistProviderIds = attachments - .filter((a) => a.modelAccessMode === "allowlist") - .map((a) => a.provider.providerId); - const allowlistedModelIds = new Set(); - if (allowlistProviderIds.length > 0) { - const restrictions = await db - .select({ modelId: resourceAiModels.modelId }) - .from(resourceAiModels) - .innerJoin( - aiModels, - eq(resourceAiModels.modelId, aiModels.modelId) - ) - .where( - and( - eq(resourceAiModels.resourceId, resourceRow.resourceId), - inArray(aiModels.providerId, allowlistProviderIds) - ) - ); - for (const row of restrictions) { - allowlistedModelIds.add(row.modelId); - } - } + const resourcePatterns = await db + .select({ + providerId: aiModels.providerId, + modelKey: aiModels.modelKey, + listType: resourceAiModels.listType, + enabled: aiModels.enabled + }) + .from(resourceAiModels) + .innerJoin(aiModels, eq(resourceAiModels.modelId, aiModels.modelId)) + .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)); return { resourceId: resourceRow.resourceId, siteResourceId: null, orgId: resourceRow.orgId, attachments, - allowlistedModelIds + resourceListsByProvider: groupPatternsByProvider(resourcePatterns) }; } @@ -305,8 +310,8 @@ async function resolveTarget(host: string): Promise { if (siteResourceRow) { const attachmentRows = await db .select({ - modelAccessMode: siteResourceAiProviders.modelAccessMode, - provider: aiProviders + provider: aiProviders, + accessMode: siteResourceAiProviders.accessMode }) .from(siteResourceAiProviders) .innerJoin( @@ -329,50 +334,65 @@ async function resolveTarget(host: string): Promise { const attachments: ProviderAttachment[] = attachmentRows.map((a) => ({ provider: a.provider, - modelAccessMode: a.modelAccessMode as ModelAccessMode + accessMode: a.accessMode })); - const allowlistProviderIds = attachments - .filter((a) => a.modelAccessMode === "allowlist") - .map((a) => a.provider.providerId); - const allowlistedModelIds = new Set(); - if (allowlistProviderIds.length > 0) { - const restrictions = await db - .select({ modelId: siteResourceAiModels.modelId }) - .from(siteResourceAiModels) - .innerJoin( - aiModels, - eq(siteResourceAiModels.modelId, aiModels.modelId) + const resourcePatterns = await db + .select({ + providerId: aiModels.providerId, + modelKey: aiModels.modelKey, + listType: siteResourceAiModels.listType, + enabled: aiModels.enabled + }) + .from(siteResourceAiModels) + .innerJoin( + aiModels, + eq(siteResourceAiModels.modelId, aiModels.modelId) + ) + .where( + eq( + siteResourceAiModels.siteResourceId, + siteResourceRow.siteResourceId ) - .where( - and( - eq( - siteResourceAiModels.siteResourceId, - siteResourceRow.siteResourceId - ), - inArray(aiModels.providerId, allowlistProviderIds) - ) - ); - for (const row of restrictions) { - allowlistedModelIds.add(row.modelId); - } - } + ); return { resourceId: null, siteResourceId: siteResourceRow.siteResourceId, orgId: siteResourceRow.orgId, attachments, - allowlistedModelIds + resourceListsByProvider: groupPatternsByProvider(resourcePatterns) }; } return null; } +function groupPatternsByProvider( + patterns: ResourceModelPattern[] +): Map { + const byProvider = new Map(); + for (const pattern of patterns) { + if (!pattern.enabled) { + continue; + } + let lists = byProvider.get(pattern.providerId); + if (!lists) { + lists = { allows: [], blocks: [] }; + byProvider.set(pattern.providerId, lists); + } + if (pattern.listType === "allow") { + lists.allows.push(pattern.modelKey); + } else { + lists.blocks.push(pattern.modelKey); + } + } + return byProvider; +} + async function selectProvider( attachments: ProviderAttachment[], - allowlistedModelIds: Set, + resourceListsByProvider: Map, requestedModel: string | undefined ): Promise { if (!requestedModel) { @@ -383,10 +403,10 @@ async function selectProvider( }; } - const providerById = new Map( + const attachmentByProviderId = new Map( attachments.map((a) => [a.provider.providerId, a]) ); - const providerIds = [...providerById.keys()]; + const providerIds = [...attachmentByProviderId.keys()]; if (providerIds.length === 0) { return { ok: false, @@ -397,48 +417,54 @@ async function selectProvider( const providerModels = await db .select({ - modelId: aiModels.modelId, providerId: aiModels.providerId, modelKey: aiModels.modelKey, + listType: aiModels.listType, enabled: aiModels.enabled }) .from(aiModels) .where(inArray(aiModels.providerId, providerIds)); + const allowsByProvider = new Map(); + const blocksByProvider = new Map(); + for (const model of providerModels) { + if (!model.enabled) { + continue; + } + const targetMap = + model.listType === "allow" ? allowsByProvider : blocksByProvider; + const existing = targetMap.get(model.providerId) ?? []; + existing.push(model.modelKey); + targetMap.set(model.providerId, existing); + } + type ModelCandidate = { provider: AiProvider; modelKey: string; }; const candidates: ModelCandidate[] = []; - for (const model of providerModels) { - if (!model.enabled) { + for (const [providerId, attachment] of attachmentByProviderId) { + const resourceLists = resourceListsByProvider.get(providerId); + const { allows, blocks } = resolveEffectiveLists({ + accessMode: attachment.accessMode, + providerAllows: allowsByProvider.get(providerId) ?? [], + providerBlocks: blocksByProvider.get(providerId) ?? [], + resourceAllows: resourceLists?.allows ?? [], + resourceBlocks: resourceLists?.blocks ?? [] + }); + + if (!isAllowedByLists(requestedModel, allows, blocks)) { continue; } - - if (!modelKeyMatches(model.modelKey, requestedModel)) { + const matchingAllow = mostSpecificMatchingAllow(requestedModel, allows); + if (!matchingAllow) { continue; } - - const attachment = providerById.get(model.providerId); - if (!attachment) { - continue; - } - - if (attachment.modelAccessMode === "catalog") { - candidates.push({ - provider: attachment.provider, - modelKey: model.modelKey - }); - continue; - } - - if (allowlistedModelIds.has(model.modelId)) { - candidates.push({ - provider: attachment.provider, - modelKey: model.modelKey - }); - } + candidates.push({ + provider: attachment.provider, + modelKey: matchingAllow + }); } if (candidates.length === 0) { @@ -504,7 +530,8 @@ export async function handleAiGatewayProxy( }); } - const { attachments, allowlistedModelIds, resourceId, orgId } = target; + const { attachments, resourceListsByProvider, resourceId, orgId } = + target; const requestUser = await resolveRequestUser(req, resourceId, orgId); if (requestUser) { @@ -529,7 +556,7 @@ export async function handleAiGatewayProxy( const selection = await selectProvider( capableAttachments, - allowlistedModelIds, + resourceListsByProvider, requestedModel ); if (!selection.ok) { diff --git a/server/routers/aiProvider/createAiModel.ts b/server/routers/aiProvider/createAiModel.ts index dfe2a62e1..4a2734bbd 100644 --- a/server/routers/aiProvider/createAiModel.ts +++ b/server/routers/aiProvider/createAiModel.ts @@ -9,6 +9,7 @@ import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { and, eq } from "drizzle-orm"; import type { CreateOrEditAiModelResponse } from "@server/routers/aiProvider/types"; +import { modelListTypeSchema } from "@server/lib/aiInferenceResource"; const paramsSchema = z.strictObject({ providerId: z.coerce.number().int().positive() @@ -17,7 +18,8 @@ const paramsSchema = z.strictObject({ const bodySchema = z.strictObject({ modelKey: z.string().nonempty(), name: z.string().nonempty(), - enabled: z.boolean().optional() + enabled: z.boolean().optional(), + listType: modelListTypeSchema.optional().default("allow") }); registry.registerPath({ @@ -69,7 +71,7 @@ export async function createAiModel( } const { providerId } = parsedParams.data; - const { modelKey, name, enabled } = parsedBody.data; + const { modelKey, name, enabled, listType } = parsedBody.data; const [provider] = req.aiProvider && req.aiProvider.providerId === providerId @@ -116,6 +118,7 @@ export async function createAiModel( providerId, modelKey, name, + listType, enabled: enabled ?? true, createdAt: now, updatedAt: now diff --git a/server/routers/aiProvider/updateAiModel.ts b/server/routers/aiProvider/updateAiModel.ts index fe9ccf273..c0d1514d2 100644 --- a/server/routers/aiProvider/updateAiModel.ts +++ b/server/routers/aiProvider/updateAiModel.ts @@ -9,6 +9,7 @@ import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { and, eq, ne } from "drizzle-orm"; import type { CreateOrEditAiModelResponse } from "@server/routers/aiProvider/types"; +import { modelListTypeSchema } from "@server/lib/aiInferenceResource"; const paramsSchema = z.strictObject({ modelId: z.coerce.number().int().positive() @@ -17,7 +18,8 @@ const paramsSchema = z.strictObject({ const bodySchema = z.strictObject({ modelKey: z.string().nonempty().optional(), name: z.string().nonempty().optional(), - enabled: z.boolean().optional() + enabled: z.boolean().optional(), + listType: modelListTypeSchema.optional() }); registry.registerPath({ @@ -128,6 +130,9 @@ export async function updateAiModel( if (body.enabled !== undefined) { updateData.enabled = body.enabled; } + if (body.listType !== undefined) { + updateData.listType = body.listType; + } const [model] = await db .update(aiModels) diff --git a/server/routers/resource/addAiModelToResource.ts b/server/routers/resource/addAiModelToResource.ts index 3121c9ffa..4777bfe4d 100644 --- a/server/routers/resource/addAiModelToResource.ts +++ b/server/routers/resource/addAiModelToResource.ts @@ -9,11 +9,14 @@ import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { - assertPublicAllowlistApiEligible, - assertModelsBelongToPublicAllowlistProviders + assertPublicModelListApiEligible, + assertPublicResourceModelEntriesValid, + modelListTypeSchema } from "@server/lib/aiInferenceResource"; + const addAiModelToResourceBodySchema = z.strictObject({ - modelId: z.int().positive() + modelId: z.number().int().positive(), + listType: modelListTypeSchema.optional().default("allow") }); const addAiModelToResourceParamsSchema = z.strictObject({ @@ -24,7 +27,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-models/add", description: - "Add a single catalog model to an inference resource allowlist. Requires at least one attached AI provider in allowlist mode. The model must belong to a provider attached in allowlist mode.", + "Add a single model to an inference resource allow/block selection. Requires at least one attached AI provider in select mode. The model must belong to a select-mode provider and its listType must match the provider catalog entry. listType defaults to allow.", tags: [OpenAPITags.PublicResource], request: { params: addAiModelToResourceParamsSchema, @@ -70,7 +73,7 @@ export async function addAiModelToResource( ); } - const { modelId } = parsedBody.data; + const { modelId, listType } = parsedBody.data; const parsedParams = addAiModelToResourceParamsSchema.safeParse( req.params @@ -98,15 +101,15 @@ export async function addAiModelToResource( ); } - const eligibleError = await assertPublicAllowlistApiEligible(resource); + const eligibleError = await assertPublicModelListApiEligible(resource); if (eligibleError) { return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); } - const modelError = await assertModelsBelongToPublicAllowlistProviders({ + const modelError = await assertPublicResourceModelEntriesValid({ orgId: resource.orgId, resourceId, - modelIds: [modelId] + models: [{ modelId, listType }] }); if (modelError) { return next(createHttpError(HttpCode.BAD_REQUEST, modelError)); @@ -131,7 +134,9 @@ export async function addAiModelToResource( ); } - await db.insert(resourceAiModels).values({ resourceId, modelId }); + await db + .insert(resourceAiModels) + .values({ resourceId, modelId, listType }); return response(res, { data: {}, diff --git a/server/routers/resource/addAiProviderToResource.ts b/server/routers/resource/addAiProviderToResource.ts index d51cf244c..bcc175c18 100644 --- a/server/routers/resource/addAiProviderToResource.ts +++ b/server/routers/resource/addAiProviderToResource.ts @@ -11,14 +11,12 @@ import { OpenAPITags, registry } from "@server/openApi"; import { isInferenceFieldsError, listPublicResourceAiProviders, - modelAccessModeSchema, resolveProviderAttachments, setPublicResourceAiProviders } from "@server/lib/aiInferenceResource"; const addAiProviderToResourceBodySchema = z.strictObject({ - providerId: z.number().int().positive(), - modelAccessMode: modelAccessModeSchema.optional() + providerId: z.number().int().positive() }); const addAiProviderToResourceParamsSchema = z.strictObject({ @@ -29,7 +27,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-providers/add", description: - "Add or replace a single AI provider attachment on an inference resource.", + "Add or replace a single AI provider attachment on an inference resource. The provider is attached in inherit mode, using its own allow/block lists.", tags: [OpenAPITags.PublicResource], request: { params: addAiProviderToResourceParamsSchema, @@ -77,7 +75,7 @@ export async function addAiProviderToResource( ); } - const { providerId, modelAccessMode } = parsedBody.data; + const { providerId } = parsedBody.data; const parsedParams = addAiProviderToResourceParamsSchema.safeParse( req.params @@ -120,18 +118,21 @@ export async function addAiProviderToResource( .filter((a) => a.providerId !== providerId) .map((a) => ({ providerId: a.providerId, - modelAccessMode: a.modelAccessMode + accessMode: a.accessMode })), - { providerId, modelAccessMode } + { providerId, accessMode: "inherit" as const } ]; const attachments = await resolveProviderAttachments({ orgId: resource.orgId, attachments: nextAttachments, - requireAtLeastOne: true + requireAtLeastOne: true, + resourceId }); if (isInferenceFieldsError(attachments)) { - return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + return next( + createHttpError(HttpCode.BAD_REQUEST, attachments.error) + ); } await setPublicResourceAiProviders(resourceId, attachments); diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index 47e70140f..e225458d1 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -109,7 +109,7 @@ const createHttpResourceSchema = z .array(resourceAiProviderAttachmentSchema) .optional() .describe( - "For inference-mode resources: AI providers to attach. Each entry may set modelAccessMode (catalog or allowlist); defaults to catalog. Model keys must be unique across attached catalog providers." + "For inference-mode resources: AI providers to attach. Providers are attached in inherit mode, using each provider's own allow/block lists. Effective allow model keys must be unique across attached providers." ) }) .refine( @@ -391,9 +391,14 @@ async function createHttpResource( let providerAttachments: ResourceAiProviderAttachment[] = []; if (effectiveMode === "inference") { + // A new resource has no model selections yet, so providers always start + // in inherit mode; select can be enabled afterwards. const resolved = await resolveProviderAttachments({ orgId, - attachments: aiProviderInputs ?? [], + attachments: (aiProviderInputs ?? []).map((p) => ({ + providerId: p.providerId, + accessMode: "inherit" as const + })), requireAtLeastOne: false }); if (isInferenceFieldsError(resolved)) { diff --git a/server/routers/resource/listResourceAiModels.ts b/server/routers/resource/listResourceAiModels.ts index 0a5dea778..f16e0c2da 100644 --- a/server/routers/resource/listResourceAiModels.ts +++ b/server/routers/resource/listResourceAiModels.ts @@ -19,7 +19,9 @@ async function query(resourceId: number) { modelId: aiModels.modelId, modelKey: aiModels.modelKey, name: aiModels.name, - enabled: aiModels.enabled + providerId: aiModels.providerId, + enabled: aiModels.enabled, + listType: resourceAiModels.listType }) .from(resourceAiModels) .innerJoin(aiModels, eq(resourceAiModels.modelId, aiModels.modelId)) @@ -34,7 +36,7 @@ registry.registerPath({ method: "get", path: "/resource/{resourceId}/ai-models", description: - "List catalog models on this resource's allowlist. Only enforced when modelAccessMode=allowlist; an empty allowlist denies all models.", + "List the models this resource has selected from its select-mode providers' allow/block lists. Providers in inherit mode are not represented here; they use their own lists.", tags: [OpenAPITags.PublicResource], request: { params: listResourceAiModelsParamsSchema diff --git a/server/routers/resource/listResourceAiProviders.ts b/server/routers/resource/listResourceAiProviders.ts index 2c81c45d5..f48ad71fe 100644 --- a/server/routers/resource/listResourceAiProviders.ts +++ b/server/routers/resource/listResourceAiProviders.ts @@ -21,7 +21,8 @@ export type ListResourceAiProvidersResponse = { registry.registerPath({ method: "get", path: "/resource/{resourceId}/ai-providers", - description: "List AI providers attached to an inference resource.", + description: + "List AI providers attached to an inference resource, including each attachment's accessMode.", tags: [OpenAPITags.PublicResource], request: { params: listResourceAiProvidersParamsSchema diff --git a/server/routers/resource/removeAiModelFromResource.ts b/server/routers/resource/removeAiModelFromResource.ts index 7b5cb012f..626b94666 100644 --- a/server/routers/resource/removeAiModelFromResource.ts +++ b/server/routers/resource/removeAiModelFromResource.ts @@ -8,7 +8,7 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { assertPublicAllowlistApiEligible } from "@server/lib/aiInferenceResource"; +import { assertPublicModelListApiEligible } from "@server/lib/aiInferenceResource"; const removeAiModelFromResourceBodySchema = z.strictObject({ modelId: z.int().positive() @@ -22,7 +22,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-models/remove", description: - "Remove a single catalog model from an inference resource allowlist. Requires at least one attached AI provider in allowlist mode.", + "Remove a single model from an inference resource allow/block list. Requires at least one attached AI provider.", tags: [OpenAPITags.PublicResource], request: { params: removeAiModelFromResourceParamsSchema, @@ -98,7 +98,7 @@ export async function removeAiModelFromResource( ); } - const eligibleError = await assertPublicAllowlistApiEligible(resource); + const eligibleError = await assertPublicModelListApiEligible(resource); if (eligibleError) { return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); } diff --git a/server/routers/resource/removeAiProviderFromResource.ts b/server/routers/resource/removeAiProviderFromResource.ts index e74b0c886..a7f9298ee 100644 --- a/server/routers/resource/removeAiProviderFromResource.ts +++ b/server/routers/resource/removeAiProviderFromResource.ts @@ -127,13 +127,14 @@ export async function removeAiProviderFromResource( .filter((a) => a.providerId !== providerId) .map((a) => ({ providerId: a.providerId, - modelAccessMode: a.modelAccessMode + accessMode: a.accessMode })); const attachments = await resolveProviderAttachments({ orgId: resource.orgId, attachments: remaining, - requireAtLeastOne: false + requireAtLeastOne: false, + resourceId }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/server/routers/resource/setResourceAiModels.ts b/server/routers/resource/setResourceAiModels.ts index 0c3323ebe..78a67fdfb 100644 --- a/server/routers/resource/setResourceAiModels.ts +++ b/server/routers/resource/setResourceAiModels.ts @@ -9,12 +9,13 @@ import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { - assertPublicAllowlistApiEligible, - assertModelsBelongToPublicAllowlistProviders + assertPublicModelListApiEligible, + assertPublicResourceModelEntriesValid, + resourceAiModelEntrySchema } from "@server/lib/aiInferenceResource"; const setResourceAiModelsBodySchema = z.strictObject({ - modelIds: z.array(z.int().positive()) + models: z.array(resourceAiModelEntrySchema) }); const setResourceAiModelsParamsSchema = z.strictObject({ @@ -25,7 +26,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-models", description: - "Replace the allowlist of catalog models for an inference resource. Requires at least one attached AI provider in allowlist mode. Models must belong to a provider attached in allowlist mode. An empty array denies all models.", + "Replace the allow/block model selection for an inference resource. Requires at least one attached AI provider in select mode. Models must belong to a select-mode provider and their listType must match the provider catalog entry. An empty array clears the selection, which denies all models for select-mode providers.", tags: [OpenAPITags.PublicResource], request: { params: setResourceAiModelsParamsSchema, @@ -71,7 +72,7 @@ export async function setResourceAiModels( ); } - const { modelIds } = parsedBody.data; + const { models } = parsedBody.data; const parsedParams = setResourceAiModelsParamsSchema.safeParse( req.params @@ -99,15 +100,22 @@ export async function setResourceAiModels( ); } - const eligibleError = await assertPublicAllowlistApiEligible(resource); + const eligibleError = await assertPublicModelListApiEligible(resource); if (eligibleError) { return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); } - const modelError = await assertModelsBelongToPublicAllowlistProviders({ + const byModelId = new Map( + models.map((m) => [m.modelId, m.listType] as const) + ); + const uniqueModels = [...byModelId.entries()].map( + ([modelId, listType]) => ({ modelId, listType }) + ); + + const modelError = await assertPublicResourceModelEntriesValid({ orgId: resource.orgId, resourceId, - modelIds + models: uniqueModels }); if (modelError) { return next(createHttpError(HttpCode.BAD_REQUEST, modelError)); @@ -118,12 +126,14 @@ export async function setResourceAiModels( .delete(resourceAiModels) .where(eq(resourceAiModels.resourceId, resourceId)); - if (modelIds.length > 0) { - await trx - .insert(resourceAiModels) - .values( - modelIds.map((modelId) => ({ resourceId, modelId })) - ); + if (uniqueModels.length > 0) { + await trx.insert(resourceAiModels).values( + uniqueModels.map((m) => ({ + resourceId, + modelId: m.modelId, + listType: m.listType + })) + ); } }); diff --git a/server/routers/resource/setResourceAiProviders.ts b/server/routers/resource/setResourceAiProviders.ts index 88552954b..ef312ac70 100644 --- a/server/routers/resource/setResourceAiProviders.ts +++ b/server/routers/resource/setResourceAiProviders.ts @@ -27,7 +27,7 @@ registry.registerPath({ method: "post", path: "/resource/{resourceId}/ai-providers", description: - "Replace the AI providers attached to an inference resource. An empty list clears all providers. Model keys must be unique across attached catalog providers.", + "Replace the AI providers attached to an inference resource. Each provider uses accessMode inherit (default, uses the provider's own allow/block lists) or select (uses the resource's selected subset of that provider's catalog). An empty list clears all providers. Effective allow model keys must be unique across attached providers.", tags: [OpenAPITags.PublicResource], request: { params: setResourceAiProvidersParamsSchema, @@ -113,7 +113,8 @@ export async function setResourceAiProviders( const attachments = await resolveProviderAttachments({ orgId: resource.orgId, attachments: providers, - requireAtLeastOne: false + requireAtLeastOne: false, + resourceId }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/server/routers/siteResource/addAiModelToSiteResource.ts b/server/routers/siteResource/addAiModelToSiteResource.ts index 0b6a004d7..8e1f8935e 100644 --- a/server/routers/siteResource/addAiModelToSiteResource.ts +++ b/server/routers/siteResource/addAiModelToSiteResource.ts @@ -9,12 +9,14 @@ import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { - assertSiteAllowlistApiEligible, - assertModelsBelongToSiteAllowlistProviders + assertSiteModelListApiEligible, + assertSiteResourceModelEntriesValid, + modelListTypeSchema } from "@server/lib/aiInferenceResource"; const addAiModelToSiteResourceBodySchema = z.strictObject({ - modelId: z.int().positive() + modelId: z.number().int().positive(), + listType: modelListTypeSchema.optional().default("allow") }); const addAiModelToSiteResourceParamsSchema = z.strictObject({ @@ -25,7 +27,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-models/add", description: - "Add a single catalog model to an inference site resource allowlist. Requires at least one attached AI provider in allowlist mode. The model must belong to a provider attached in allowlist mode.", + "Add a single model to an inference site resource allow/block selection. Requires at least one attached AI provider in select mode. The model must belong to a select-mode provider and its listType must match the provider catalog entry. listType defaults to allow.", tags: [OpenAPITags.PrivateResource], request: { params: addAiModelToSiteResourceParamsSchema, @@ -73,7 +75,7 @@ export async function addAiModelToSiteResource( ); } - const { modelId } = parsedBody.data; + const { modelId, listType } = parsedBody.data; const parsedParams = addAiModelToSiteResourceParamsSchema.safeParse( req.params @@ -102,15 +104,15 @@ export async function addAiModelToSiteResource( } const eligibleError = - await assertSiteAllowlistApiEligible(siteResource); + await assertSiteModelListApiEligible(siteResource); if (eligibleError) { return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); } - const modelError = await assertModelsBelongToSiteAllowlistProviders({ + const modelError = await assertSiteResourceModelEntriesValid({ orgId: siteResource.orgId, siteResourceId, - modelIds: [modelId] + models: [{ modelId, listType }] }); if (modelError) { return next(createHttpError(HttpCode.BAD_REQUEST, modelError)); @@ -135,9 +137,11 @@ export async function addAiModelToSiteResource( ); } - await db - .insert(siteResourceAiModels) - .values({ siteResourceId, modelId }); + await db.insert(siteResourceAiModels).values({ + siteResourceId, + modelId, + listType + }); return response(res, { data: {}, diff --git a/server/routers/siteResource/addAiProviderToSiteResource.ts b/server/routers/siteResource/addAiProviderToSiteResource.ts index bae346347..79a0ae093 100644 --- a/server/routers/siteResource/addAiProviderToSiteResource.ts +++ b/server/routers/siteResource/addAiProviderToSiteResource.ts @@ -11,14 +11,12 @@ import { OpenAPITags, registry } from "@server/openApi"; import { isInferenceFieldsError, listSiteResourceAiProviders, - modelAccessModeSchema, resolveProviderAttachments, setSiteResourceAiProviders } from "@server/lib/aiInferenceResource"; const addAiProviderToSiteResourceBodySchema = z.strictObject({ - providerId: z.number().int().positive(), - modelAccessMode: modelAccessModeSchema.optional() + providerId: z.number().int().positive() }); const addAiProviderToSiteResourceParamsSchema = z.strictObject({ @@ -29,7 +27,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-providers/add", description: - "Add or replace a single AI provider attachment on an inference site resource.", + "Add or replace a single AI provider attachment on an inference site resource. The provider is attached in inherit mode, using its own allow/block lists.", tags: [OpenAPITags.PrivateResource], request: { params: addAiProviderToSiteResourceParamsSchema, @@ -77,7 +75,7 @@ export async function addAiProviderToSiteResource( ); } - const { providerId, modelAccessMode } = parsedBody.data; + const { providerId } = parsedBody.data; const parsedParams = addAiProviderToSiteResourceParamsSchema.safeParse( req.params @@ -120,18 +118,21 @@ export async function addAiProviderToSiteResource( .filter((a) => a.providerId !== providerId) .map((a) => ({ providerId: a.providerId, - modelAccessMode: a.modelAccessMode + accessMode: a.accessMode })), - { providerId, modelAccessMode } + { providerId, accessMode: "inherit" as const } ]; const attachments = await resolveProviderAttachments({ orgId: siteResource.orgId, attachments: nextAttachments, - requireAtLeastOne: true + requireAtLeastOne: true, + siteResourceId }); if (isInferenceFieldsError(attachments)) { - return next(createHttpError(HttpCode.BAD_REQUEST, attachments.error)); + return next( + createHttpError(HttpCode.BAD_REQUEST, attachments.error) + ); } await setSiteResourceAiProviders(siteResourceId, attachments); diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 31f631ce2..15b82cb56 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -90,7 +90,7 @@ const createSiteResourceSchema = z .array(resourceAiProviderAttachmentSchema) .optional() .describe( - "For inference-mode site resources: AI providers to attach. Each entry may set modelAccessMode (catalog or allowlist); defaults to catalog. Model keys must be unique across attached catalog providers." + "For inference-mode site resources: AI providers to attach. Providers are attached in inherit mode, using each provider's own allow/block lists. Effective allow model keys must be unique across attached providers." ) }) .strict() @@ -350,9 +350,14 @@ export async function createSiteResource( let providerAttachments: ResourceAiProviderAttachment[] = []; if (mode === "inference") { + // A new site resource has no model selections yet, so providers + // always start in inherit mode; select can be enabled afterwards. const resolved = await resolveProviderAttachments({ orgId, - attachments: aiProviderInputs ?? [], + attachments: (aiProviderInputs ?? []).map((p) => ({ + providerId: p.providerId, + accessMode: "inherit" as const + })), requireAtLeastOne: false }); if (isInferenceFieldsError(resolved)) { diff --git a/server/routers/siteResource/listSiteResourceAiModels.ts b/server/routers/siteResource/listSiteResourceAiModels.ts index 26e709140..fd65d29ea 100644 --- a/server/routers/siteResource/listSiteResourceAiModels.ts +++ b/server/routers/siteResource/listSiteResourceAiModels.ts @@ -19,7 +19,9 @@ async function query(siteResourceId: number) { modelId: aiModels.modelId, modelKey: aiModels.modelKey, name: aiModels.name, - enabled: aiModels.enabled + providerId: aiModels.providerId, + enabled: aiModels.enabled, + listType: siteResourceAiModels.listType }) .from(siteResourceAiModels) .innerJoin(aiModels, eq(siteResourceAiModels.modelId, aiModels.modelId)) @@ -34,7 +36,7 @@ registry.registerPath({ method: "get", path: "/site-resource/{siteResourceId}/ai-models", description: - "List catalog models on this site resource's allowlist. Only enforced when modelAccessMode=allowlist; an empty allowlist denies all models.", + "List the models this site resource has selected from its select-mode providers' allow/block lists. Providers in inherit mode are not represented here; they use their own lists.", tags: [OpenAPITags.PrivateResource], request: { params: listSiteResourceAiModelsParamsSchema diff --git a/server/routers/siteResource/listSiteResourceAiProviders.ts b/server/routers/siteResource/listSiteResourceAiProviders.ts index bcbe37e8b..7d7671dad 100644 --- a/server/routers/siteResource/listSiteResourceAiProviders.ts +++ b/server/routers/siteResource/listSiteResourceAiProviders.ts @@ -21,7 +21,8 @@ export type ListSiteResourceAiProvidersResponse = { registry.registerPath({ method: "get", path: "/site-resource/{siteResourceId}/ai-providers", - description: "List AI providers attached to an inference site resource.", + description: + "List AI providers attached to an inference site resource, including each attachment's accessMode.", tags: [OpenAPITags.PrivateResource], request: { params: listSiteResourceAiProvidersParamsSchema diff --git a/server/routers/siteResource/removeAiModelFromSiteResource.ts b/server/routers/siteResource/removeAiModelFromSiteResource.ts index eb8d649a2..c52dfb43d 100644 --- a/server/routers/siteResource/removeAiModelFromSiteResource.ts +++ b/server/routers/siteResource/removeAiModelFromSiteResource.ts @@ -8,7 +8,7 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { assertSiteAllowlistApiEligible } from "@server/lib/aiInferenceResource"; +import { assertSiteModelListApiEligible } from "@server/lib/aiInferenceResource"; const removeAiModelFromSiteResourceBodySchema = z.strictObject({ modelId: z.int().positive() @@ -22,7 +22,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-models/remove", description: - "Remove a single catalog model from an inference site resource allowlist. Requires at least one attached AI provider in allowlist mode.", + "Remove a single model from an inference site resource allow/block list. Requires at least one attached AI provider.", tags: [OpenAPITags.PrivateResource], request: { params: removeAiModelFromSiteResourceParamsSchema, @@ -98,7 +98,7 @@ export async function removeAiModelFromSiteResource( } const eligibleError = - await assertSiteAllowlistApiEligible(siteResource); + await assertSiteModelListApiEligible(siteResource); if (eligibleError) { return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); } diff --git a/server/routers/siteResource/removeAiProviderFromSiteResource.ts b/server/routers/siteResource/removeAiProviderFromSiteResource.ts index 3d4392820..ffb4f1ff8 100644 --- a/server/routers/siteResource/removeAiProviderFromSiteResource.ts +++ b/server/routers/siteResource/removeAiProviderFromSiteResource.ts @@ -126,13 +126,14 @@ export async function removeAiProviderFromSiteResource( .filter((a) => a.providerId !== providerId) .map((a) => ({ providerId: a.providerId, - modelAccessMode: a.modelAccessMode + accessMode: a.accessMode })); const attachments = await resolveProviderAttachments({ orgId: siteResource.orgId, attachments: remaining, - requireAtLeastOne: false + requireAtLeastOne: false, + siteResourceId }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/server/routers/siteResource/setSiteResourceAiModels.ts b/server/routers/siteResource/setSiteResourceAiModels.ts index 391634553..eeacbbd23 100644 --- a/server/routers/siteResource/setSiteResourceAiModels.ts +++ b/server/routers/siteResource/setSiteResourceAiModels.ts @@ -9,12 +9,13 @@ import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { - assertSiteAllowlistApiEligible, - assertModelsBelongToSiteAllowlistProviders + assertSiteModelListApiEligible, + assertSiteResourceModelEntriesValid, + resourceAiModelEntrySchema } from "@server/lib/aiInferenceResource"; const setSiteResourceAiModelsBodySchema = z.strictObject({ - modelIds: z.array(z.int().positive()) + models: z.array(resourceAiModelEntrySchema) }); const setSiteResourceAiModelsParamsSchema = z.strictObject({ @@ -25,7 +26,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-models", description: - "Replace the allowlist of catalog models for an inference site resource. Requires at least one attached AI provider in allowlist mode. Models must belong to a provider attached in allowlist mode. An empty array denies all models.", + "Replace the allow/block model selection for an inference site resource. Requires at least one attached AI provider in select mode. Models must belong to a select-mode provider and their listType must match the provider catalog entry. An empty array clears the selection, which denies all models for select-mode providers.", tags: [OpenAPITags.PrivateResource], request: { params: setSiteResourceAiModelsParamsSchema, @@ -73,7 +74,7 @@ export async function setSiteResourceAiModels( ); } - const { modelIds } = parsedBody.data; + const { models } = parsedBody.data; const parsedParams = setSiteResourceAiModelsParamsSchema.safeParse( req.params @@ -102,15 +103,22 @@ export async function setSiteResourceAiModels( } const eligibleError = - await assertSiteAllowlistApiEligible(siteResource); + await assertSiteModelListApiEligible(siteResource); if (eligibleError) { return next(createHttpError(HttpCode.BAD_REQUEST, eligibleError)); } - const modelError = await assertModelsBelongToSiteAllowlistProviders({ + const byModelId = new Map( + models.map((m) => [m.modelId, m.listType] as const) + ); + const uniqueModels = [...byModelId.entries()].map( + ([modelId, listType]) => ({ modelId, listType }) + ); + + const modelError = await assertSiteResourceModelEntriesValid({ orgId: siteResource.orgId, siteResourceId, - modelIds + models: uniqueModels }); if (modelError) { return next(createHttpError(HttpCode.BAD_REQUEST, modelError)); @@ -121,11 +129,12 @@ export async function setSiteResourceAiModels( .delete(siteResourceAiModels) .where(eq(siteResourceAiModels.siteResourceId, siteResourceId)); - if (modelIds.length > 0) { + if (uniqueModels.length > 0) { await trx.insert(siteResourceAiModels).values( - modelIds.map((modelId) => ({ + uniqueModels.map((m) => ({ siteResourceId, - modelId + modelId: m.modelId, + listType: m.listType })) ); } diff --git a/server/routers/siteResource/setSiteResourceAiProviders.ts b/server/routers/siteResource/setSiteResourceAiProviders.ts index fd685bb20..a6c8a52c7 100644 --- a/server/routers/siteResource/setSiteResourceAiProviders.ts +++ b/server/routers/siteResource/setSiteResourceAiProviders.ts @@ -27,7 +27,7 @@ registry.registerPath({ method: "post", path: "/site-resource/{siteResourceId}/ai-providers", description: - "Replace the AI providers attached to an inference site resource. An empty list clears all providers. Model keys must be unique across attached catalog providers.", + "Replace the AI providers attached to an inference site resource. Each provider uses accessMode inherit (default, uses the provider's own allow/block lists) or select (uses the site resource's selected subset of that provider's catalog). An empty list clears all providers. Effective allow model keys must be unique across attached providers.", tags: [OpenAPITags.PrivateResource], request: { params: setSiteResourceAiProvidersParamsSchema, @@ -115,7 +115,8 @@ export async function setSiteResourceAiProviders( const attachments = await resolveProviderAttachments({ orgId: siteResource.orgId, attachments: providers, - requireAtLeastOne: false + requireAtLeastOne: false, + siteResourceId }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx index f90143492..10db41571 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx @@ -12,6 +12,7 @@ import { } from "@app/components/Settings"; import { TagInput, type Tag } from "@app/components/tags/tag-input"; import { Button } from "@app/components/ui/button"; +import { Label } from "@app/components/ui/label"; import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; @@ -21,6 +22,8 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useTranslations } from "next-intl"; import { useEffect, useState } from "react"; +type ModelListType = "allow" | "block"; + export default function AiProviderModelsPage() { const { provider } = useAiProviderContext(); const { env } = useEnvContext(); @@ -28,8 +31,14 @@ export default function AiProviderModelsPage() { const queryClient = useQueryClient(); const t = useTranslations(); const [saveLoading, setSaveLoading] = useState(false); - const [tags, setTags] = useState([]); - const [activeTagIndex, setActiveTagIndex] = useState(null); + const [allowTags, setAllowTags] = useState([]); + const [blockTags, setBlockTags] = useState([]); + const [activeAllowTagIndex, setActiveAllowTagIndex] = useState< + number | null + >(null); + const [activeBlockTagIndex, setActiveBlockTagIndex] = useState< + number | null + >(null); const modelsQuery = useQuery( aiProviderQueries.providerModels({ providerId: provider.providerId }) @@ -37,11 +46,21 @@ export default function AiProviderModelsPage() { useEffect(() => { if (!modelsQuery.data) return; - setTags( - modelsQuery.data.map((model) => ({ - id: String(model.modelId), - text: model.modelKey - })) + setAllowTags( + modelsQuery.data + .filter((model) => (model.listType ?? "allow") === "allow") + .map((model) => ({ + id: String(model.modelId), + text: model.modelKey + })) + ); + setBlockTags( + modelsQuery.data + .filter((model) => model.listType === "block") + .map((model) => ({ + id: String(model.modelId), + text: model.modelKey + })) ); }, [modelsQuery.data]); @@ -52,27 +71,74 @@ export default function AiProviderModelsPage() { const existingByKey = new Map( existing.map((model) => [model.modelKey, model]) ); - const nextKeys = new Set( - tags.map((tag) => tag.text.trim()).filter(Boolean) + + const nextAllow = new Set( + allowTags.map((tag) => tag.text.trim()).filter(Boolean) + ); + const nextBlock = new Set( + blockTags.map((tag) => tag.text.trim()).filter(Boolean) ); - const toCreate = [...nextKeys].filter( - (key) => !existingByKey.has(key) - ); - const toDelete = existing.filter( - (model) => !nextKeys.has(model.modelKey) - ); + const overlap = [...nextAllow].filter((key) => nextBlock.has(key)); + if (overlap.length > 0) { + toast({ + variant: "destructive", + title: t("aiProviderModelsErrorUpdate"), + description: t("aiProviderModelsOverlapError", { + keys: overlap.join(", ") + }) + }); + return; + } + + const desired = new Map(); + for (const key of nextAllow) { + desired.set(key, "allow"); + } + for (const key of nextBlock) { + desired.set(key, "block"); + } + + const toCreate: { modelKey: string; listType: ModelListType }[] = + []; + const toUpdate: { + modelId: number; + listType: ModelListType; + }[] = []; + const toDelete: number[] = []; + + for (const [modelKey, listType] of desired) { + const existingModel = existingByKey.get(modelKey); + if (!existingModel) { + toCreate.push({ modelKey, listType }); + continue; + } + if ((existingModel.listType ?? "allow") !== listType) { + toUpdate.push({ + modelId: existingModel.modelId, + listType + }); + } + } + + for (const model of existing) { + if (!desired.has(model.modelKey)) { + toDelete.push(model.modelId); + } + } await Promise.all([ - ...toCreate.map((modelKey) => + ...toCreate.map(({ modelKey, listType }) => api.put(`/ai-provider/${provider.providerId}/model`, { modelKey, - name: modelKey + name: modelKey, + listType }) ), - ...toDelete.map((model) => - api.delete(`/ai-model/${model.modelId}`) - ) + ...toUpdate.map(({ modelId, listType }) => + api.post(`/ai-model/${modelId}`, { listType }) + ), + ...toDelete.map((modelId) => api.delete(`/ai-model/${modelId}`)) ]); await queryClient.invalidateQueries( @@ -113,24 +179,59 @@ export default function AiProviderModelsPage() { - { - const next = - typeof newTags === "function" - ? newTags(tags) - : newTags; - setTags(next as Tag[]); - }} - allowDuplicates={false} - sortTags - delimiterList={[",", "Enter"]} - disabled={modelsQuery.isLoading || saveLoading} - /> +
+ + { + const next = + typeof newTags === "function" + ? newTags(allowTags) + : newTags; + setAllowTags(next as Tag[]); + }} + allowDuplicates={false} + sortTags + delimiterList={[",", "Enter"]} + disabled={modelsQuery.isLoading || saveLoading} + /> +

+ {t("aiProviderModelsAllowDescription")} +

+
+ +
+ + { + const next = + typeof newTags === "function" + ? newTags(blockTags) + : newTags; + setBlockTags(next as Tag[]); + }} + allowDuplicates={false} + sortTags + delimiterList={[",", "Enter"]} + disabled={modelsQuery.isLoading || saveLoading} + /> +

+ {t("aiProviderModelsBlockDescription")} +

+
diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx index 48304d350..f9ad664a0 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx @@ -130,8 +130,7 @@ export default function PrivateResourceInferencePage() { await api.post(`/site-resource/${siteResource.id}/ai-providers`, { providers: data.providerIds.map((providerId) => ({ - providerId, - modelAccessMode: "catalog" + providerId })) }); @@ -258,12 +257,10 @@ export default function PrivateResourceInferencePage() { cols={2} hideFreeDomain defaultSubdomain={ - httpConfigSubdomain ?? - undefined + httpConfigSubdomain ?? undefined } defaultDomainId={ - httpConfigDomainId ?? - undefined + httpConfigDomainId ?? undefined } defaultFullDomain={ httpConfigFullDomain ?? diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx index 8741159b4..ad7230591 100644 --- a/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx +++ b/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx @@ -102,8 +102,7 @@ export default function PublicResourceInferencePage() { try { await api.post(`/resource/${resource.resourceId}/ai-providers`, { providers: data.providerIds.map((providerId) => ({ - providerId, - modelAccessMode: "catalog" + providerId })) }); diff --git a/src/app/[orgId]/settings/resources/public/create/page.tsx b/src/app/[orgId]/settings/resources/public/create/page.tsx index a5088d84e..4d1c24efb 100644 --- a/src/app/[orgId]/settings/resources/public/create/page.tsx +++ b/src/app/[orgId]/settings/resources/public/create/page.tsx @@ -497,8 +497,7 @@ export default function Page() { if (resourceType === "inference") { Object.assign(payload, { aiProviders: selectedProviders.map((provider) => ({ - providerId: parseInt(provider.id, 10), - modelAccessMode: "catalog" + providerId: parseInt(provider.id, 10) })) }); } else if (resourceType === "ssh") { diff --git a/src/lib/privateResourceForm.ts b/src/lib/privateResourceForm.ts index f186ffb9d..669a96363 100644 --- a/src/lib/privateResourceForm.ts +++ b/src/lib/privateResourceForm.ts @@ -217,8 +217,7 @@ export function buildCreateSiteResourcePayload( }), ...(data.mode === "inference" && { aiProviders: (data.providerIds ?? []).map((providerId) => ({ - providerId, - modelAccessMode: "catalog" as const + providerId })), ssl: data.ssl ?? false, domainId: data.httpConfigDomainId diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 9a0cb2403..6b7070f5b 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -1288,10 +1288,10 @@ export const resourceQueries = { AxiosResponse<{ providers: Array<{ providerId: number; - modelAccessMode: "catalog" | "allowlist"; name: string; type: string; enabled: boolean; + accessMode: "inherit" | "select"; }>; }> >(`/site-resource/${siteResourceId}/ai-providers`, { @@ -1308,10 +1308,10 @@ export const resourceQueries = { AxiosResponse<{ providers: Array<{ providerId: number; - modelAccessMode: "catalog" | "allowlist"; name: string; type: string; enabled: boolean; + accessMode: "inherit" | "select"; }>; }> >(`/resource/${resourceId}/ai-providers`, { From d699455b383f3c42e1ebb229d50339e50559f8df Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 7 Aug 2026 11:23:07 -0400 Subject: [PATCH 076/296] support changing capabilities on provider integrations --- messages/en-US.json | 2 +- server/lib/aiCapabilities.ts | 5 +- server/routers/aiProvider/createAiProvider.ts | 9 +++ server/routers/aiProvider/updateAiProvider.ts | 17 +---- server/routers/aiProvider/validation.ts | 10 +++ .../[providerId]/general/page.tsx | 69 +++++-------------- .../settings/ai-providers/create/page.tsx | 59 ++++------------ src/lib/aiProviderFormSchema.ts | 10 +-- 8 files changed, 60 insertions(+), 121 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 8cc780829..2d77bb365 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1729,7 +1729,7 @@ "aiProviderErrorRoutingModeTarget": "Site targets routing is only available for custom providers", "aiProviderErrorCapabilitiesRequired": "Select at least one API capability", "aiProviderCapabilities": "API Capabilities", - "aiProviderCapabilitiesDescription": "Which API formats this provider accepts. Built-in providers use fixed capabilities.", + "aiProviderCapabilitiesDescription": "Select which API formats this provider can handle. Known providers start with recommended defaults.", "aiProviderCapabilitiesCustomDescription": "Select which API formats this custom provider can handle", "aiProviderCapabilitiesSelect": "Select capabilities", "aiProviderCapabilitiesEmpty": "No capabilities found", diff --git a/server/lib/aiCapabilities.ts b/server/lib/aiCapabilities.ts index f7b370691..356122e7f 100644 --- a/server/lib/aiCapabilities.ts +++ b/server/lib/aiCapabilities.ts @@ -261,8 +261,11 @@ export function resolveCapabilitiesForCreate(input: { type: AiProviderType; capabilities?: AiCapability[] | null; }): AiCapability[] { + if (input.capabilities != null) { + return parseCapabilities(input.capabilities); + } if (input.type === "custom") { - return parseCapabilities(input.capabilities ?? []); + return []; } return [...AI_PROVIDER_CAPABILITY_DEFAULTS[input.type]]; } diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index 995e9eaa0..195c8040e 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -124,6 +124,15 @@ export async function createAiProvider( capabilities }); + if (resolvedCapabilities.length === 0) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "At least one capability is required" + ) + ); + } + const [provider] = await db .insert(aiProviders) .values({ diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts index 7cbf1145a..1f22dc6ef 100644 --- a/server/routers/aiProvider/updateAiProvider.ts +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -131,20 +131,9 @@ export async function updateAiProvider( ? body.authType : (existing.authType as AiProviderAuthType); - if (body.capabilities !== undefined && providerType !== "custom") { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "Capabilities can only be updated for custom providers" - ) - ); - } - const nextCapabilities = - providerType === "custom" - ? body.capabilities !== undefined - ? body.capabilities - : parseCapabilities(existing.capabilities) + body.capabilities !== undefined + ? body.capabilities : parseCapabilities(existing.capabilities); const validation = z @@ -195,7 +184,7 @@ export async function updateAiProvider( if (body.authType !== undefined) { updateData.authType = body.authType; } - if (providerType === "custom" && body.capabilities !== undefined) { + if (body.capabilities !== undefined) { updateData.capabilities = serializeCapabilities(body.capabilities); } diff --git a/server/routers/aiProvider/validation.ts b/server/routers/aiProvider/validation.ts index a56ccad1c..cc1346325 100644 --- a/server/routers/aiProvider/validation.ts +++ b/server/routers/aiProvider/validation.ts @@ -112,5 +112,15 @@ export function refineProviderUpstreamFields( path: ["capabilities"] }); } + } else if ( + data.capabilities !== undefined && + data.capabilities !== null && + data.capabilities.length === 0 + ) { + ctx.addIssue({ + code: "custom", + message: "At least one capability is required", + path: ["capabilities"] + }); } } diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx index 62ffa6eb6..6dabdbaad 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx @@ -12,10 +12,7 @@ import { SettingsSectionHeader, SettingsSectionTitle } from "@app/components/Settings"; -import { - AiProviderCapabilitiesSelect, - capabilityLabelKey -} from "@app/components/AiProviderCapabilitiesSelect"; +import { AiProviderCapabilitiesSelect } from "@app/components/AiProviderCapabilitiesSelect"; import { SwitchInput } from "@app/components/SwitchInput"; import { Button } from "@app/components/ui/button"; import { @@ -49,7 +46,6 @@ export default function AiProviderGeneralPage() { const router = useRouter(); const t = useTranslations(); const [saveLoading, setSaveLoading] = useState(false); - const isCustom = provider.type === "custom"; const generalSchema = useMemo( () => @@ -63,10 +59,7 @@ export default function AiProviderGeneralPage() { capabilities: z.array(z.enum(AI_CAPABILITIES)).optional() }) .superRefine((data, ctx) => { - if ( - isCustom && - (!data.capabilities || data.capabilities.length === 0) - ) { + if (!data.capabilities || data.capabilities.length === 0) { ctx.addIssue({ code: "custom", message: t("aiProviderErrorCapabilitiesRequired"), @@ -74,7 +67,7 @@ export default function AiProviderGeneralPage() { }); } }), - [t, isCustom] + [t] ); type GeneralFormValues = z.infer; @@ -97,11 +90,9 @@ export default function AiProviderGeneralPage() { capabilities?: AiCapability[]; } = { name: values.name.trim(), - enabled: values.enabled + enabled: values.enabled, + capabilities: values.capabilities ?? [] }; - if (isCustom) { - body.capabilities = values.capabilities ?? []; - } const res = await api.post< AxiosResponse @@ -211,46 +202,20 @@ export default function AiProviderGeneralPage() { )} - {isCustom ? ( - - ) : ( -
- {( - provider.capabilities ?? - [] - ).map((cap) => ( - - {t( - capabilityLabelKey( - cap - ) - )} - - ))} -
- )} + - {isCustom - ? t( - "aiProviderCapabilitiesCustomDescription" - ) - : t( - "aiProviderCapabilitiesDescription" - )} + {t( + "aiProviderCapabilitiesDescription" + )} diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 0d9541e54..999782bc3 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -20,10 +20,7 @@ import { } from "@app/components/Settings"; import HeaderTitle from "@app/components/SettingsSectionTitle"; import { AiProviderAuthTypeSelect } from "@app/components/AiProviderAuthTypeSelect"; -import { - AiProviderCapabilitiesSelect, - capabilityLabelKey -} from "@app/components/AiProviderCapabilitiesSelect"; +import { AiProviderCapabilitiesSelect } from "@app/components/AiProviderCapabilitiesSelect"; import { AiProviderTypeSelect } from "@app/components/AiProviderTypeSelect"; import { HeadersInput } from "@app/components/HeadersInput"; import { StrategySelect } from "@app/components/StrategySelect"; @@ -93,14 +90,12 @@ export default function CreateAiProviderPage() { const providerType = form.watch("type"); const routingMode = form.watch("routingMode"); const authType = form.watch("authType"); - const capabilities = form.watch("capabilities"); const showUpstream = showsUpstreamUrlField(providerType, routingMode); const requireUpstream = upstreamUrlRequired(providerType, routingMode); const showRoutingMode = providerType === "custom"; const showTargets = providerType === "custom" && routingMode === "target"; const showApiKey = authTypeRequiresApiKey(authType ?? "bearer"); - const showCapabilitiesSelect = providerType === "custom"; async function createTargets( providerId: number, @@ -326,48 +321,20 @@ export default function CreateAiProviderPage() { )} - {showCapabilitiesSelect ? ( - - ) : ( -
- {( - capabilities ?? - defaultCapabilitiesForProvider( - providerType - ) - ).map((cap) => ( - - {t( - capabilityLabelKey( - cap - ) - )} - - ))} -
- )} + - {showCapabilitiesSelect - ? t( - "aiProviderCapabilitiesCustomDescription" - ) - : t( - "aiProviderCapabilitiesDescription" - )} + {t( + "aiProviderCapabilitiesDescription" + )} diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index 794d8ad62..04e38a71a 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -97,10 +97,7 @@ export function createAiProviderFormSchema(t: TranslateFn) { }); } - if ( - data.type === "custom" && - (!data.capabilities || data.capabilities.length === 0) - ) { + if (!data.capabilities || data.capabilities.length === 0) { ctx.addIssue({ code: "custom", message: t("aiProviderErrorCapabilitiesRequired"), @@ -187,8 +184,7 @@ export function toAiProviderCreatePayload(values: AiProviderFormValues) { upstreamUrl, apiKey: values.apiKey?.trim() ? values.apiKey.trim() : undefined, authType: values.authType ?? "bearer", - capabilities: - values.type === "custom" ? (values.capabilities ?? []) : undefined, + capabilities: values.capabilities ?? [], headers: values.headers && values.headers.length > 0 ? values.headers : null, skipTlsVerification: values.skipTlsVerification, @@ -216,7 +212,7 @@ export function toAiProviderUpdatePayload(values: AiProviderFormValues) { enabled: values.enabled ?? true }; - if (values.type === "custom" && values.capabilities) { + if (values.capabilities) { payload.capabilities = values.capabilities; } From 297cb9c8f2f0ecbcb9410455a9626231a9c76a8b Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 7 Aug 2026 12:25:29 -0400 Subject: [PATCH 077/296] set provider default capabilities --- server/lib/aiCapabilities.ts | 37 -------------- server/lib/aiProviderDefaults.ts | 51 ++++++++++++++++--- server/routers/aiProvider/createAiProvider.ts | 12 ++--- src/lib/aiProviderFormSchema.ts | 7 +-- 4 files changed, 51 insertions(+), 56 deletions(-) diff --git a/server/lib/aiCapabilities.ts b/server/lib/aiCapabilities.ts index 356122e7f..2b3691493 100644 --- a/server/lib/aiCapabilities.ts +++ b/server/lib/aiCapabilities.ts @@ -1,5 +1,4 @@ import type { Request } from "express"; -import type { AiProviderType } from "@server/lib/aiProviderDefaults"; export const AI_CAPABILITIES = [ "openai_chat", @@ -188,20 +187,6 @@ export const AI_CAPABILITY_DEFS: Record = } }; -export const AI_PROVIDER_CAPABILITY_DEFAULTS: Record< - Exclude, - readonly AiCapability[] -> = { - openai: ["openai_chat"], - anthropic: ["anthropic_messages"], - googleGemini: ["gemini_generate_content"], - vertexAi: ["google_generate_content"], - bedrock: ["bedrock_converse"], - microsoftFoundry: ["openai_chat"], - openRouter: ["openai_chat"], - vercelAiGateway: ["openai_chat"] -}; - export function isAiCapability(value: unknown): value is AiCapability { return ( typeof value === "string" && @@ -256,25 +241,3 @@ export function providerHasCapability( : capabilities; return list.includes(capability); } - -export function resolveCapabilitiesForCreate(input: { - type: AiProviderType; - capabilities?: AiCapability[] | null; -}): AiCapability[] { - if (input.capabilities != null) { - return parseCapabilities(input.capabilities); - } - if (input.type === "custom") { - return []; - } - return [...AI_PROVIDER_CAPABILITY_DEFAULTS[input.type]]; -} - -export function defaultsForProviderType( - type: AiProviderType -): readonly AiCapability[] { - if (type === "custom") { - return []; - } - return AI_PROVIDER_CAPABILITY_DEFAULTS[type]; -} diff --git a/server/lib/aiProviderDefaults.ts b/server/lib/aiProviderDefaults.ts index f24330ec8..a8b3a0ca0 100644 --- a/server/lib/aiProviderDefaults.ts +++ b/server/lib/aiProviderDefaults.ts @@ -1,4 +1,8 @@ import { decrypt, encrypt } from "@server/lib/crypto"; +import { + parseCapabilities, + type AiCapability +} from "@server/lib/aiCapabilities"; export type AiProviderType = | "openai" @@ -28,6 +32,7 @@ export type AiProviderRoutingMode = "url" | "target"; type AiProviderDefaults = { upstreamUrl: string | null; authType: AiProviderAuthType; + capabilities: readonly AiCapability[]; }; export const AI_PROVIDER_DEFAULTS: Record< @@ -36,35 +41,43 @@ export const AI_PROVIDER_DEFAULTS: Record< > = { openai: { upstreamUrl: "https://api.openai.com/v1", - authType: "bearer" + authType: "bearer", + capabilities: ["openai_chat", "openai_responses"] }, anthropic: { upstreamUrl: "https://api.anthropic.com", - authType: "x-api-key" + authType: "x-api-key", + capabilities: ["anthropic_messages"] }, googleGemini: { upstreamUrl: "https://generativelanguage.googleapis.com", - authType: "x-goog-api-key" + authType: "x-goog-api-key", + capabilities: ["gemini_generate_content"] }, vertexAi: { upstreamUrl: null, - authType: "bearer" + authType: "bearer", + capabilities: ["google_generate_content", "google_raw_predict"] }, bedrock: { upstreamUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", - authType: "bearer" + authType: "bearer", + capabilities: ["bedrock_converse"] }, microsoftFoundry: { upstreamUrl: null, - authType: "bearer" + authType: "bearer", + capabilities: ["openai_chat", "openai_responses", "anthropic_messages"] }, openRouter: { upstreamUrl: "https://openrouter.ai/api/v1", - authType: "bearer" + authType: "bearer", + capabilities: ["openai_chat"] }, vercelAiGateway: { upstreamUrl: "https://ai-gateway.vercel.sh/v1", - authType: "bearer" + authType: "bearer", + capabilities: ["openai_chat", "openai_responses"] } }; @@ -225,3 +238,25 @@ export function applyAiProviderAuthHeaders( break; } } + +export function resolveCapabilitiesForCreate(input: { + type: AiProviderType; + capabilities?: AiCapability[] | null; +}): AiCapability[] { + if (input.capabilities != null) { + return parseCapabilities(input.capabilities); + } + if (input.type === "custom") { + return []; + } + return [...AI_PROVIDER_DEFAULTS[input.type].capabilities]; +} + +export function defaultsForProviderType( + type: AiProviderType +): readonly AiCapability[] { + if (type === "custom") { + return []; + } + return AI_PROVIDER_DEFAULTS[type].capabilities; +} diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index 195c8040e..f115d674a 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -9,7 +9,11 @@ import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { encrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; -import { resolveAiProviderCreateFields } from "@server/lib/aiProviderDefaults"; +import { + resolveAiProviderCreateFields, + resolveCapabilitiesForCreate, + serializeAiProviderHeaders +} from "@server/lib/aiProviderDefaults"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import { toPublicAiProvider } from "@server/routers/aiProvider/types"; import { @@ -20,11 +24,7 @@ import { aiRoutingModeSchema, refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; -import { serializeAiProviderHeaders } from "@server/lib/aiProviderDefaults"; -import { - resolveCapabilitiesForCreate, - serializeCapabilities -} from "@server/lib/aiCapabilities"; +import { serializeCapabilities } from "@server/lib/aiCapabilities"; const paramsSchema = z.strictObject({ orgId: z.string().nonempty() diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index 04e38a71a..2f75c92ec 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -3,15 +3,12 @@ import { AI_PROVIDER_AUTH_TYPES, AI_PROVIDER_DEFAULTS, authTypeRequiresApiKey, + defaultsForProviderType, providerRequiresUpstreamUrl, type AiProviderAuthType, type AiProviderType } from "@server/lib/aiProviderDefaults"; -import { - AI_CAPABILITIES, - defaultsForProviderType, - type AiCapability -} from "@server/lib/aiCapabilities"; +import { AI_CAPABILITIES, type AiCapability } from "@server/lib/aiCapabilities"; type TranslateFn = (key: string) => string; From 6689a8d93ea7d4c2dcdd0dc8e4e9f0e2fe070c0f Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 7 Aug 2026 19:05:28 +0200 Subject: [PATCH 078/296] =?UTF-8?q?=F0=9F=9A=A7=20wip:=20test=20alert=20ru?= =?UTF-8?q?le?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- messages/en-US.json | 4 + server/auth/actions.ts | 1 + server/private/routers/alertRule/index.ts | 3 +- .../routers/alertRule/testSiteAlertRule.ts | 73 +++++++++++++ server/private/routers/external.ts | 8 ++ .../AlertRuleGraphEditor.tsx | 100 +++++++++++++----- 6 files changed, 163 insertions(+), 26 deletions(-) create mode 100644 server/private/routers/alertRule/testSiteAlertRule.ts diff --git a/messages/en-US.json b/messages/en-US.json index aff3be28b..2dd438a12 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1804,6 +1804,10 @@ "alertingRulesBannerDescription": "Each rule ties together what to watch (a site, health check, or resource), when to fire (for example offline or unhealthy), and how to notify your team via email, webhooks, or integrations. Use this list to create, enable, and manage those rules.", "alertingHealthChecksBannerTitle": "Monitor Health & Resources", "alertingHealthChecksBannerDescription": "Health checks are HTTP or TCP monitors you define once. You can then use them as sources in alert rules so you get notified when a target becomes healthy or unhealthy. Health checks on resources also appear here.", + "alertingTestRule": "Test Alert Rule", + "alertingNoActionsTitle": "No actions configured", + "alertingNoActionsSaveDescription": "Add at least one action so this rule can notify someone when it fires.", + "alertingNoActionsTestDescription": "Add at least one action before you can test this rule.", "standaloneHcTableTitle": "Health Checks", "standaloneHcSearchPlaceholder": "Search health checks…", "standaloneHcAddButton": "Create Health Check", diff --git a/server/auth/actions.ts b/server/auth/actions.ts index 741d7a057..4cb46f518 100644 --- a/server/auth/actions.ts +++ b/server/auth/actions.ts @@ -151,6 +151,7 @@ export enum ActionsEnum { createAlertRule = "createAlertRule", updateAlertRule = "updateAlertRule", deleteAlertRule = "deleteAlertRule", + testAlertRule = "testAlertRule", listAlertRules = "listAlertRules", listOrgLabels = "listOrgLabels", createOrgLabel = "createOrgLabel", diff --git a/server/private/routers/alertRule/index.ts b/server/private/routers/alertRule/index.ts index 19e35f7dc..e80a9ba16 100644 --- a/server/private/routers/alertRule/index.ts +++ b/server/private/routers/alertRule/index.ts @@ -15,4 +15,5 @@ export * from "./createAlertRule"; export * from "./updateAlertRule"; export * from "./deleteAlertRule"; export * from "./listAlertRules"; -export * from "./getAlertRule"; \ No newline at end of file +export * from "./getAlertRule"; +export * from "./testSiteAlertRule"; diff --git a/server/private/routers/alertRule/testSiteAlertRule.ts b/server/private/routers/alertRule/testSiteAlertRule.ts new file mode 100644 index 000000000..149ac9313 --- /dev/null +++ b/server/private/routers/alertRule/testSiteAlertRule.ts @@ -0,0 +1,73 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db } from "@server/db"; +import { + alertRules, + alertSites, + alertHealthChecks, + alertResources +} from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, asc, desc, eq, inArray, like, or, sql } from "drizzle-orm"; +import { ListAlertRulesResponse } from "@server/routers/alertRule/types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +const querySchema = z.strictObject({ + event: z.enum(["site_offline", "site_online", "site_toggle"]) +}); + +export async function testSiteAlertRule( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + const { orgId } = parsedParams.data; + + const parsedQuery = querySchema.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error).toString() + ) + ); + } + const { event } = parsedQuery.data; + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/private/routers/external.ts b/server/private/routers/external.ts index fab026418..c94bce1df 100644 --- a/server/private/routers/external.ts +++ b/server/private/routers/external.ts @@ -808,6 +808,14 @@ authenticated.get( alertRule.listAlertRules ); +authenticated.get( + "/org/:orgId/test-site-alert-rule/:alertRuleId", + verifyValidLicense, + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.testAlertRule), + alertRule.testSiteAlertRule +); + authenticated.get( "/org/:orgId/alert-rule/:alertRuleId", verifyValidLicense, diff --git a/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx b/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx index 7cac1960f..a10f8f3b3 100644 --- a/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx +++ b/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx @@ -6,7 +6,9 @@ import { AlertRuleSourceFields, AlertRuleTriggerFields } from "@app/components/alert-rule-editor/AlertRuleFields"; +import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; import { SettingsContainer } from "@app/components/Settings"; +import { SwitchInput } from "@app/components/SwitchInput"; import { Button } from "@app/components/ui/button"; import { Card, CardContent } from "@app/components/ui/card"; import { @@ -19,6 +21,7 @@ import { FormMessage } from "@app/components/ui/form"; import { Input } from "@app/components/ui/input"; +import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { buildFormSchema, @@ -27,19 +30,15 @@ import { type AlertRuleFormValues } from "@app/lib/alertRuleForm"; import { createApiClient, formatAxiosError } from "@app/lib/api"; -import { useEnvContext } from "@app/hooks/useEnvContext"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; import type { CreateAlertRuleResponse } from "@server/routers/alertRule/types"; import type { AxiosResponse } from "axios"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { ChevronLeft, Cog, Flag, Zap } from "lucide-react"; -import Link from "next/link"; -import { useRouter } from "next/navigation"; -import { useMemo, useState, type ReactNode } from "react"; -import { useFieldArray, useForm, type Resolver } from "react-hook-form"; +import { Cog, Flag, Zap, ZapIcon } from "lucide-react"; import { useTranslations } from "next-intl"; -import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; -import { SwitchInput } from "@app/components/SwitchInput"; -import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { useRouter } from "next/navigation"; +import { useActionState, useMemo, useTransition, type ReactNode } from "react"; +import { useFieldArray, useForm, type Resolver } from "react-hook-form"; import { Badge } from "../ui/badge"; const FORM_ID = "alert-rule-form"; @@ -115,7 +114,6 @@ export default function AlertRuleGraphEditor({ const t = useTranslations(); const router = useRouter(); const api = createApiClient(useEnvContext()); - const [isSaving, setIsSaving] = useState(false); const schema = useMemo(() => buildFormSchema(t), [t]); const form = useForm({ resolver: zodResolver(schema) as Resolver, @@ -127,8 +125,22 @@ export default function AlertRuleGraphEditor({ name: "actions" }); - const onSubmit = form.handleSubmit(async (values) => { - setIsSaving(true); + const saveAlert = async () => { + const isValid = await form.trigger(); + if (!isValid) { + const values = form.getValues(); + if (values.actions.length === 0) { + toast({ + variant: "warning", + title: t("alertingNoActionsTitle"), + description: t("alertingNoActionsSaveDescription") + }); + } + return; + } + + const values = form.getValues(); + try { const payload = formValuesToApiPayload(values); if (isNew) { @@ -158,14 +170,37 @@ export default function AlertRuleGraphEditor({ description: formatAxiosError(e), variant: "destructive" }); - } finally { - setIsSaving(false); } - }); + // const submit = form.handleSubmit(async (values) => { + + // }); + + // await submit(); + }; + + const testAlert = async () => { + const isValid = await form.trigger(); + if (!isValid) { + const values = form.getValues(); + if (values.actions.length === 0) { + toast({ + variant: "warning", + title: t("alertingNoActionsTitle"), + description: t("alertingNoActionsTestDescription") + }); + } + return; + } + + const values = form.getValues(); + }; + + const [, formAction, isSaving] = useActionState(saveAlert, null); + const [isTestingAlert, startTransition] = useTransition(); return (
- +
@@ -263,14 +298,29 @@ export default function AlertRuleGraphEditor({ )} /> - +
+ + + +
From 12056aebc6ac219d27814249b27d19426b6be5f7 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 7 Aug 2026 13:54:14 -0400 Subject: [PATCH 079/296] Add basic cost calculations for testing --- config/models.json | 7559 ++++++++++++++++++++++++++ package-lock.json | 7 + package.json | 3 +- server/lib/aiModelPricing.ts | 230 + server/lib/aiUsageExtraction.ts | 468 ++ server/routers/aiGateway/pipeline.ts | 111 +- 6 files changed, 8374 insertions(+), 4 deletions(-) create mode 100644 config/models.json create mode 100644 server/lib/aiModelPricing.ts create mode 100644 server/lib/aiUsageExtraction.ts diff --git a/config/models.json b/config/models.json new file mode 100644 index 000000000..dccd0a370 --- /dev/null +++ b/config/models.json @@ -0,0 +1,7559 @@ +{ + "data": [ + { + "id": "ai21.j2-mid-v1", + "name": "ai21.j2-mid-v1", + "provider": "bedrock", + "input_cost_per_token": 0.0000125, + "output_cost_per_token": 0.0000125, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ai21.j2-ultra-v1", + "name": "ai21.j2-ultra-v1", + "provider": "bedrock", + "input_cost_per_token": 0.0000188, + "output_cost_per_token": 0.0000188, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ai21.jamba-1-5-large-v1:0", + "name": "ai21.jamba-1-5-large-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ai21.jamba-1-5-mini-v1:0", + "name": "ai21.jamba-1-5-mini-v1:0", + "provider": "bedrock", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ai21.jamba-instruct-v1:0", + "name": "ai21.jamba-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.writer.palmyra-x4-v1:0", + "name": "us.writer.palmyra-x4-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.writer.palmyra-x5-v1:0", + "name": "us.writer.palmyra-x5-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "writer.palmyra-x4-v1:0", + "name": "writer.palmyra-x4-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "writer.palmyra-x5-v1:0", + "name": "writer.palmyra-x5-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "amazon.nova-lite-v1:0", + "name": "amazon.nova-lite-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 6e-8, + "output_cost_per_token": 2.4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "amazon.nova-2-lite-v1:0", + "name": "amazon.nova-2-lite-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "amazon.nova-2-pro-preview-20251202-v1:0", + "name": "amazon.nova-2-pro-preview-20251202-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000021875, + "output_cost_per_token": 0.0000175, + "cache_read_input_token_cost": 5.46875e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.amazon.nova-2-lite-v1:0", + "name": "apac.amazon.nova-2-lite-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 3.3e-7, + "output_cost_per_token": 0.00000275, + "cache_read_input_token_cost": 8.25e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.amazon.nova-2-pro-preview-20251202-v1:0", + "name": "apac.amazon.nova-2-pro-preview-20251202-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000021875, + "output_cost_per_token": 0.0000175, + "cache_read_input_token_cost": 5.46875e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.amazon.nova-2-lite-v1:0", + "name": "eu.amazon.nova-2-lite-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 3.3e-7, + "output_cost_per_token": 0.00000275, + "cache_read_input_token_cost": 8.25e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.amazon.nova-2-pro-preview-20251202-v1:0", + "name": "eu.amazon.nova-2-pro-preview-20251202-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000021875, + "output_cost_per_token": 0.0000175, + "cache_read_input_token_cost": 5.46875e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.amazon.nova-2-lite-v1:0", + "name": "us.amazon.nova-2-lite-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 3.3e-7, + "output_cost_per_token": 0.00000275, + "cache_read_input_token_cost": 8.25e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.amazon.nova-2-pro-preview-20251202-v1:0", + "name": "us.amazon.nova-2-pro-preview-20251202-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000021875, + "output_cost_per_token": 0.0000175, + "cache_read_input_token_cost": 5.46875e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "amazon.nova-micro-v1:0", + "name": "amazon.nova-micro-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 3.5e-8, + "output_cost_per_token": 1.4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "amazon.nova-pro-v1:0", + "name": "amazon.nova-pro-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 8e-7, + "output_cost_per_token": 0.0000032, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "twelvelabs.pegasus-1-2-v1:0", + "name": "twelvelabs.pegasus-1-2-v1:0", + "provider": "bedrock", + "input_cost_per_token": null, + "output_cost_per_token": 0.0000075, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.twelvelabs.pegasus-1-2-v1:0", + "name": "us.twelvelabs.pegasus-1-2-v1:0", + "provider": "bedrock", + "input_cost_per_token": null, + "output_cost_per_token": 0.0000075, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.twelvelabs.pegasus-1-2-v1:0", + "name": "eu.twelvelabs.pegasus-1-2-v1:0", + "provider": "bedrock", + "input_cost_per_token": null, + "output_cost_per_token": 0.0000075, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "amazon.titan-text-express-v1", + "name": "amazon.titan-text-express-v1", + "provider": "bedrock", + "input_cost_per_token": 0.0000013, + "output_cost_per_token": 0.0000017, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "amazon.titan-text-lite-v1", + "name": "amazon.titan-text-lite-v1", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "amazon.titan-text-premier-v1:0", + "name": "amazon.titan-text-premier-v1:0", + "provider": "bedrock", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-3-5-haiku-20241022-v1:0", + "name": "anthropic.claude-3-5-haiku-20241022-v1:0", + "provider": "bedrock", + "input_cost_per_token": 8e-7, + "output_cost_per_token": 0.000004, + "cache_read_input_token_cost": 8e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-haiku-4-5-20251001-v1:0", + "name": "anthropic.claude-haiku-4-5-20251001-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-haiku-4-5@20251001", + "name": "anthropic.claude-haiku-4-5@20251001", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "name": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "name": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-3-7-sonnet-20240620-v1:0", + "name": "anthropic.claude-3-7-sonnet-20240620-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000036, + "output_cost_per_token": 0.000018, + "cache_read_input_token_cost": 3.6e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-3-7-sonnet-20250219-v1:0", + "name": "anthropic.claude-3-7-sonnet-20250219-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-3-haiku-20240307-v1:0", + "name": "anthropic.claude-3-haiku-20240307-v1:0", + "provider": "bedrock", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-3-opus-20240229-v1:0", + "name": "anthropic.claude-3-opus-20240229-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-3-sonnet-20240229-v1:0", + "name": "anthropic.claude-3-sonnet-20240229-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-instant-v1", + "name": "anthropic.claude-instant-v1", + "provider": "bedrock", + "input_cost_per_token": 8e-7, + "output_cost_per_token": 0.0000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-opus-4-1-20250805-v1:0", + "name": "anthropic.claude-opus-4-1-20250805-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-opus-4-20250514-v1:0", + "name": "anthropic.claude-opus-4-20250514-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-opus-4-5-20251101-v1:0", + "name": "anthropic.claude-opus-4-5-20251101-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-opus-4-6-v1", + "name": "anthropic.claude-opus-4-6-v1", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-opus-4-6-v1", + "name": "global.anthropic.claude-opus-4-6-v1", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-opus-4-6-v1", + "name": "us.anthropic.claude-opus-4-6-v1", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-opus-4-6-v1", + "name": "eu.anthropic.claude-opus-4-6-v1", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "au.anthropic.claude-opus-4-6-v1", + "name": "au.anthropic.claude-opus-4-6-v1", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-opus-4-7", + "name": "anthropic.claude-opus-4-7", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-mythos-preview", + "name": "anthropic.claude-mythos-preview", + "provider": "bedrock", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-opus-4-7", + "name": "global.anthropic.claude-opus-4-7", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-opus-4-7", + "name": "us.anthropic.claude-opus-4-7", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-opus-4-7", + "name": "eu.anthropic.claude-opus-4-7", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "au.anthropic.claude-opus-4-7", + "name": "au.anthropic.claude-opus-4-7", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-fable-5", + "name": "anthropic.claude-fable-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00005, + "cache_read_input_token_cost": 0.000001, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-fable-5", + "name": "global.anthropic.claude-fable-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00005, + "cache_read_input_token_cost": 0.000001, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-fable-5", + "name": "us.anthropic.claude-fable-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000011, + "output_cost_per_token": 0.000055, + "cache_read_input_token_cost": 0.0000011, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-fable-5", + "name": "eu.anthropic.claude-fable-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000011, + "output_cost_per_token": 0.000055, + "cache_read_input_token_cost": 0.0000011, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-opus-5", + "name": "anthropic.claude-opus-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-opus-5", + "name": "global.anthropic.claude-opus-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-opus-5", + "name": "us.anthropic.claude-opus-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-opus-5", + "name": "eu.anthropic.claude-opus-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "au.anthropic.claude-opus-5", + "name": "au.anthropic.claude-opus-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "jp.anthropic.claude-opus-5", + "name": "jp.anthropic.claude-opus-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-opus-4-8", + "name": "anthropic.claude-opus-4-8", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-opus-4-8", + "name": "global.anthropic.claude-opus-4-8", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-opus-4-8", + "name": "us.anthropic.claude-opus-4-8", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-opus-4-8", + "name": "eu.anthropic.claude-opus-4-8", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "au.anthropic.claude-opus-4-8", + "name": "au.anthropic.claude-opus-4-8", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "jp.anthropic.claude-opus-4-8", + "name": "jp.anthropic.claude-opus-4-8", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "jp.anthropic.claude-opus-4-7", + "name": "jp.anthropic.claude-opus-4-7", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-sonnet-5", + "name": "anthropic.claude-sonnet-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-sonnet-5", + "name": "global.anthropic.claude-sonnet-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-sonnet-5", + "name": "us.anthropic.claude-sonnet-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000022, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 2.2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-sonnet-5", + "name": "eu.anthropic.claude-sonnet-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000022, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 2.2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "au.anthropic.claude-sonnet-5", + "name": "au.anthropic.claude-sonnet-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000022, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 2.2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "jp.anthropic.claude-sonnet-5", + "name": "jp.anthropic.claude-sonnet-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000022, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 2.2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-sonnet-4-6", + "name": "anthropic.claude-sonnet-4-6", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-sonnet-4-6", + "name": "global.anthropic.claude-sonnet-4-6", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-sonnet-4-6", + "name": "us.anthropic.claude-sonnet-4-6", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000033, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 3.3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-sonnet-4-6", + "name": "eu.anthropic.claude-sonnet-4-6", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000033, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 3.3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "au.anthropic.claude-sonnet-4-6", + "name": "au.anthropic.claude-sonnet-4-6", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000033, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 3.3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "jp.anthropic.claude-sonnet-4-6", + "name": "jp.anthropic.claude-sonnet-4-6", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000033, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 3.3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-sonnet-4-20250514-v1:0", + "name": "anthropic.claude-sonnet-4-20250514-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-v1", + "name": "anthropic.claude-v1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "anthropic.claude-v2:1", + "name": "anthropic.claude-v2:1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.amazon.nova-lite-v1:0", + "name": "apac.amazon.nova-lite-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 6.3e-8, + "output_cost_per_token": 2.52e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.amazon.nova-micro-v1:0", + "name": "apac.amazon.nova-micro-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 3.7e-8, + "output_cost_per_token": 1.48e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.amazon.nova-pro-v1:0", + "name": "apac.amazon.nova-pro-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 8.4e-7, + "output_cost_per_token": 0.00000336, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.anthropic.claude-3-5-sonnet-20240620-v1:0", + "name": "apac.anthropic.claude-3-5-sonnet-20240620-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.anthropic.claude-3-5-sonnet-20241022-v2:0", + "name": "apac.anthropic.claude-3-5-sonnet-20241022-v2:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.anthropic.claude-3-haiku-20240307-v1:0", + "name": "apac.anthropic.claude-3-haiku-20240307-v1:0", + "provider": "bedrock", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.anthropic.claude-haiku-4-5-20251001-v1:0", + "name": "apac.anthropic.claude-haiku-4-5-20251001-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000055, + "cache_read_input_token_cost": 1.1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.anthropic.claude-3-sonnet-20240229-v1:0", + "name": "apac.anthropic.claude-3-sonnet-20240229-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "apac.anthropic.claude-sonnet-4-20250514-v1:0", + "name": "apac.anthropic.claude-sonnet-4-20250514-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "au.anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "au.anthropic.claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000033, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 3.3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/command-r-plus", + "name": "command-r-plus", + "provider": "azure", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-haiku-4-5", + "name": "claude-haiku-4-5", + "provider": "azure_ai", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-opus-4-5", + "name": "claude-opus-4-5", + "provider": "azure_ai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-opus-4-6", + "name": "claude-opus-4-6", + "provider": "azure_ai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-opus-4-7", + "name": "claude-opus-4-7", + "provider": "azure_ai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-fable-5", + "name": "claude-fable-5", + "provider": "azure_ai", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00005, + "cache_read_input_token_cost": 0.000001, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-opus-5", + "name": "claude-opus-5", + "provider": "azure_ai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-opus-4-8", + "name": "claude-opus-4-8", + "provider": "azure_ai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-opus-4-1", + "name": "claude-opus-4-1", + "provider": "azure_ai", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-sonnet-4-5", + "name": "claude-sonnet-4-5", + "provider": "azure_ai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-sonnet-5", + "name": "claude-sonnet-5", + "provider": "azure_ai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/claude-sonnet-4-6", + "name": "claude-sonnet-4-6", + "provider": "azure_ai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/computer-use-preview", + "name": "computer-use-preview", + "provider": "azure", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/gpt-oss-120b", + "name": "gpt-oss-120b", + "provider": "azure_ai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/gpt-5.5", + "name": "gpt-5.5", + "provider": "azure_ai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/gpt-5.5-2026-04-23", + "name": "gpt-5.5-2026-04-23", + "provider": "azure_ai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/gpt-5.4", + "name": "gpt-5.4", + "provider": "azure_ai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 2.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/gpt-5.4-2026-03-05", + "name": "gpt-5.4-2026-03-05", + "provider": "azure_ai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 2.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/gpt-5.4-mini", + "name": "gpt-5.4-mini", + "provider": "azure_ai", + "input_cost_per_token": 7.5e-7, + "output_cost_per_token": 0.0000045, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/gpt-5.4-mini-2026-03-17", + "name": "gpt-5.4-mini-2026-03-17", + "provider": "azure_ai", + "input_cost_per_token": 7.5e-7, + "output_cost_per_token": 0.0000045, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/gpt-5.4-nano", + "name": "gpt-5.4-nano", + "provider": "azure_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/gpt-5.4-nano-2026-03-17", + "name": "gpt-5.4-nano-2026-03-17", + "provider": "azure_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/model_router", + "name": "model_router", + "provider": "azure_ai", + "input_cost_per_token": 1.4e-7, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-4o-2024-08-06", + "name": "eu/gpt-4o-2024-08-06", + "provider": "azure", + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-4o-2024-11-20", + "name": "eu/gpt-4o-2024-11-20", + "provider": "azure", + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-4o-mini-2024-07-18", + "name": "eu/gpt-4o-mini-2024-07-18", + "provider": "azure", + "input_cost_per_token": 1.65e-7, + "output_cost_per_token": 6.6e-7, + "cache_read_input_token_cost": 8.3e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5-2025-08-07", + "name": "eu/gpt-5-2025-08-07", + "provider": "azure", + "input_cost_per_token": 0.000001375, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 1.375e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5-mini-2025-08-07", + "name": "eu/gpt-5-mini-2025-08-07", + "provider": "azure", + "input_cost_per_token": 2.75e-7, + "output_cost_per_token": 0.0000022, + "cache_read_input_token_cost": 2.75e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.1", + "name": "eu/gpt-5.1", + "provider": "azure", + "input_cost_per_token": 0.00000138, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 1.4e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.1-chat", + "name": "eu/gpt-5.1-chat", + "provider": "azure", + "input_cost_per_token": 0.00000138, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 1.4e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5-nano-2025-08-07", + "name": "eu/gpt-5-nano-2025-08-07", + "provider": "azure", + "input_cost_per_token": 5.5e-8, + "output_cost_per_token": 4.4e-7, + "cache_read_input_token_cost": 5.5e-9, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/o1-2024-12-17", + "name": "eu/o1-2024-12-17", + "provider": "azure", + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/o1-mini-2024-09-12", + "name": "eu/o1-mini-2024-09-12", + "provider": "azure", + "input_cost_per_token": 0.00000121, + "output_cost_per_token": 0.00000484, + "cache_read_input_token_cost": 6.05e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/o1-preview-2024-09-12", + "name": "eu/o1-preview-2024-09-12", + "provider": "azure", + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/o3-mini-2025-01-31", + "name": "eu/o3-mini-2025-01-31", + "provider": "azure", + "input_cost_per_token": 0.00000121, + "output_cost_per_token": 0.00000484, + "cache_read_input_token_cost": 6.05e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/global-standard/gpt-4o-2024-08-06", + "name": "global-standard/gpt-4o-2024-08-06", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/global-standard/gpt-4o-2024-11-20", + "name": "global-standard/gpt-4o-2024-11-20", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/global-standard/gpt-4o-mini", + "name": "global-standard/gpt-4o-mini", + "provider": "azure", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/global/gpt-4o-2024-08-06", + "name": "global/gpt-4o-2024-08-06", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/global/gpt-4o-2024-11-20", + "name": "global/gpt-4o-2024-11-20", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/global/gpt-5.1", + "name": "global/gpt-5.1", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/global/gpt-5.1-chat", + "name": "global/gpt-5.1-chat", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-3.5-turbo", + "name": "gpt-3.5-turbo", + "provider": "azure", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-3.5-turbo-0125", + "name": "gpt-3.5-turbo-0125", + "provider": "azure", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-35-turbo", + "name": "gpt-35-turbo", + "provider": "azure", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-35-turbo-0125", + "name": "gpt-35-turbo-0125", + "provider": "azure", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-35-turbo-1106", + "name": "gpt-35-turbo-1106", + "provider": "azure", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-35-turbo-16k", + "name": "gpt-35-turbo-16k", + "provider": "azure", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000004, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-35-turbo-16k-0613", + "name": "gpt-35-turbo-16k-0613", + "provider": "azure", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000004, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4", + "name": "gpt-4", + "provider": "azure", + "input_cost_per_token": 0.00003, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4-0125-preview", + "name": "gpt-4-0125-preview", + "provider": "azure", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4-0613", + "name": "gpt-4-0613", + "provider": "azure", + "input_cost_per_token": 0.00003, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4-1106-preview", + "name": "gpt-4-1106-preview", + "provider": "azure", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4-32k", + "name": "gpt-4-32k", + "provider": "azure", + "input_cost_per_token": 0.00006, + "output_cost_per_token": 0.00012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4-32k-0613", + "name": "gpt-4-32k-0613", + "provider": "azure", + "input_cost_per_token": 0.00006, + "output_cost_per_token": 0.00012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4-turbo", + "name": "gpt-4-turbo", + "provider": "azure", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4-turbo-2024-04-09", + "name": "gpt-4-turbo-2024-04-09", + "provider": "azure", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4-turbo-vision-preview", + "name": "gpt-4-turbo-vision-preview", + "provider": "azure", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4.1", + "name": "gpt-4.1", + "provider": "azure", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4.1-2025-04-14", + "name": "gpt-4.1-2025-04-14", + "provider": "azure", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4.1-mini", + "name": "gpt-4.1-mini", + "provider": "azure", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.0000016, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4.1-mini-2025-04-14", + "name": "gpt-4.1-mini-2025-04-14", + "provider": "azure", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.0000016, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4.1-nano", + "name": "gpt-4.1-nano", + "provider": "azure", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4.1-nano-2025-04-14", + "name": "gpt-4.1-nano-2025-04-14", + "provider": "azure", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4.5-preview", + "name": "gpt-4.5-preview", + "provider": "azure", + "input_cost_per_token": 0.000075, + "output_cost_per_token": 0.00015, + "cache_read_input_token_cost": 0.0000375, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4o", + "name": "gpt-4o", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4o-2024-05-13", + "name": "gpt-4o-2024-05-13", + "provider": "azure", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4o-2024-08-06", + "name": "gpt-4o-2024-08-06", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4o-2024-11-20", + "name": "gpt-4o-2024-11-20", + "provider": "azure", + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-audio-2025-08-28", + "name": "gpt-audio-2025-08-28", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-audio-1.5-2026-02-23", + "name": "gpt-audio-1.5-2026-02-23", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-audio-mini-2025-10-06", + "name": "gpt-audio-mini-2025-10-06", + "provider": "azure", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4o-audio-preview-2024-12-17", + "name": "gpt-4o-audio-preview-2024-12-17", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4o-mini", + "name": "gpt-4o-mini", + "provider": "azure", + "input_cost_per_token": 1.65e-7, + "output_cost_per_token": 6.6e-7, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4o-mini-2024-07-18", + "name": "gpt-4o-mini-2024-07-18", + "provider": "azure", + "input_cost_per_token": 1.65e-7, + "output_cost_per_token": 6.6e-7, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-4o-mini-audio-preview-2024-12-17", + "name": "gpt-4o-mini-audio-preview-2024-12-17", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.1-2025-11-13", + "name": "gpt-5.1-2025-11-13", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.1-chat-2025-11-13", + "name": "gpt-5.1-chat-2025-11-13", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5", + "name": "gpt-5", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5-2025-08-07", + "name": "gpt-5-2025-08-07", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5-chat", + "name": "gpt-5-chat", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5-chat-latest", + "name": "gpt-5-chat-latest", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5-mini", + "name": "gpt-5-mini", + "provider": "azure", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5-mini-2025-08-07", + "name": "gpt-5-mini-2025-08-07", + "provider": "azure", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5-nano", + "name": "gpt-5-nano", + "provider": "azure", + "input_cost_per_token": 5e-8, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 5e-9, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5-nano-2025-08-07", + "name": "gpt-5-nano-2025-08-07", + "provider": "azure", + "input_cost_per_token": 5e-8, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 5e-9, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.1", + "name": "gpt-5.1", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.1-chat", + "name": "gpt-5.1-chat", + "provider": "azure", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.2", + "name": "gpt-5.2", + "provider": "azure", + "input_cost_per_token": 0.00000175, + "output_cost_per_token": 0.000014, + "cache_read_input_token_cost": 1.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.2-2025-12-11", + "name": "gpt-5.2-2025-12-11", + "provider": "azure", + "input_cost_per_token": 0.00000175, + "output_cost_per_token": 0.000014, + "cache_read_input_token_cost": 1.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.2-chat", + "name": "gpt-5.2-chat", + "provider": "azure", + "input_cost_per_token": 0.00000175, + "output_cost_per_token": 0.000014, + "cache_read_input_token_cost": 1.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.2-chat-2025-12-11", + "name": "gpt-5.2-chat-2025-12-11", + "provider": "azure", + "input_cost_per_token": 0.00000175, + "output_cost_per_token": 0.000014, + "cache_read_input_token_cost": 1.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.3-chat", + "name": "gpt-5.3-chat", + "provider": "azure", + "input_cost_per_token": 0.00000175, + "output_cost_per_token": 0.000014, + "cache_read_input_token_cost": 1.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.4", + "name": "gpt-5.4", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 2.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.4", + "name": "us/gpt-5.4", + "provider": "azure", + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 2.8e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.4", + "name": "eu/gpt-5.4", + "provider": "azure", + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 2.8e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.4-2026-03-05", + "name": "gpt-5.4-2026-03-05", + "provider": "azure", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 2.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.4-2026-03-05", + "name": "us/gpt-5.4-2026-03-05", + "provider": "azure", + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 2.8e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.4-2026-03-05", + "name": "eu/gpt-5.4-2026-03-05", + "provider": "azure", + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 2.8e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.6", + "name": "gpt-5.6", + "provider": "azure", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.6-sol", + "name": "gpt-5.6-sol", + "provider": "azure", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.6-terra", + "name": "gpt-5.6-terra", + "provider": "azure", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.6-luna", + "name": "gpt-5.6-luna", + "provider": "azure", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": 2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.6", + "name": "us/gpt-5.6", + "provider": "azure", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.000033, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.6-sol", + "name": "us/gpt-5.6-sol", + "provider": "azure", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.000033, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.6-terra", + "name": "us/gpt-5.6-terra", + "provider": "azure", + "input_cost_per_token": 0.0000022, + "output_cost_per_token": 0.0000132, + "cache_read_input_token_cost": 2.2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.6-luna", + "name": "us/gpt-5.6-luna", + "provider": "azure", + "input_cost_per_token": 2.2e-7, + "output_cost_per_token": 0.00000132, + "cache_read_input_token_cost": 2.2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.6", + "name": "eu/gpt-5.6", + "provider": "azure", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.000033, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.6-sol", + "name": "eu/gpt-5.6-sol", + "provider": "azure", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.000033, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.6-terra", + "name": "eu/gpt-5.6-terra", + "provider": "azure", + "input_cost_per_token": 0.0000022, + "output_cost_per_token": 0.0000132, + "cache_read_input_token_cost": 2.2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.6-luna", + "name": "eu/gpt-5.6-luna", + "provider": "azure", + "input_cost_per_token": 2.2e-7, + "output_cost_per_token": 0.00000132, + "cache_read_input_token_cost": 2.2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.5", + "name": "gpt-5.5", + "provider": "azure", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.5", + "name": "us/gpt-5.5", + "provider": "azure", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.000033, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.5", + "name": "eu/gpt-5.5", + "provider": "azure", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.000033, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.5-2026-04-23", + "name": "gpt-5.5-2026-04-23", + "provider": "azure", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.5-2026-04-23", + "name": "us/gpt-5.5-2026-04-23", + "provider": "azure", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.000033, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/eu/gpt-5.5-2026-04-23", + "name": "eu/gpt-5.5-2026-04-23", + "provider": "azure", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.000033, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.4-mini", + "name": "gpt-5.4-mini", + "provider": "azure", + "input_cost_per_token": 7.5e-7, + "output_cost_per_token": 0.0000045, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.4-mini-2026-03-17", + "name": "gpt-5.4-mini-2026-03-17", + "provider": "azure", + "input_cost_per_token": 7.5e-7, + "output_cost_per_token": 0.0000045, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.4-nano", + "name": "gpt-5.4-nano", + "provider": "azure", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-5.4-nano-2026-03-17", + "name": "gpt-5.4-nano-2026-03-17", + "provider": "azure", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/mistral-large-2402", + "name": "mistral-large-2402", + "provider": "azure", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/mistral-large-latest", + "name": "mistral-large-latest", + "provider": "azure", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o1", + "name": "o1", + "provider": "azure", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": 0.0000075, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o1-2024-12-17", + "name": "o1-2024-12-17", + "provider": "azure", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": 0.0000075, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o1-mini", + "name": "o1-mini", + "provider": "azure", + "input_cost_per_token": 0.00000121, + "output_cost_per_token": 0.00000484, + "cache_read_input_token_cost": 6.05e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o1-mini-2024-09-12", + "name": "o1-mini-2024-09-12", + "provider": "azure", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000044, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o1-preview", + "name": "o1-preview", + "provider": "azure", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": 0.0000075, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o1-preview-2024-09-12", + "name": "o1-preview-2024-09-12", + "provider": "azure", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": 0.0000075, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o3", + "name": "o3", + "provider": "azure", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o3-2025-04-16", + "name": "o3-2025-04-16", + "provider": "azure", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o3-mini", + "name": "o3-mini", + "provider": "azure", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000044, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o3-mini-2025-01-31", + "name": "o3-mini-2025-01-31", + "provider": "azure", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000044, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o4-mini", + "name": "o4-mini", + "provider": "azure", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000044, + "cache_read_input_token_cost": 2.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/o4-mini-2025-04-16", + "name": "o4-mini-2025-04-16", + "provider": "azure", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000044, + "cache_read_input_token_cost": 2.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-4.1-2025-04-14", + "name": "us/gpt-4.1-2025-04-14", + "provider": "azure", + "input_cost_per_token": 0.0000022, + "output_cost_per_token": 0.0000088, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-4.1-mini-2025-04-14", + "name": "us/gpt-4.1-mini-2025-04-14", + "provider": "azure", + "input_cost_per_token": 4.4e-7, + "output_cost_per_token": 0.00000176, + "cache_read_input_token_cost": 1.1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-4.1-nano-2025-04-14", + "name": "us/gpt-4.1-nano-2025-04-14", + "provider": "azure", + "input_cost_per_token": 1.1e-7, + "output_cost_per_token": 4.4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-4o-2024-08-06", + "name": "us/gpt-4o-2024-08-06", + "provider": "azure", + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 0.000001375, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-4o-2024-11-20", + "name": "us/gpt-4o-2024-11-20", + "provider": "azure", + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-4o-mini-2024-07-18", + "name": "us/gpt-4o-mini-2024-07-18", + "provider": "azure", + "input_cost_per_token": 1.65e-7, + "output_cost_per_token": 6.6e-7, + "cache_read_input_token_cost": 8.3e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5-2025-08-07", + "name": "us/gpt-5-2025-08-07", + "provider": "azure", + "input_cost_per_token": 0.000001375, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 1.375e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5-mini-2025-08-07", + "name": "us/gpt-5-mini-2025-08-07", + "provider": "azure", + "input_cost_per_token": 2.75e-7, + "output_cost_per_token": 0.0000022, + "cache_read_input_token_cost": 2.75e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5-nano-2025-08-07", + "name": "us/gpt-5-nano-2025-08-07", + "provider": "azure", + "input_cost_per_token": 5.5e-8, + "output_cost_per_token": 4.4e-7, + "cache_read_input_token_cost": 5.5e-9, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.1", + "name": "us/gpt-5.1", + "provider": "azure", + "input_cost_per_token": 0.00000138, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 1.4e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/gpt-5.1-chat", + "name": "us/gpt-5.1-chat", + "provider": "azure", + "input_cost_per_token": 0.00000138, + "output_cost_per_token": 0.000011, + "cache_read_input_token_cost": 1.4e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/o1-2024-12-17", + "name": "us/o1-2024-12-17", + "provider": "azure", + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/o1-mini-2024-09-12", + "name": "us/o1-mini-2024-09-12", + "provider": "azure", + "input_cost_per_token": 0.00000121, + "output_cost_per_token": 0.00000484, + "cache_read_input_token_cost": 6.05e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/o1-preview-2024-09-12", + "name": "us/o1-preview-2024-09-12", + "provider": "azure", + "input_cost_per_token": 0.0000165, + "output_cost_per_token": 0.000066, + "cache_read_input_token_cost": 0.00000825, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/o3-2025-04-16", + "name": "us/o3-2025-04-16", + "provider": "azure", + "input_cost_per_token": 0.0000022, + "output_cost_per_token": 0.0000088, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/o3-mini-2025-01-31", + "name": "us/o3-mini-2025-01-31", + "provider": "azure", + "input_cost_per_token": 0.00000121, + "output_cost_per_token": 0.00000484, + "cache_read_input_token_cost": 6.05e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/us/o4-mini-2025-04-16", + "name": "us/o4-mini-2025-04-16", + "provider": "azure", + "input_cost_per_token": 0.00000121, + "output_cost_per_token": 0.00000484, + "cache_read_input_token_cost": 3.1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Llama-3.2-11B-Vision-Instruct", + "name": "Llama-3.2-11B-Vision-Instruct", + "provider": "azure_ai", + "input_cost_per_token": 3.7e-7, + "output_cost_per_token": 3.7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Llama-3.2-90B-Vision-Instruct", + "name": "Llama-3.2-90B-Vision-Instruct", + "provider": "azure_ai", + "input_cost_per_token": 0.00000204, + "output_cost_per_token": 0.00000204, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Llama-3.3-70B-Instruct", + "name": "Llama-3.3-70B-Instruct", + "provider": "azure_ai", + "input_cost_per_token": 7.1e-7, + "output_cost_per_token": 7.1e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8", + "name": "Llama-4-Maverick-17B-128E-Instruct-FP8", + "provider": "azure_ai", + "input_cost_per_token": 0.00000141, + "output_cost_per_token": 3.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Llama-4-Scout-17B-16E-Instruct", + "name": "Llama-4-Scout-17B-16E-Instruct", + "provider": "azure_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 7.8e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Meta-Llama-3-70B-Instruct", + "name": "Meta-Llama-3-70B-Instruct", + "provider": "azure_ai", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 3.7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Meta-Llama-3.1-405B-Instruct", + "name": "Meta-Llama-3.1-405B-Instruct", + "provider": "azure_ai", + "input_cost_per_token": 0.00000533, + "output_cost_per_token": 0.000016, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Meta-Llama-3.1-70B-Instruct", + "name": "Meta-Llama-3.1-70B-Instruct", + "provider": "azure_ai", + "input_cost_per_token": 0.00000268, + "output_cost_per_token": 0.00000354, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Meta-Llama-3.1-8B-Instruct", + "name": "Meta-Llama-3.1-8B-Instruct", + "provider": "azure_ai", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 6.1e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-3-medium-128k-instruct", + "name": "Phi-3-medium-128k-instruct", + "provider": "azure_ai", + "input_cost_per_token": 1.7e-7, + "output_cost_per_token": 6.8e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-3-medium-4k-instruct", + "name": "Phi-3-medium-4k-instruct", + "provider": "azure_ai", + "input_cost_per_token": 1.7e-7, + "output_cost_per_token": 6.8e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-3-mini-128k-instruct", + "name": "Phi-3-mini-128k-instruct", + "provider": "azure_ai", + "input_cost_per_token": 1.3e-7, + "output_cost_per_token": 5.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-3-mini-4k-instruct", + "name": "Phi-3-mini-4k-instruct", + "provider": "azure_ai", + "input_cost_per_token": 1.3e-7, + "output_cost_per_token": 5.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-3-small-128k-instruct", + "name": "Phi-3-small-128k-instruct", + "provider": "azure_ai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-3-small-8k-instruct", + "name": "Phi-3-small-8k-instruct", + "provider": "azure_ai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-3.5-MoE-instruct", + "name": "Phi-3.5-MoE-instruct", + "provider": "azure_ai", + "input_cost_per_token": 1.6e-7, + "output_cost_per_token": 6.4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-3.5-mini-instruct", + "name": "Phi-3.5-mini-instruct", + "provider": "azure_ai", + "input_cost_per_token": 1.3e-7, + "output_cost_per_token": 5.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-3.5-vision-instruct", + "name": "Phi-3.5-vision-instruct", + "provider": "azure_ai", + "input_cost_per_token": 1.3e-7, + "output_cost_per_token": 5.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-4", + "name": "Phi-4", + "provider": "azure_ai", + "input_cost_per_token": 1.25e-7, + "output_cost_per_token": 5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-4-mini-instruct", + "name": "Phi-4-mini-instruct", + "provider": "azure_ai", + "input_cost_per_token": 7.5e-8, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-4-multimodal-instruct", + "name": "Phi-4-multimodal-instruct", + "provider": "azure_ai", + "input_cost_per_token": 8e-8, + "output_cost_per_token": 3.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-4-mini-reasoning", + "name": "Phi-4-mini-reasoning", + "provider": "azure_ai", + "input_cost_per_token": 8e-8, + "output_cost_per_token": 3.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/Phi-4-reasoning", + "name": "Phi-4-reasoning", + "provider": "azure_ai", + "input_cost_per_token": 1.25e-7, + "output_cost_per_token": 5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/MAI-DS-R1", + "name": "MAI-DS-R1", + "provider": "azure_ai", + "input_cost_per_token": 0.00000135, + "output_cost_per_token": 0.0000054, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/deepseek-v3.2", + "name": "deepseek-v3.2", + "provider": "azure_ai", + "input_cost_per_token": 5.8e-7, + "output_cost_per_token": 0.00000168, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/deepseek-v3.2-speciale", + "name": "deepseek-v3.2-speciale", + "provider": "azure_ai", + "input_cost_per_token": 5.8e-7, + "output_cost_per_token": 0.00000168, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/deepseek-r1", + "name": "deepseek-r1", + "provider": "azure_ai", + "input_cost_per_token": 0.00000135, + "output_cost_per_token": 0.0000054, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/deepseek-v3", + "name": "deepseek-v3", + "provider": "azure_ai", + "input_cost_per_token": 0.00000114, + "output_cost_per_token": 0.00000456, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/deepseek-v3-0324", + "name": "deepseek-v3-0324", + "provider": "azure_ai", + "input_cost_per_token": 0.00000114, + "output_cost_per_token": 0.00000456, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/deepseek-v3.1", + "name": "deepseek-v3.1", + "provider": "azure_ai", + "input_cost_per_token": 0.00000123, + "output_cost_per_token": 0.00000494, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/deepseek-v4-pro", + "name": "deepseek-v4-pro", + "provider": "azure_ai", + "input_cost_per_token": 0.00000174, + "output_cost_per_token": 0.00000348, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/deepseek-v4-flash", + "name": "deepseek-v4-flash", + "provider": "azure_ai", + "input_cost_per_token": 1.9e-7, + "output_cost_per_token": 5.1e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/global/grok-3", + "name": "global/grok-3", + "provider": "azure_ai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/global/grok-3-mini", + "name": "global/grok-3-mini", + "provider": "azure_ai", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000127, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/grok-3", + "name": "grok-3", + "provider": "azure_ai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/grok-3-mini", + "name": "grok-3-mini", + "provider": "azure_ai", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000127, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/grok-4", + "name": "grok-4", + "provider": "azure_ai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/grok-4-fast-non-reasoning", + "name": "grok-4-fast-non-reasoning", + "provider": "azure_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/grok-4-fast-reasoning", + "name": "grok-4-fast-reasoning", + "provider": "azure_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/grok-4-1-fast-non-reasoning", + "name": "grok-4-1-fast-non-reasoning", + "provider": "azure_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/grok-4-1-fast-reasoning", + "name": "grok-4-1-fast-reasoning", + "provider": "azure_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/grok-code-fast-1", + "name": "grok-code-fast-1", + "provider": "azure_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/jais-30b-chat", + "name": "jais-30b-chat", + "provider": "azure_ai", + "input_cost_per_token": 0.0032, + "output_cost_per_token": 0.00971, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/jamba-instruct", + "name": "jamba-instruct", + "provider": "azure_ai", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/kimi-k2.5", + "name": "kimi-k2.5", + "provider": "azure_ai", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/kimi-k2.6", + "name": "kimi-k2.6", + "provider": "azure_ai", + "input_cost_per_token": 9.5e-7, + "output_cost_per_token": 0.000004, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/ministral-3b", + "name": "ministral-3b", + "provider": "azure_ai", + "input_cost_per_token": 4e-8, + "output_cost_per_token": 4e-8, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/mistral-large", + "name": "mistral-large", + "provider": "azure_ai", + "input_cost_per_token": 0.000004, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/mistral-large-2407", + "name": "mistral-large-2407", + "provider": "azure_ai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/mistral-large-latest", + "name": "mistral-large-latest", + "provider": "azure_ai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/mistral-large-3", + "name": "mistral-large-3", + "provider": "azure_ai", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/mistral-medium-2505", + "name": "mistral-medium-2505", + "provider": "azure_ai", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/mistral-nemo", + "name": "mistral-nemo", + "provider": "azure_ai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/mistral-small", + "name": "mistral-small", + "provider": "azure_ai", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure_ai/mistral-small-2503", + "name": "mistral-small-2503", + "provider": "azure_ai", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-northeast-1/anthropic.claude-instant-v1", + "name": "ap-northeast-1/anthropic.claude-instant-v1", + "provider": "bedrock", + "input_cost_per_token": 0.00000223, + "output_cost_per_token": 0.00000755, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-northeast-1/anthropic.claude-v1", + "name": "ap-northeast-1/anthropic.claude-v1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-northeast-1/anthropic.claude-v2:1", + "name": "ap-northeast-1/anthropic.claude-v2:1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-northeast-1/deepseek.v3.2", + "name": "ap-northeast-1/deepseek.v3.2", + "provider": "bedrock", + "input_cost_per_token": 7.4e-7, + "output_cost_per_token": 0.00000222, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-northeast-1/minimax.minimax-m2.1", + "name": "ap-northeast-1/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-northeast-1/minimax.minimax-m2.5", + "name": "ap-northeast-1/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking", + "name": "ap-northeast-1/moonshotai.kimi-k2-thinking", + "provider": "bedrock", + "input_cost_per_token": 7.3e-7, + "output_cost_per_token": 0.00000303, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-northeast-1/moonshotai.kimi-k2.5", + "name": "ap-northeast-1/moonshotai.kimi-k2.5", + "provider": "bedrock", + "input_cost_per_token": 7.2e-7, + "output_cost_per_token": 0.0000036, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-northeast-1/qwen.qwen3-coder-next", + "name": "ap-northeast-1/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/moonshotai.kimi-k2-thinking", + "name": "moonshotai.kimi-k2-thinking", + "provider": "bedrock", + "input_cost_per_token": 7.3e-7, + "output_cost_per_token": 0.00000303, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/moonshotai.kimi-k2.5", + "name": "moonshotai.kimi-k2.5", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.00000303, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0", + "name": "ap-south-1/meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000318, + "output_cost_per_token": 0.0000042, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0", + "name": "ap-south-1/meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 7.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-south-1/deepseek.v3.2", + "name": "ap-south-1/deepseek.v3.2", + "provider": "bedrock", + "input_cost_per_token": 7.4e-7, + "output_cost_per_token": 0.00000222, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-south-1/minimax.minimax-m2.1", + "name": "ap-south-1/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-south-1/minimax.minimax-m2.5", + "name": "ap-south-1/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-south-1/moonshotai.kimi-k2-thinking", + "name": "ap-south-1/moonshotai.kimi-k2-thinking", + "provider": "bedrock", + "input_cost_per_token": 7.1e-7, + "output_cost_per_token": 0.00000294, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-south-1/moonshotai.kimi-k2.5", + "name": "ap-south-1/moonshotai.kimi-k2.5", + "provider": "bedrock", + "input_cost_per_token": 7.2e-7, + "output_cost_per_token": 0.0000036, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-south-1/qwen.qwen3-coder-next", + "name": "ap-south-1/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-southeast-2/minimax.minimax-m2.5", + "name": "ap-southeast-2/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3.09e-7, + "output_cost_per_token": 0.000001236, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-southeast-3/deepseek.v3.2", + "name": "ap-southeast-3/deepseek.v3.2", + "provider": "bedrock", + "input_cost_per_token": 7.4e-7, + "output_cost_per_token": 0.00000222, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-southeast-3/minimax.minimax-m2.1", + "name": "ap-southeast-3/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-southeast-3/minimax.minimax-m2.5", + "name": "ap-southeast-3/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-southeast-3/moonshotai.kimi-k2.5", + "name": "ap-southeast-3/moonshotai.kimi-k2.5", + "provider": "bedrock", + "input_cost_per_token": 7.2e-7, + "output_cost_per_token": 0.0000036, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ap-southeast-3/qwen.qwen3-coder-next", + "name": "ap-southeast-3/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0", + "name": "ca-central-1/meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000305, + "output_cost_per_token": 0.00000403, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0", + "name": "ca-central-1/meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3.5e-7, + "output_cost_per_token": 6.9e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-north-1/deepseek.v3.2", + "name": "eu-north-1/deepseek.v3.2", + "provider": "bedrock", + "input_cost_per_token": 7.4e-7, + "output_cost_per_token": 0.00000222, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-north-1/minimax.minimax-m2.1", + "name": "eu-north-1/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-north-1/minimax.minimax-m2.5", + "name": "eu-north-1/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-north-1/moonshotai.kimi-k2.5", + "name": "eu-north-1/moonshotai.kimi-k2.5", + "provider": "bedrock", + "input_cost_per_token": 7.2e-7, + "output_cost_per_token": 0.0000036, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-central-1/anthropic.claude-instant-v1", + "name": "eu-central-1/anthropic.claude-instant-v1", + "provider": "bedrock", + "input_cost_per_token": 0.00000248, + "output_cost_per_token": 0.00000838, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-central-1/anthropic.claude-v1", + "name": "eu-central-1/anthropic.claude-v1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-central-1/anthropic.claude-v2:1", + "name": "eu-central-1/anthropic.claude-v2:1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-central-1/minimax.minimax-m2.1", + "name": "eu-central-1/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-central-1/minimax.minimax-m2.5", + "name": "eu-central-1/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-central-1/qwen.qwen3-coder-next", + "name": "eu-central-1/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0", + "name": "eu-west-1/meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000286, + "output_cost_per_token": 0.00000378, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0", + "name": "eu-west-1/meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3.2e-7, + "output_cost_per_token": 6.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-1/minimax.minimax-m2.1", + "name": "eu-west-1/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-1/minimax.minimax-m2.5", + "name": "eu-west-1/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-1/qwen.qwen3-coder-next", + "name": "eu-west-1/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0", + "name": "eu-west-2/meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000345, + "output_cost_per_token": 0.00000455, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0", + "name": "eu-west-2/meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3.9e-7, + "output_cost_per_token": 7.8e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-2/minimax.minimax-m2.1", + "name": "eu-west-2/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 4.7e-7, + "output_cost_per_token": 0.00000186, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-2/minimax.minimax-m2.5", + "name": "eu-west-2/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 4.7e-7, + "output_cost_per_token": 0.00000186, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-2/qwen.qwen3-coder-next", + "name": "eu-west-2/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 7.8e-7, + "output_cost_per_token": 0.00000186, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2", + "name": "eu-west-3/mistral.mistral-7b-instruct-v0:2", + "provider": "bedrock", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 2.6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-3/mistral.mistral-large-2402-v1:0", + "name": "eu-west-3/mistral.mistral-large-2402-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000104, + "output_cost_per_token": 0.0000312, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1", + "name": "eu-west-3/mistral.mixtral-8x7b-instruct-v0:1", + "provider": "bedrock", + "input_cost_per_token": 5.9e-7, + "output_cost_per_token": 9.1e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-south-1/minimax.minimax-m2.1", + "name": "eu-south-1/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-south-1/minimax.minimax-m2.5", + "name": "eu-south-1/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/eu-south-1/qwen.qwen3-coder-next", + "name": "eu-south-1/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + "name": "invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/sa-east-1/meta.llama3-70b-instruct-v1:0", + "name": "sa-east-1/meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000445, + "output_cost_per_token": 0.00000588, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0", + "name": "sa-east-1/meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.00000101, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/sa-east-1/deepseek.v3.2", + "name": "sa-east-1/deepseek.v3.2", + "provider": "bedrock", + "input_cost_per_token": 7.4e-7, + "output_cost_per_token": 0.00000222, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/sa-east-1/minimax.minimax-m2.1", + "name": "sa-east-1/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/sa-east-1/minimax.minimax-m2.5", + "name": "sa-east-1/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3.6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/sa-east-1/moonshotai.kimi-k2-thinking", + "name": "sa-east-1/moonshotai.kimi-k2-thinking", + "provider": "bedrock", + "input_cost_per_token": 7.3e-7, + "output_cost_per_token": 0.00000303, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/sa-east-1/moonshotai.kimi-k2.5", + "name": "sa-east-1/moonshotai.kimi-k2.5", + "provider": "bedrock", + "input_cost_per_token": 7.2e-7, + "output_cost_per_token": 0.0000036, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/sa-east-1/qwen.qwen3-coder-next", + "name": "sa-east-1/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.00000144, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/anthropic.claude-instant-v1", + "name": "us-east-1/anthropic.claude-instant-v1", + "provider": "bedrock", + "input_cost_per_token": 8e-7, + "output_cost_per_token": 0.0000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/anthropic.claude-v1", + "name": "us-east-1/anthropic.claude-v1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/anthropic.claude-v2:1", + "name": "us-east-1/anthropic.claude-v2:1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/meta.llama3-70b-instruct-v1:0", + "name": "us-east-1/meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000265, + "output_cost_per_token": 0.0000035, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/meta.llama3-8b-instruct-v1:0", + "name": "us-east-1/meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/mistral.mistral-7b-instruct-v0:2", + "name": "us-east-1/mistral.mistral-7b-instruct-v0:2", + "provider": "bedrock", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/mistral.mistral-large-2402-v1:0", + "name": "us-east-1/mistral.mistral-large-2402-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1", + "name": "us-east-1/mistral.mixtral-8x7b-instruct-v0:1", + "provider": "bedrock", + "input_cost_per_token": 4.5e-7, + "output_cost_per_token": 7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/deepseek.v3.2", + "name": "us-east-1/deepseek.v3.2", + "provider": "bedrock", + "input_cost_per_token": 6.2e-7, + "output_cost_per_token": 0.00000185, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/minimax.minimax-m2.1", + "name": "us-east-1/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/minimax.minimax-m2.5", + "name": "us-east-1/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/moonshotai.kimi-k2-thinking", + "name": "us-east-1/moonshotai.kimi-k2-thinking", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/moonshotai.kimi-k2.5", + "name": "us-east-1/moonshotai.kimi-k2.5", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/qwen.qwen3-coder-next", + "name": "us-east-1/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-2/deepseek.v3.2", + "name": "us-east-2/deepseek.v3.2", + "provider": "bedrock", + "input_cost_per_token": 6.2e-7, + "output_cost_per_token": 0.00000185, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-2/minimax.minimax-m2.1", + "name": "us-east-2/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-2/minimax.minimax-m2.5", + "name": "us-east-2/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-2/moonshotai.kimi-k2-thinking", + "name": "us-east-2/moonshotai.kimi-k2-thinking", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-2/moonshotai.kimi-k2.5", + "name": "us-east-2/moonshotai.kimi-k2.5", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-2/qwen.qwen3-coder-next", + "name": "us-east-2/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/amazon.nova-pro-v1:0", + "name": "us-gov-east-1/amazon.nova-pro-v1:0", + "provider": "bedrock", + "input_cost_per_token": 9.6e-7, + "output_cost_per_token": 0.00000384, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/amazon.titan-text-express-v1", + "name": "us-gov-east-1/amazon.titan-text-express-v1", + "provider": "bedrock", + "input_cost_per_token": 0.0000013, + "output_cost_per_token": 0.0000017, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/amazon.titan-text-lite-v1", + "name": "us-gov-east-1/amazon.titan-text-lite-v1", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/amazon.titan-text-premier-v1:0", + "name": "us-gov-east-1/amazon.titan-text-premier-v1:0", + "provider": "bedrock", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0", + "name": "us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000036, + "output_cost_per_token": 0.000018, + "cache_read_input_token_cost": 3.6e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0", + "name": "us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000036, + "output_cost_per_token": 0.000018, + "cache_read_input_token_cost": 3.6e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0", + "name": "us-gov-east-1/claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000036, + "output_cost_per_token": 0.000018, + "cache_read_input_token_cost": 3.6e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0", + "name": "us-gov-east-1/meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000265, + "output_cost_per_token": 0.0000035, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0", + "name": "us-gov-east-1/meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.00000265, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/amazon.nova-pro-v1:0", + "name": "us-gov-west-1/amazon.nova-pro-v1:0", + "provider": "bedrock", + "input_cost_per_token": 9.6e-7, + "output_cost_per_token": 0.00000384, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/amazon.titan-text-express-v1", + "name": "us-gov-west-1/amazon.titan-text-express-v1", + "provider": "bedrock", + "input_cost_per_token": 0.0000013, + "output_cost_per_token": 0.0000017, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/amazon.titan-text-lite-v1", + "name": "us-gov-west-1/amazon.titan-text-lite-v1", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/amazon.titan-text-premier-v1:0", + "name": "us-gov-west-1/amazon.titan-text-premier-v1:0", + "provider": "bedrock", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0", + "name": "us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000036, + "output_cost_per_token": 0.000018, + "cache_read_input_token_cost": 3.6e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0", + "name": "us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000036, + "output_cost_per_token": 0.000018, + "cache_read_input_token_cost": 3.6e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0", + "name": "us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000036, + "output_cost_per_token": 0.000018, + "cache_read_input_token_cost": 3.6e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0", + "name": "us-gov-west-1/claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000036, + "output_cost_per_token": 0.000018, + "cache_read_input_token_cost": 3.6e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0", + "name": "us-gov-west-1/meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000265, + "output_cost_per_token": 0.0000035, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0", + "name": "us-gov-west-1/meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.00000265, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0", + "name": "us-west-1/meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000265, + "output_cost_per_token": 0.0000035, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-1/meta.llama3-8b-instruct-v1:0", + "name": "us-west-1/meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/anthropic.claude-instant-v1", + "name": "us-west-2/anthropic.claude-instant-v1", + "provider": "bedrock", + "input_cost_per_token": 8e-7, + "output_cost_per_token": 0.0000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/anthropic.claude-v1", + "name": "us-west-2/anthropic.claude-v1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/anthropic.claude-v2:1", + "name": "us-west-2/anthropic.claude-v2:1", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/mistral.mistral-7b-instruct-v0:2", + "name": "us-west-2/mistral.mistral-7b-instruct-v0:2", + "provider": "bedrock", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/mistral.mistral-large-2402-v1:0", + "name": "us-west-2/mistral.mistral-large-2402-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1", + "name": "us-west-2/mistral.mixtral-8x7b-instruct-v0:1", + "provider": "bedrock", + "input_cost_per_token": 4.5e-7, + "output_cost_per_token": 7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/deepseek.v3.2", + "name": "us-west-2/deepseek.v3.2", + "provider": "bedrock", + "input_cost_per_token": 6.2e-7, + "output_cost_per_token": 0.00000185, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/minimax.minimax-m2.1", + "name": "us-west-2/minimax.minimax-m2.1", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/minimax.minimax-m2.5", + "name": "us-west-2/minimax.minimax-m2.5", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/moonshotai.kimi-k2-thinking", + "name": "us-west-2/moonshotai.kimi-k2-thinking", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/moonshotai.kimi-k2.5", + "name": "us-west-2/moonshotai.kimi-k2.5", + "provider": "bedrock", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/qwen.qwen3-coder-next", + "name": "us-west-2/qwen.qwen3-coder-next", + "provider": "bedrock", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0", + "name": "us.anthropic.claude-3-5-haiku-20241022-v1:0", + "provider": "bedrock", + "input_cost_per_token": 8e-7, + "output_cost_per_token": 0.000004, + "cache_read_input_token_cost": 8e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "chatgpt-4o-latest", + "name": "chatgpt-4o-latest", + "provider": "openai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-haiku-4-5-20251001", + "name": "claude-haiku-4-5-20251001", + "provider": "anthropic", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-haiku-4-5", + "name": "claude-haiku-4-5", + "provider": "anthropic", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-3-7-sonnet-20250219", + "name": "claude-3-7-sonnet-20250219", + "provider": "anthropic", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-3-haiku-20240307", + "name": "claude-3-haiku-20240307", + "provider": "anthropic", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-3-opus-20240229", + "name": "claude-3-opus-20240229", + "provider": "anthropic", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-4-opus-20250514", + "name": "claude-4-opus-20250514", + "provider": "anthropic", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-4-sonnet-20250514", + "name": "claude-4-sonnet-20250514", + "provider": "anthropic", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-sonnet-4-5", + "name": "claude-sonnet-4-5", + "provider": "anthropic", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-sonnet-4-5-20250929", + "name": "claude-sonnet-4-5-20250929", + "provider": "anthropic", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-sonnet-5", + "name": "claude-sonnet-5", + "provider": "anthropic", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-sonnet-4-6", + "name": "claude-sonnet-4-6", + "provider": "anthropic", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-sonnet-4-5-20250929-v1:0", + "name": "claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-1", + "name": "claude-opus-4-1", + "provider": "anthropic", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-1-20250805", + "name": "claude-opus-4-1-20250805", + "provider": "anthropic", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-20250514", + "name": "claude-opus-4-20250514", + "provider": "anthropic", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-5-20251101", + "name": "claude-opus-4-5-20251101", + "provider": "anthropic", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-5", + "name": "claude-opus-4-5", + "provider": "anthropic", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-6", + "name": "claude-opus-4-6", + "provider": "anthropic", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-6-20260205", + "name": "claude-opus-4-6-20260205", + "provider": "anthropic", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-7", + "name": "claude-opus-4-7", + "provider": "anthropic", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-7-20260416", + "name": "claude-opus-4-7-20260416", + "provider": "anthropic", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-fable-5", + "name": "claude-fable-5", + "provider": "anthropic", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00005, + "cache_read_input_token_cost": 0.000001, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-5", + "name": "claude-opus-5", + "provider": "anthropic", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-opus-4-8", + "name": "claude-opus-4-8", + "provider": "anthropic", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "claude-sonnet-4-20250514", + "name": "claude-sonnet-4-20250514", + "provider": "anthropic", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "cohere.command-light-text-v14", + "name": "cohere.command-light-text-v14", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "cohere.command-r-plus-v1:0", + "name": "cohere.command-r-plus-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "cohere.command-r-v1:0", + "name": "cohere.command-r-v1:0", + "provider": "bedrock", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "cohere.command-text-v14", + "name": "cohere.command-text-v14", + "provider": "bedrock", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "computer-use-preview", + "name": "computer-use-preview", + "provider": "azure", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "deepseek.v3-v1:0", + "name": "deepseek.v3-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 5.8e-7, + "output_cost_per_token": 0.00000168, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "deepseek.v3.2", + "name": "deepseek.v3.2", + "provider": "bedrock_converse", + "input_cost_per_token": 6.2e-7, + "output_cost_per_token": 0.00000185, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.amazon.nova-lite-v1:0", + "name": "eu.amazon.nova-lite-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 7.8e-8, + "output_cost_per_token": 3.12e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.amazon.nova-micro-v1:0", + "name": "eu.amazon.nova-micro-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 4.6e-8, + "output_cost_per_token": 1.84e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.amazon.nova-pro-v1:0", + "name": "eu.amazon.nova-pro-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.00000105, + "output_cost_per_token": 0.0000042, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-3-5-haiku-20241022-v1:0", + "name": "eu.anthropic.claude-3-5-haiku-20241022-v1:0", + "provider": "bedrock", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-haiku-4-5-20251001-v1:0", + "name": "eu.anthropic.claude-haiku-4-5-20251001-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000055, + "cache_read_input_token_cost": 1.1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-3-5-sonnet-20240620-v1:0", + "name": "eu.anthropic.claude-3-5-sonnet-20240620-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-3-5-sonnet-20241022-v2:0", + "name": "eu.anthropic.claude-3-5-sonnet-20241022-v2:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-3-7-sonnet-20250219-v1:0", + "name": "eu.anthropic.claude-3-7-sonnet-20250219-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-3-haiku-20240307-v1:0", + "name": "eu.anthropic.claude-3-haiku-20240307-v1:0", + "provider": "bedrock", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-3-opus-20240229-v1:0", + "name": "eu.anthropic.claude-3-opus-20240229-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-3-sonnet-20240229-v1:0", + "name": "eu.anthropic.claude-3-sonnet-20240229-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-opus-4-1-20250805-v1:0", + "name": "eu.anthropic.claude-opus-4-1-20250805-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-opus-4-20250514-v1:0", + "name": "eu.anthropic.claude-opus-4-20250514-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-sonnet-4-20250514-v1:0", + "name": "eu.anthropic.claude-sonnet-4-20250514-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000033, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 3.3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.meta.llama3-2-1b-instruct-v1:0", + "name": "eu.meta.llama3-2-1b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 1.3e-7, + "output_cost_per_token": 1.3e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.meta.llama3-2-3b-instruct-v1:0", + "name": "eu.meta.llama3-2-3b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 1.9e-7, + "output_cost_per_token": 1.9e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.mistral.pixtral-large-2502-v1:0", + "name": "eu.mistral.pixtral-large-2502-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-3.5-turbo", + "name": "ft:gpt-3.5-turbo", + "provider": "openai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-3.5-turbo-0125", + "name": "ft:gpt-3.5-turbo-0125", + "provider": "openai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-3.5-turbo-0613", + "name": "ft:gpt-3.5-turbo-0613", + "provider": "openai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-3.5-turbo-1106", + "name": "ft:gpt-3.5-turbo-1106", + "provider": "openai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-4-0613", + "name": "ft:gpt-4-0613", + "provider": "openai", + "input_cost_per_token": 0.00003, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-4o-2024-08-06", + "name": "ft:gpt-4o-2024-08-06", + "provider": "openai", + "input_cost_per_token": 0.00000375, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 0.000001875, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-4o-2024-11-20", + "name": "ft:gpt-4o-2024-11-20", + "provider": "openai", + "input_cost_per_token": 0.00000375, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-4o-mini-2024-07-18", + "name": "ft:gpt-4o-mini-2024-07-18", + "provider": "openai", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": 1.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-4.1-2025-04-14", + "name": "ft:gpt-4.1-2025-04-14", + "provider": "openai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 7.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-4.1-mini-2025-04-14", + "name": "ft:gpt-4.1-mini-2025-04-14", + "provider": "openai", + "input_cost_per_token": 8e-7, + "output_cost_per_token": 0.0000032, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:gpt-4.1-nano-2025-04-14", + "name": "ft:gpt-4.1-nano-2025-04-14", + "provider": "openai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 8e-7, + "cache_read_input_token_cost": 5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "ft:o4-mini-2025-04-16", + "name": "ft:o4-mini-2025-04-16", + "provider": "openai", + "input_cost_per_token": 0.000004, + "output_cost_per_token": 0.000016, + "cache_read_input_token_cost": 0.000001, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-2.0-flash", + "name": "gemini-2.0-flash", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-2.0-flash-001", + "name": "gemini-2.0-flash-001", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": 3.75e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-2.0-flash-lite", + "name": "gemini-2.0-flash-lite", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 7.5e-8, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": 1.875e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-2.0-flash-lite-001", + "name": "gemini-2.0-flash-lite-001", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 7.5e-8, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": 1.875e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-2.5-flash", + "name": "gemini-2.5-flash", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini-3.1-flash-lite-preview", + "name": "gemini-3.1-flash-lite-preview", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": 0.0000015 + }, + { + "id": "gemini-3.1-flash-lite", + "name": "gemini-3.1-flash-lite", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": 0.0000015 + }, + { + "id": "gemini-3.5-flash-lite", + "name": "gemini-3.5-flash-lite", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini-2.5-flash-lite", + "name": "gemini-2.5-flash-lite", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 1e-8, + "output_cost_per_reasoning_token": 4e-7 + }, + { + "id": "gemini-2.5-flash-lite-preview-09-2025", + "name": "gemini-2.5-flash-lite-preview-09-2025", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 1e-8, + "output_cost_per_reasoning_token": 4e-7 + }, + { + "id": "gemini-2.5-flash-preview-09-2025", + "name": "gemini-2.5-flash-preview-09-2025", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini-2.5-flash-lite-preview-06-17", + "name": "gemini-2.5-flash-lite-preview-06-17", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": 4e-7 + }, + { + "id": "gemini-2.5-pro", + "name": "gemini-2.5-pro", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-3-pro-preview", + "name": "gemini-3-pro-preview", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-3.1-pro-preview", + "name": "gemini-3.1-pro-preview", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-3.1-pro-preview-customtools", + "name": "gemini-3.1-pro-preview-customtools", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/gemini-3-pro-preview", + "name": "gemini-3-pro-preview", + "provider": "vertex_ai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/gemini-3-flash-preview", + "name": "gemini-3-flash-preview", + "provider": "vertex_ai", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": 5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/gemini-3.5-flash", + "name": "gemini-3.5-flash", + "provider": "vertex_ai", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000009, + "cache_read_input_token_cost": 1.5e-7, + "output_cost_per_reasoning_token": 0.000009 + }, + { + "id": "vertex_ai/gemini-3.6-flash", + "name": "gemini-3.6-flash", + "provider": "vertex_ai", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.0000075, + "cache_read_input_token_cost": 1.5e-7, + "output_cost_per_reasoning_token": 0.0000075 + }, + { + "id": "vertex_ai/gemini-3.1-pro-preview", + "name": "gemini-3.1-pro-preview", + "provider": "vertex_ai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/gemini-3.1-pro-preview-customtools", + "name": "gemini-3.1-pro-preview-customtools", + "provider": "vertex_ai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-2.5-pro-preview-tts", + "name": "gemini-2.5-pro-preview-tts", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-robotics-er-1.5-preview", + "name": "gemini-robotics-er-1.5-preview", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 0, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini/gemini-robotics-er-1.5-preview", + "name": "gemini-robotics-er-1.5-preview", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 0, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini/gemini-robotics-er-2-preview", + "name": "gemini-robotics-er-2-preview", + "provider": "gemini", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": 0.00001 + }, + { + "id": "gemini/gemini-robotics-er-1.6-preview", + "name": "gemini-robotics-er-1.6-preview", + "provider": "gemini", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": 0.000005 + }, + { + "id": "gemini-2.5-computer-use-preview-10-2025", + "name": "gemini-2.5-computer-use-preview-10-2025", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-2.0-flash", + "name": "gemini-2.0-flash", + "provider": "gemini", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-2.0-flash-001", + "name": "gemini-2.0-flash-001", + "provider": "gemini", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-2.0-flash-lite", + "name": "gemini-2.0-flash-lite", + "provider": "gemini", + "input_cost_per_token": 7.5e-8, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": 1.875e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-2.5-flash", + "name": "gemini-2.5-flash", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini/gemini-2.5-flash-lite", + "name": "gemini-2.5-flash-lite", + "provider": "gemini", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 1e-8, + "output_cost_per_reasoning_token": 4e-7 + }, + { + "id": "gemini/gemini-2.5-flash-lite-preview-09-2025", + "name": "gemini-2.5-flash-lite-preview-09-2025", + "provider": "gemini", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 1e-8, + "output_cost_per_reasoning_token": 4e-7 + }, + { + "id": "gemini/gemini-2.5-flash-preview-09-2025", + "name": "gemini-2.5-flash-preview-09-2025", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini/gemini-flash-latest", + "name": "gemini-flash-latest", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini/gemini-flash-lite-latest", + "name": "gemini-flash-lite-latest", + "provider": "gemini", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": 4e-7 + }, + { + "id": "gemini/gemini-2.5-flash-lite-preview-06-17", + "name": "gemini-2.5-flash-lite-preview-06-17", + "provider": "gemini", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": 4e-7 + }, + { + "id": "gemini/gemini-2.5-pro", + "name": "gemini-2.5-pro", + "provider": "gemini", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-2.5-computer-use-preview-10-2025", + "name": "gemini-2.5-computer-use-preview-10-2025", + "provider": "gemini", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-3-pro-preview", + "name": "gemini-3-pro-preview", + "provider": "gemini", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-3.1-flash-lite-preview", + "name": "gemini-3.1-flash-lite-preview", + "provider": "gemini", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": 0.0000015 + }, + { + "id": "gemini/gemini-3.1-flash-lite", + "name": "gemini-3.1-flash-lite", + "provider": "gemini", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": 0.0000015 + }, + { + "id": "gemini/gemini-3.5-flash-lite", + "name": "gemini-3.5-flash-lite", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini/gemini-3-flash-preview", + "name": "gemini-3-flash-preview", + "provider": "gemini", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": 5e-8, + "output_cost_per_reasoning_token": 0.000003 + }, + { + "id": "gemini/gemini-3.5-flash", + "name": "gemini-3.5-flash", + "provider": "gemini", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000009, + "cache_read_input_token_cost": 1.5e-7, + "output_cost_per_reasoning_token": 0.000009 + }, + { + "id": "gemini/gemini-3.6-flash", + "name": "gemini-3.6-flash", + "provider": "gemini", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.0000075, + "cache_read_input_token_cost": 1.5e-7, + "output_cost_per_reasoning_token": 0.0000075 + }, + { + "id": "gemini/gemini-omni-flash-preview", + "name": "gemini-omni-flash-preview", + "provider": "gemini", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000009, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": 0.000009 + }, + { + "id": "gemini/gemini-3.1-pro-preview", + "name": "gemini-3.1-pro-preview", + "provider": "gemini", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-3.1-pro-preview-customtools", + "name": "gemini-3.1-pro-preview-customtools", + "provider": "gemini", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-3-flash-preview", + "name": "gemini-3-flash-preview", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": 5e-8, + "output_cost_per_reasoning_token": 0.000003 + }, + { + "id": "gemini-omni-flash-preview", + "name": "gemini-omni-flash-preview", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000009, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": 0.000009 + }, + { + "id": "gemini-3.5-flash", + "name": "gemini-3.5-flash", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000009, + "cache_read_input_token_cost": 1.5e-7, + "output_cost_per_reasoning_token": 0.000009 + }, + { + "id": "gemini-3.6-flash", + "name": "gemini-3.6-flash", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.0000075, + "cache_read_input_token_cost": 1.5e-7, + "output_cost_per_reasoning_token": 0.0000075 + }, + { + "id": "gemini/gemini-2.5-pro-preview-tts", + "name": "gemini-2.5-pro-preview-tts", + "provider": "gemini", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-exp-1114", + "name": "gemini-exp-1114", + "provider": "gemini", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-exp-1206", + "name": "gemini-exp-1206", + "provider": "gemini", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-gemma-2-27b-it", + "name": "gemini-gemma-2-27b-it", + "provider": "gemini", + "input_cost_per_token": 3.5e-7, + "output_cost_per_token": 0.00000105, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-gemma-2-9b-it", + "name": "gemini-gemma-2-9b-it", + "provider": "gemini", + "input_cost_per_token": 3.5e-7, + "output_cost_per_token": 0.00000105, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemma-3-27b-it", + "name": "gemma-3-27b-it", + "provider": "gemini", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/learnlm-1.5-pro-experimental", + "name": "learnlm-1.5-pro-experimental", + "provider": "gemini", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/lyria-3-clip-preview", + "name": "lyria-3-clip-preview", + "provider": "gemini", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/lyria-3-pro-preview", + "name": "lyria-3-pro-preview", + "provider": "gemini", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "google.gemma-3-12b-it", + "name": "google.gemma-3-12b-it", + "provider": "bedrock_converse", + "input_cost_per_token": 9e-8, + "output_cost_per_token": 2.9e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "google.gemma-3-27b-it", + "name": "google.gemma-3-27b-it", + "provider": "bedrock_converse", + "input_cost_per_token": 2.3e-7, + "output_cost_per_token": 3.8e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "google.gemma-3-4b-it", + "name": "google.gemma-3-4b-it", + "provider": "bedrock_converse", + "input_cost_per_token": 4e-8, + "output_cost_per_token": 8e-8, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-sonnet-4-20250514-v1:0", + "name": "global.anthropic.claude-sonnet-4-20250514-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-haiku-4-5-20251001-v1:0", + "name": "global.anthropic.claude-haiku-4-5-20251001-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.amazon.nova-2-lite-v1:0", + "name": "global.amazon.nova-2-lite-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-3.5-turbo", + "name": "gpt-3.5-turbo", + "provider": "openai", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-3.5-turbo-0125", + "name": "gpt-3.5-turbo-0125", + "provider": "openai", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-3.5-turbo-1106", + "name": "gpt-3.5-turbo-1106", + "provider": "openai", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k", + "provider": "openai", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000004, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4", + "name": "gpt-4", + "provider": "openai", + "input_cost_per_token": 0.00003, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4-0125-preview", + "name": "gpt-4-0125-preview", + "provider": "openai", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4-0314", + "name": "gpt-4-0314", + "provider": "openai", + "input_cost_per_token": 0.00003, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4-0613", + "name": "gpt-4-0613", + "provider": "openai", + "input_cost_per_token": 0.00003, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4-1106-preview", + "name": "gpt-4-1106-preview", + "provider": "openai", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4-turbo", + "name": "gpt-4-turbo", + "provider": "openai", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4-turbo-2024-04-09", + "name": "gpt-4-turbo-2024-04-09", + "provider": "openai", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4-turbo-preview", + "name": "gpt-4-turbo-preview", + "provider": "openai", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4.1", + "name": "gpt-4.1", + "provider": "openai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4.1-2025-04-14", + "name": "gpt-4.1-2025-04-14", + "provider": "openai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4.1-mini", + "name": "gpt-4.1-mini", + "provider": "openai", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.0000016, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4.1-mini-2025-04-14", + "name": "gpt-4.1-mini-2025-04-14", + "provider": "openai", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.0000016, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4.1-nano", + "name": "gpt-4.1-nano", + "provider": "openai", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4.1-nano-2025-04-14", + "name": "gpt-4.1-nano-2025-04-14", + "provider": "openai", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o", + "name": "gpt-4o", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-2024-05-13", + "name": "gpt-4o-2024-05-13", + "provider": "openai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-2024-08-06", + "name": "gpt-4o-2024-08-06", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-2024-11-20", + "name": "gpt-4o-2024-11-20", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-audio-preview", + "name": "gpt-4o-audio-preview", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-audio-preview-2024-12-17", + "name": "gpt-4o-audio-preview-2024-12-17", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-audio-preview-2025-06-03", + "name": "gpt-4o-audio-preview-2025-06-03", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-audio", + "name": "gpt-audio", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-audio-1.5", + "name": "gpt-audio-1.5", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-audio-2025-08-28", + "name": "gpt-audio-2025-08-28", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-audio-mini", + "name": "gpt-audio-mini", + "provider": "openai", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-audio-mini-2025-10-06", + "name": "gpt-audio-mini-2025-10-06", + "provider": "openai", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-audio-mini-2025-12-15", + "name": "gpt-audio-mini-2025-12-15", + "provider": "openai", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-mini", + "name": "gpt-4o-mini", + "provider": "openai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-mini-2024-07-18", + "name": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-mini-audio-preview", + "name": "gpt-4o-mini-audio-preview", + "provider": "openai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-mini-audio-preview-2024-12-17", + "name": "gpt-4o-mini-audio-preview-2024-12-17", + "provider": "openai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-mini-search-preview", + "name": "gpt-4o-mini-search-preview", + "provider": "openai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-mini-search-preview-2025-03-11", + "name": "gpt-4o-mini-search-preview-2025-03-11", + "provider": "openai", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-search-preview", + "name": "gpt-4o-search-preview", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-4o-search-preview-2025-03-11", + "name": "gpt-4o-search-preview-2025-03-11", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00000125, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5", + "name": "gpt-5", + "provider": "openai", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.1", + "name": "gpt-5.1", + "provider": "openai", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.1-2025-11-13", + "name": "gpt-5.1-2025-11-13", + "provider": "openai", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.1-chat-latest", + "name": "gpt-5.1-chat-latest", + "provider": "openai", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.2", + "name": "gpt-5.2", + "provider": "openai", + "input_cost_per_token": 0.00000175, + "output_cost_per_token": 0.000014, + "cache_read_input_token_cost": 1.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.2-2025-12-11", + "name": "gpt-5.2-2025-12-11", + "provider": "openai", + "input_cost_per_token": 0.00000175, + "output_cost_per_token": 0.000014, + "cache_read_input_token_cost": 1.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.2-chat-latest", + "name": "gpt-5.2-chat-latest", + "provider": "openai", + "input_cost_per_token": 0.00000175, + "output_cost_per_token": 0.000014, + "cache_read_input_token_cost": 1.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.3-chat-latest", + "name": "gpt-5.3-chat-latest", + "provider": "openai", + "input_cost_per_token": 0.00000175, + "output_cost_per_token": 0.000014, + "cache_read_input_token_cost": 1.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.6", + "name": "gpt-5.6", + "provider": "openai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.6-sol", + "name": "gpt-5.6-sol", + "provider": "openai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.6-terra", + "name": "gpt-5.6-terra", + "provider": "openai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000012, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.6-luna", + "name": "gpt-5.6-luna", + "provider": "openai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": 2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.5", + "name": "gpt-5.5", + "provider": "openai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.5-2026-04-23", + "name": "gpt-5.5-2026-04-23", + "provider": "openai", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.00003, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.4", + "name": "gpt-5.4", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 2.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.4-2026-03-05", + "name": "gpt-5.4-2026-03-05", + "provider": "openai", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 2.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.4-mini", + "name": "gpt-5.4-mini", + "provider": "openai", + "input_cost_per_token": 7.5e-7, + "output_cost_per_token": 0.0000045, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.4-mini-2026-03-17", + "name": "gpt-5.4-mini-2026-03-17", + "provider": "openai", + "input_cost_per_token": 7.5e-7, + "output_cost_per_token": 0.0000045, + "cache_read_input_token_cost": 7.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.4-nano", + "name": "gpt-5.4-nano", + "provider": "openai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5.4-nano-2026-03-17", + "name": "gpt-5.4-nano-2026-03-17", + "provider": "openai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5-2025-08-07", + "name": "gpt-5-2025-08-07", + "provider": "openai", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5-chat", + "name": "gpt-5-chat", + "provider": "openai", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5-chat-latest", + "name": "gpt-5-chat-latest", + "provider": "openai", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5-mini", + "name": "gpt-5-mini", + "provider": "openai", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5-mini-2025-08-07", + "name": "gpt-5-mini-2025-08-07", + "provider": "openai", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5-nano", + "name": "gpt-5-nano", + "provider": "openai", + "input_cost_per_token": 5e-8, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 5e-9, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5-nano-2025-08-07", + "name": "gpt-5-nano-2025-08-07", + "provider": "openai", + "input_cost_per_token": 5e-8, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 5e-9, + "output_cost_per_reasoning_token": null + }, + { + "id": "jp.anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "jp.anthropic.claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000033, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 3.3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "jp.anthropic.claude-haiku-4-5-20251001-v1:0", + "name": "jp.anthropic.claude-haiku-4-5-20251001-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000055, + "cache_read_input_token_cost": 1.1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama2-13b-chat-v1", + "name": "meta.llama2-13b-chat-v1", + "provider": "bedrock", + "input_cost_per_token": 7.5e-7, + "output_cost_per_token": 0.000001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama2-70b-chat-v1", + "name": "meta.llama2-70b-chat-v1", + "provider": "bedrock", + "input_cost_per_token": 0.00000195, + "output_cost_per_token": 0.00000256, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-1-405b-instruct-v1:0", + "name": "meta.llama3-1-405b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000532, + "output_cost_per_token": 0.000016, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-1-70b-instruct-v1:0", + "name": "meta.llama3-1-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 9.9e-7, + "output_cost_per_token": 9.9e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-1-8b-instruct-v1:0", + "name": "meta.llama3-1-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 2.2e-7, + "output_cost_per_token": 2.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-2-11b-instruct-v1:0", + "name": "meta.llama3-2-11b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3.5e-7, + "output_cost_per_token": 3.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-2-1b-instruct-v1:0", + "name": "meta.llama3-2-1b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 1e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-2-3b-instruct-v1:0", + "name": "meta.llama3-2-3b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-2-90b-instruct-v1:0", + "name": "meta.llama3-2-90b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-3-70b-instruct-v1:0", + "name": "meta.llama3-3-70b-instruct-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 7.2e-7, + "output_cost_per_token": 7.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-70b-instruct-v1:0", + "name": "meta.llama3-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000265, + "output_cost_per_token": 0.0000035, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama3-8b-instruct-v1:0", + "name": "meta.llama3-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama4-maverick-17b-instruct-v1:0", + "name": "meta.llama4-maverick-17b-instruct-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 2.4e-7, + "output_cost_per_token": 9.7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "meta.llama4-scout-17b-instruct-v1:0", + "name": "meta.llama4-scout-17b-instruct-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 1.7e-7, + "output_cost_per_token": 6.6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "minimax.minimax-m2", + "name": "minimax.minimax-m2", + "provider": "bedrock_converse", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "minimax.minimax-m2.1", + "name": "minimax.minimax-m2.1", + "provider": "bedrock_converse", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "minimax.minimax-m2.5", + "name": "minimax.minimax-m2.5", + "provider": "bedrock_converse", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.devstral-2-123b", + "name": "mistral.devstral-2-123b", + "provider": "bedrock_converse", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.magistral-small-2509", + "name": "mistral.magistral-small-2509", + "provider": "bedrock_converse", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.ministral-3-14b-instruct", + "name": "mistral.ministral-3-14b-instruct", + "provider": "bedrock_converse", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.ministral-3-3b-instruct", + "name": "mistral.ministral-3-3b-instruct", + "provider": "bedrock_converse", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 1e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.ministral-3-8b-instruct", + "name": "mistral.ministral-3-8b-instruct", + "provider": "bedrock_converse", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.mistral-7b-instruct-v0:2", + "name": "mistral.mistral-7b-instruct-v0:2", + "provider": "bedrock", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.mistral-large-2402-v1:0", + "name": "mistral.mistral-large-2402-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.mistral-large-2407-v1:0", + "name": "mistral.mistral-large-2407-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000009, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.mistral-large-3-675b-instruct", + "name": "mistral.mistral-large-3-675b-instruct", + "provider": "bedrock_converse", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.mistral-small-2402-v1:0", + "name": "mistral.mistral-small-2402-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.mixtral-8x7b-instruct-v0:1", + "name": "mistral.mixtral-8x7b-instruct-v0:1", + "provider": "bedrock", + "input_cost_per_token": 4.5e-7, + "output_cost_per_token": 7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.voxtral-mini-3b-2507", + "name": "mistral.voxtral-mini-3b-2507", + "provider": "bedrock_converse", + "input_cost_per_token": 4e-8, + "output_cost_per_token": 4e-8, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "mistral.voxtral-small-24b-2507", + "name": "mistral.voxtral-small-24b-2507", + "provider": "bedrock_converse", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "moonshot.kimi-k2-thinking", + "name": "moonshot.kimi-k2-thinking", + "provider": "bedrock_converse", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "moonshotai.kimi-k2.5", + "name": "moonshotai.kimi-k2.5", + "provider": "bedrock_converse", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "nvidia.nemotron-nano-12b-v2", + "name": "nvidia.nemotron-nano-12b-v2", + "provider": "bedrock_converse", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "nvidia.nemotron-nano-9b-v2", + "name": "nvidia.nemotron-nano-9b-v2", + "provider": "bedrock_converse", + "input_cost_per_token": 6e-8, + "output_cost_per_token": 2.3e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "nvidia.nemotron-nano-3-30b", + "name": "nvidia.nemotron-nano-3-30b", + "provider": "bedrock_converse", + "input_cost_per_token": 6e-8, + "output_cost_per_token": 2.4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "nvidia.nemotron-super-3-120b", + "name": "nvidia.nemotron-super-3-120b", + "provider": "bedrock_converse", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "o1", + "name": "o1", + "provider": "openai", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": 0.0000075, + "output_cost_per_reasoning_token": null + }, + { + "id": "o1-2024-12-17", + "name": "o1-2024-12-17", + "provider": "openai", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.00006, + "cache_read_input_token_cost": 0.0000075, + "output_cost_per_reasoning_token": null + }, + { + "id": "o3", + "name": "o3", + "provider": "openai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "o3-2025-04-16", + "name": "o3-2025-04-16", + "provider": "openai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "o3-mini", + "name": "o3-mini", + "provider": "openai", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000044, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "o3-mini-2025-01-31", + "name": "o3-mini-2025-01-31", + "provider": "openai", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000044, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "o4-mini", + "name": "o4-mini", + "provider": "openai", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000044, + "cache_read_input_token_cost": 2.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "o4-mini-2025-04-16", + "name": "o4-mini-2025-04-16", + "provider": "openai", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000044, + "cache_read_input_token_cost": 2.75e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "openai.gpt-oss-120b-1:0", + "name": "openai.gpt-oss-120b-1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "openai.gpt-oss-20b-1:0", + "name": "openai.gpt-oss-20b-1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 7e-8, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "openai.gpt-oss-safeguard-120b", + "name": "openai.gpt-oss-safeguard-120b", + "provider": "bedrock_converse", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "openai.gpt-oss-safeguard-20b", + "name": "openai.gpt-oss-safeguard-20b", + "provider": "bedrock_converse", + "input_cost_per_token": 7e-8, + "output_cost_per_token": 2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "qwen.qwen3-coder-480b-a35b-v1:0", + "name": "qwen.qwen3-coder-480b-a35b-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 2.2e-7, + "output_cost_per_token": 0.0000018, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "qwen.qwen3-235b-a22b-2507-v1:0", + "name": "qwen.qwen3-235b-a22b-2507-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 2.2e-7, + "output_cost_per_token": 8.8e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "qwen.qwen3-coder-30b-a3b-v1:0", + "name": "qwen.qwen3-coder-30b-a3b-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "qwen.qwen3-32b-v1:0", + "name": "qwen.qwen3-32b-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "qwen.qwen3-next-80b-a3b", + "name": "qwen.qwen3-next-80b-a3b", + "provider": "bedrock_converse", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "qwen.qwen3-vl-235b-a22b", + "name": "qwen.qwen3-vl-235b-a22b", + "provider": "bedrock_converse", + "input_cost_per_token": 5.3e-7, + "output_cost_per_token": 0.00000266, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "qwen.qwen3-coder-next", + "name": "qwen.qwen3-coder-next", + "provider": "bedrock_converse", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.amazon.nova-lite-v1:0", + "name": "us.amazon.nova-lite-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 6e-8, + "output_cost_per_token": 2.4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.amazon.nova-micro-v1:0", + "name": "us.amazon.nova-micro-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 3.5e-8, + "output_cost_per_token": 1.4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.amazon.nova-premier-v1:0", + "name": "us.amazon.nova-premier-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.0000125, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.amazon.nova-pro-v1:0", + "name": "us.amazon.nova-pro-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 8e-7, + "output_cost_per_token": 0.0000032, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-3-5-haiku-20241022-v1:0", + "name": "us.anthropic.claude-3-5-haiku-20241022-v1:0", + "provider": "bedrock", + "input_cost_per_token": 8e-7, + "output_cost_per_token": 0.000004, + "cache_read_input_token_cost": 8e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "name": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000055, + "cache_read_input_token_cost": 1.1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "name": "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "name": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "name": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-3-haiku-20240307-v1:0", + "name": "us.anthropic.claude-3-haiku-20240307-v1:0", + "provider": "bedrock", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-3-opus-20240229-v1:0", + "name": "us.anthropic.claude-3-opus-20240229-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-3-sonnet-20240229-v1:0", + "name": "us.anthropic.claude-3-sonnet-20240229-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-opus-4-1-20250805-v1:0", + "name": "us.anthropic.claude-opus-4-1-20250805-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000033, + "output_cost_per_token": 0.0000165, + "cache_read_input_token_cost": 3.3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0", + "name": "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000036, + "output_cost_per_token": 0.000018, + "cache_read_input_token_cost": 3.6e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "au.anthropic.claude-haiku-4-5-20251001-v1:0", + "name": "au.anthropic.claude-haiku-4-5-20251001-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000011, + "output_cost_per_token": 0.0000055, + "cache_read_input_token_cost": 1.1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-opus-4-20250514-v1:0", + "name": "us.anthropic.claude-opus-4-20250514-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-opus-4-5-20251101-v1:0", + "name": "us.anthropic.claude-opus-4-5-20251101-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.0000055, + "output_cost_per_token": 0.0000275, + "cache_read_input_token_cost": 5.5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "global.anthropic.claude-opus-4-5-20251101-v1:0", + "name": "global.anthropic.claude-opus-4-5-20251101-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.anthropic.claude-opus-4-5-20251101-v1:0", + "name": "eu.anthropic.claude-opus-4-5-20251101-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.anthropic.claude-sonnet-4-20250514-v1:0", + "name": "us.anthropic.claude-sonnet-4-20250514-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.deepseek.r1-v1:0", + "name": "us.deepseek.r1-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.00000135, + "output_cost_per_token": 0.0000054, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.deepseek.v3.2", + "name": "us.deepseek.v3.2", + "provider": "bedrock_converse", + "input_cost_per_token": 6.2e-7, + "output_cost_per_token": 0.00000185, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "eu.deepseek.v3.2", + "name": "eu.deepseek.v3.2", + "provider": "bedrock_converse", + "input_cost_per_token": 7.4e-7, + "output_cost_per_token": 0.00000222, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama3-1-405b-instruct-v1:0", + "name": "us.meta.llama3-1-405b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.00000532, + "output_cost_per_token": 0.000016, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama3-1-70b-instruct-v1:0", + "name": "us.meta.llama3-1-70b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 9.9e-7, + "output_cost_per_token": 9.9e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama3-1-8b-instruct-v1:0", + "name": "us.meta.llama3-1-8b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 2.2e-7, + "output_cost_per_token": 2.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama3-2-11b-instruct-v1:0", + "name": "us.meta.llama3-2-11b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 3.5e-7, + "output_cost_per_token": 3.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama3-2-1b-instruct-v1:0", + "name": "us.meta.llama3-2-1b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 1e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama3-2-3b-instruct-v1:0", + "name": "us.meta.llama3-2-3b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama3-2-90b-instruct-v1:0", + "name": "us.meta.llama3-2-90b-instruct-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama3-3-70b-instruct-v1:0", + "name": "us.meta.llama3-3-70b-instruct-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 7.2e-7, + "output_cost_per_token": 7.2e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama4-maverick-17b-instruct-v1:0", + "name": "us.meta.llama4-maverick-17b-instruct-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 2.4e-7, + "output_cost_per_token": 9.7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.meta.llama4-scout-17b-instruct-v1:0", + "name": "us.meta.llama4-scout-17b-instruct-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 1.7e-7, + "output_cost_per_token": 6.6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "us.mistral.pixtral-large-2502-v1:0", + "name": "us.mistral.pixtral-large-2502-v1:0", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-5-haiku", + "name": "claude-3-5-haiku", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-5-haiku@20241022", + "name": "claude-3-5-haiku@20241022", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-haiku-4-5", + "name": "claude-haiku-4-5", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-haiku-4-5@20251001", + "name": "claude-haiku-4-5@20251001", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000005, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-5-sonnet", + "name": "claude-3-5-sonnet", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-5-sonnet@20240620", + "name": "claude-3-5-sonnet@20240620", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-7-sonnet@20250219", + "name": "claude-3-7-sonnet@20250219", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-haiku", + "name": "claude-3-haiku", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-haiku@20240307", + "name": "claude-3-haiku@20240307", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.00000125, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-opus", + "name": "claude-3-opus", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-opus@20240229", + "name": "claude-3-opus@20240229", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-sonnet", + "name": "claude-3-sonnet", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-3-sonnet@20240229", + "name": "claude-3-sonnet@20240229", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4", + "name": "claude-opus-4", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-1", + "name": "claude-opus-4-1", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-1@20250805", + "name": "claude-opus-4-1@20250805", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-5", + "name": "claude-opus-4-5", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-5@20251101", + "name": "claude-opus-4-5@20251101", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-6", + "name": "claude-opus-4-6", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-6@default", + "name": "claude-opus-4-6@default", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-7", + "name": "claude-opus-4-7", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-7@default", + "name": "claude-opus-4-7@default", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-fable-5", + "name": "claude-fable-5", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00005, + "cache_read_input_token_cost": 0.000001, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-fable-5@default", + "name": "claude-fable-5@default", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00005, + "cache_read_input_token_cost": 0.000001, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-5", + "name": "claude-opus-5", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-5@default", + "name": "claude-opus-5@default", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-8", + "name": "claude-opus-4-8", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4-8@default", + "name": "claude-opus-4-8@default", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000025, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-sonnet-4-5", + "name": "claude-sonnet-4-5", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-sonnet-5", + "name": "claude-sonnet-5", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-sonnet-4-6", + "name": "claude-sonnet-4-6", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-sonnet-4-5@20250929", + "name": "claude-sonnet-4-5@20250929", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-opus-4@20250514", + "name": "claude-opus-4@20250514", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, + "cache_read_input_token_cost": 0.0000015, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-sonnet-4", + "name": "claude-sonnet-4", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-sonnet-4@20250514", + "name": "claude-sonnet-4@20250514", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistralai/codestral-2@001", + "name": "mistralai/codestral-2@001", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 9e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/codestral-2", + "name": "codestral-2", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 9e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/codestral-2@001", + "name": "codestral-2@001", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 9e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistralai/codestral-2", + "name": "mistralai/codestral-2", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 9e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/codestral-2501", + "name": "codestral-2501", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/codestral@2405", + "name": "codestral@2405", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/codestral@latest", + "name": "codestral@latest", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/deepseek-ai/deepseek-v3.1-maas", + "name": "deepseek-ai/deepseek-v3.1-maas", + "provider": "vertex_ai-deepseek_models", + "input_cost_per_token": 0.00000135, + "output_cost_per_token": 0.0000054, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/deepseek-ai/deepseek-v3.2-maas", + "name": "deepseek-ai/deepseek-v3.2-maas", + "provider": "vertex_ai-deepseek_models", + "input_cost_per_token": 5.6e-7, + "output_cost_per_token": 0.00000168, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/deepseek-ai/deepseek-r1-0528-maas", + "name": "deepseek-ai/deepseek-r1-0528-maas", + "provider": "vertex_ai-deepseek_models", + "input_cost_per_token": 0.00000135, + "output_cost_per_token": 0.0000054, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/gemini-3.1-flash-lite-preview", + "name": "gemini-3.1-flash-lite-preview", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": 0.0000015 + }, + { + "id": "vertex_ai/gemini-3.1-flash-lite", + "name": "gemini-3.1-flash-lite", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.0000015, + "cache_read_input_token_cost": 2.5e-8, + "output_cost_per_reasoning_token": 0.0000015 + }, + { + "id": "vertex_ai/gemini-3.5-flash-lite", + "name": "gemini-3.5-flash-lite", + "provider": "vertex_ai-language-models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "vertex_ai/jamba-1.5", + "name": "jamba-1.5", + "provider": "vertex_ai-ai21_models", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/jamba-1.5-large", + "name": "jamba-1.5-large", + "provider": "vertex_ai-ai21_models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/jamba-1.5-large@001", + "name": "jamba-1.5-large@001", + "provider": "vertex_ai-ai21_models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000008, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/jamba-1.5-mini", + "name": "jamba-1.5-mini", + "provider": "vertex_ai-ai21_models", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/jamba-1.5-mini@001", + "name": "jamba-1.5-mini@001", + "provider": "vertex_ai-ai21_models", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama-3.1-405b-instruct-maas", + "name": "meta/llama-3.1-405b-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000016, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama-3.1-70b-instruct-maas", + "name": "meta/llama-3.1-70b-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama-3.1-8b-instruct-maas", + "name": "meta/llama-3.1-8b-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama-3.2-90b-vision-instruct-maas", + "name": "meta/llama-3.2-90b-vision-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas", + "name": "meta/llama-4-maverick-17b-128e-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 3.5e-7, + "output_cost_per_token": 0.00000115, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas", + "name": "meta/llama-4-maverick-17b-16e-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 3.5e-7, + "output_cost_per_token": 0.00000115, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas", + "name": "meta/llama-4-scout-17b-128e-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas", + "name": "meta/llama-4-scout-17b-16e-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 7e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama3-405b-instruct-maas", + "name": "meta/llama3-405b-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama3-70b-instruct-maas", + "name": "meta/llama3-70b-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/meta/llama3-8b-instruct-maas", + "name": "meta/llama3-8b-instruct-maas", + "provider": "vertex_ai-llama_models", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/minimaxai/minimax-m2-maas", + "name": "minimaxai/minimax-m2-maas", + "provider": "vertex_ai-minimax_models", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/moonshotai/kimi-k2-thinking-maas", + "name": "moonshotai/kimi-k2-thinking-maas", + "provider": "vertex_ai-moonshot_models", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/zai-org/glm-4.7-maas", + "name": "zai-org/glm-4.7-maas", + "provider": "vertex_ai-zai_models", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000022, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/zai-org/glm-5-maas", + "name": "zai-org/glm-5-maas", + "provider": "vertex_ai-zai_models", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.0000032, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-medium-3", + "name": "mistral-medium-3", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-medium-3@001", + "name": "mistral-medium-3@001", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistralai/mistral-medium-3", + "name": "mistralai/mistral-medium-3", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistralai/mistral-medium-3@001", + "name": "mistralai/mistral-medium-3@001", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 4e-7, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-large-2411", + "name": "mistral-large-2411", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-large@2407", + "name": "mistral-large@2407", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-large@2411-001", + "name": "mistral-large@2411-001", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-large@latest", + "name": "mistral-large@latest", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-nemo@2407", + "name": "mistral-nemo@2407", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-nemo@latest", + "name": "mistral-nemo@latest", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-small-2503", + "name": "mistral-small-2503", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/mistral-small-2503@001", + "name": "mistral-small-2503@001", + "provider": "vertex_ai-mistral_models", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/google/gemma-4-26b-a4b-it-maas", + "name": "google/gemma-4-26b-a4b-it-maas", + "provider": "vertex_ai-openai_models", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/openai/gpt-oss-120b-maas", + "name": "openai/gpt-oss-120b-maas", + "provider": "vertex_ai-openai_models", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/openai/gpt-oss-20b-maas", + "name": "openai/gpt-oss-20b-maas", + "provider": "vertex_ai-openai_models", + "input_cost_per_token": 7.5e-8, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/xai/grok-4.1-fast-non-reasoning", + "name": "xai/grok-4.1-fast-non-reasoning", + "provider": "vertex_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 5e-7, + "cache_read_input_token_cost": 5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/xai/grok-4.1-fast-reasoning", + "name": "xai/grok-4.1-fast-reasoning", + "provider": "vertex_ai", + "input_cost_per_token": 2e-7, + "output_cost_per_token": 5e-7, + "cache_read_input_token_cost": 5e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/xai/grok-4.20-non-reasoning", + "name": "xai/grok-4.20-non-reasoning", + "provider": "vertex_ai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/xai/grok-4.20-reasoning", + "name": "xai/grok-4.20-reasoning", + "provider": "vertex_ai", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas", + "name": "qwen/qwen3-235b-a22b-instruct-2507-maas", + "provider": "vertex_ai-qwen_models", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 0.000001, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", + "name": "qwen/qwen3-coder-480b-a35b-instruct-maas", + "provider": "vertex_ai-qwen_models", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000004, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas", + "name": "qwen/qwen3-next-80b-a3b-instruct-maas", + "provider": "vertex_ai-qwen_models", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas", + "name": "qwen/qwen3-next-80b-a3b-thinking-maas", + "provider": "vertex_ai-qwen_models", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 0.0000012, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "zai.glm-4.7", + "name": "zai.glm-4.7", + "provider": "bedrock_converse", + "input_cost_per_token": 6e-7, + "output_cost_per_token": 0.0000022, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "zai.glm-5", + "name": "zai.glm-5", + "provider": "bedrock_converse", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.0000032, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "zai.glm-4.7-flash", + "name": "zai.glm-4.7-flash", + "provider": "bedrock_converse", + "input_cost_per_token": 7e-8, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5-search-api", + "name": "gpt-5-search-api", + "provider": "openai", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gpt-5-search-api-2025-10-14", + "name": "gpt-5-search-api-2025-10-14", + "provider": "openai", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-2.0-flash-lite-001", + "name": "gemini-2.0-flash-lite-001", + "provider": "gemini", + "input_cost_per_token": 7.5e-8, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": 1.875e-8, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-2.5-flash-native-audio-latest", + "name": "gemini-2.5-flash-native-audio-latest", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-2.5-flash-native-audio-preview-09-2025", + "name": "gemini-2.5-flash-native-audio-preview-09-2025", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-2.5-flash-native-audio-preview-12-2025", + "name": "gemini-2.5-flash-native-audio-preview-12-2025", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-3.1-flash-live-preview", + "name": "gemini-3.1-flash-live-preview", + "provider": "gemini", + "input_cost_per_token": 7.5e-7, + "output_cost_per_token": 0.0000045, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-2.5-flash-native-audio-latest", + "name": "gemini-2.5-flash-native-audio-latest", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-2.5-flash-native-audio-preview-09-2025", + "name": "gemini-2.5-flash-native-audio-preview-09-2025", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-2.5-flash-native-audio-preview-12-2025", + "name": "gemini-2.5-flash-native-audio-preview-12-2025", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-3.1-flash-live-preview", + "name": "gemini-3.1-flash-live-preview", + "provider": "gemini", + "input_cost_per_token": 7.5e-7, + "output_cost_per_token": 0.0000045, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-flash-latest", + "name": "gemini-flash-latest", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "gemini-flash-lite-latest", + "name": "gemini-flash-lite-latest", + "provider": "gemini", + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": 1e-8, + "output_cost_per_reasoning_token": 4e-7 + }, + { + "id": "gemini-pro-latest", + "name": "gemini-pro-latest", + "provider": "gemini", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini/gemini-pro-latest", + "name": "gemini-pro-latest", + "provider": "gemini", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 1.25e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "gemini-exp-1206", + "name": "gemini-exp-1206", + "provider": "gemini", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 3e-8, + "output_cost_per_reasoning_token": 0.0000025 + }, + { + "id": "vertex_ai/claude-sonnet-5@default", + "name": "claude-sonnet-5@default", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "vertex_ai/claude-sonnet-4-6@default", + "name": "claude-sonnet-4-6@default", + "provider": "vertex_ai-anthropic_models", + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock_mantle/openai.gpt-oss-120b", + "name": "openai.gpt-oss-120b", + "provider": "bedrock_mantle", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock_mantle/openai.gpt-oss-20b", + "name": "openai.gpt-oss-20b", + "provider": "bedrock_mantle", + "input_cost_per_token": 7.5e-8, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock_mantle/openai.gpt-oss-safeguard-120b", + "name": "openai.gpt-oss-safeguard-120b", + "provider": "bedrock_mantle", + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock_mantle/openai.gpt-oss-safeguard-20b", + "name": "openai.gpt-oss-safeguard-20b", + "provider": "bedrock_mantle", + "input_cost_per_token": 7.5e-8, + "output_cost_per_token": 3e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock_mantle/google.gemma-4-31b", + "name": "google.gemma-4-31b", + "provider": "bedrock_mantle", + "input_cost_per_token": 1.4e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock_mantle/google.gemma-4-26b-a4b", + "name": "google.gemma-4-26b-a4b", + "provider": "bedrock_mantle", + "input_cost_per_token": 1.3e-7, + "output_cost_per_token": 4e-7, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock_mantle/google.gemma-4-e2b", + "name": "google.gemma-4-e2b", + "provider": "bedrock_mantle", + "input_cost_per_token": 4e-8, + "output_cost_per_token": 8e-8, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock_mantle/xai.grok-4.3", + "name": "xai.grok-4.3", + "provider": "bedrock_mantle", + "input_cost_per_token": 0.00000125, + "output_cost_per_token": 0.0000025, + "cache_read_input_token_cost": 2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-east-1/zai.glm-5", + "name": "us-east-1/zai.glm-5", + "provider": "bedrock", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.0000032, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-west-2/zai.glm-5", + "name": "us-west-2/zai.glm-5", + "provider": "bedrock", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.0000032, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "name": "us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000012, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": 1.2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "name": "us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "provider": "bedrock", + "input_cost_per_token": 0.0000012, + "output_cost_per_token": 0.000006, + "cache_read_input_token_cost": 1.2e-7, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-3.5-turbo-instruct-0914", + "name": "gpt-3.5-turbo-instruct-0914", + "provider": "azure_text", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-35-turbo-instruct", + "name": "gpt-35-turbo-instruct", + "provider": "azure_text", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "azure/gpt-35-turbo-instruct-0914", + "name": "gpt-35-turbo-instruct-0914", + "provider": "azure_text", + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000002, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "text-unicorn", + "name": "text-unicorn", + "provider": "vertex_ai-text-models", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.000028, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + }, + { + "id": "text-unicorn@001", + "name": "text-unicorn@001", + "provider": "vertex_ai-text-models", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.000028, + "cache_read_input_token_cost": null, + "output_cost_per_reasoning_token": null + } + ], + "success": true, + "error": false, + "message": "Model catalog retrieved successfully", + "status": 200 +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 975d49ed8..507268725 100644 --- a/package-lock.json +++ b/package-lock.json @@ -65,6 +65,7 @@ "express": "5.2.1", "express-rate-limit": "8.5.2", "glob": "13.0.6", + "gpt-tokenizer": "^3.4.0", "helmet": "8.2.0", "http-errors": "2.0.1", "input-otp": "1.4.2", @@ -12760,6 +12761,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gpt-tokenizer": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/gpt-tokenizer/-/gpt-tokenizer-3.4.0.tgz", + "integrity": "sha512-wxFLnhIXTDjYebd9A9pGl3e31ZpSypbpIJSOswbgop5jLte/AsZVDvjlbEuVFlsqZixVKqbcoNmRlFDf6pz/UQ==", + "license": "MIT" + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", diff --git a/package.json b/package.json index 058314d9f..e94b2fda3 100644 --- a/package.json +++ b/package.json @@ -33,9 +33,9 @@ }, "dependencies": { "@asteasolutions/zod-to-openapi": "8.5.0", + "@aws-sdk/client-s3": "3.1056.0", "@devolutions/iron-remote-desktop": "https://static.pangolin.net/packages/devolutions-iron-remote-desktop-0.0.0.tgz", "@devolutions/iron-remote-desktop-rdp": "https://static.pangolin.net/packages/devolutions-iron-remote-desktop-rdp-0.0.1.tgz", - "@aws-sdk/client-s3": "3.1056.0", "@headlessui/react": "2.2.10", "@hookform/resolvers": "5.4.0", "@monaco-editor/react": "4.7.0", @@ -88,6 +88,7 @@ "express": "5.2.1", "express-rate-limit": "8.5.2", "glob": "13.0.6", + "gpt-tokenizer": "^3.4.0", "helmet": "8.2.0", "http-errors": "2.0.1", "input-otp": "1.4.2", diff --git a/server/lib/aiModelPricing.ts b/server/lib/aiModelPricing.ts new file mode 100644 index 000000000..3edf4e9f4 --- /dev/null +++ b/server/lib/aiModelPricing.ts @@ -0,0 +1,230 @@ +import fs from "node:fs"; +import path from "node:path"; +import { APP_PATH } from "@server/lib/consts"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import type { AiUsage } from "@server/lib/aiUsageExtraction"; +import logger from "@server/logger"; + +// config/models.json is a runtime asset (same category as config.yml or the +// MaxMind DBs) - not part of the source tree. Its shape mirrors litellm's +// public model_prices_and_context_window.json: a flat list of +// { id, name, provider, input_cost_per_token, output_cost_per_token, +// cache_read_input_token_cost, output_cost_per_reasoning_token }, where +// `provider` is litellm's provider bucket, not our AiProviderType. +const MODELS_JSON_PATH = path.join(APP_PATH, "models.json"); + +export type AiModelPricingEntry = { + id: string; + name: string; + provider: string; + input_cost_per_token: number | null; + output_cost_per_token: number | null; + cache_read_input_token_cost: number | null; + output_cost_per_reasoning_token: number | null; +}; + +export type AiModelPricing = { + inputCostPerToken: number | null; + outputCostPerToken: number | null; + cacheReadInputTokenCost: number | null; + outputCostPerReasoningToken: number | null; + // True when the match came from a different provider bucket than the one + // mapped to this provider's type (e.g. an openRouter/custom model id that + // only matched by stripping a "vendor/" prefix against the whole table). + // Costs found this way are a best-effort approximation, not a guarantee + // the upstream provider bills at the same rate. + approximate: boolean; +}; + +// Which litellm provider buckets to search for each of our provider types. +// Several of our provider types (openRouter, vercelAiGateway, custom) proxy +// arbitrary underlying models and have no dedicated bucket in the pricing +// data, so they fall back to a global search across all buckets. +const PROVIDER_PRICING_BUCKETS: Record< + Exclude, + string[] +> = { + openai: ["openai"], + anthropic: ["anthropic"], + googleGemini: ["gemini"], + vertexAi: [ + "vertex_ai-language-models", + "vertex_ai", + "vertex_ai-anthropic_models", + "vertex_ai-mistral_models", + "vertex_ai-deepseek_models", + "vertex_ai-ai21_models", + "vertex_ai-llama_models", + "vertex_ai-minimax_models", + "vertex_ai-moonshot_models", + "vertex_ai-zai_models", + "vertex_ai-openai_models", + "vertex_ai-qwen_models", + "vertex_ai-text-models" + ], + bedrock: ["bedrock_converse", "bedrock", "bedrock_mantle"], + microsoftFoundry: ["azure", "azure_ai", "azure_text"], + openRouter: [], + vercelAiGateway: [] +}; + +let modelsById: Map | null = null; + +function loadModels(): Map { + if (modelsById) { + return modelsById; + } + + const byId = new Map(); + try { + if (fs.existsSync(MODELS_JSON_PATH)) { + const raw = fs.readFileSync(MODELS_JSON_PATH, "utf-8"); + const parsed = JSON.parse(raw) as { data: AiModelPricingEntry[] }; + for (const entry of parsed.data ?? []) { + for (const key of [entry.id, entry.name]) { + if (!key) continue; + const list = byId.get(key) ?? []; + list.push(entry); + byId.set(key, list); + } + } + } else { + logger.debug( + `AI model pricing file not found at ${MODELS_JSON_PATH}; cost calculation will fall back to unknown pricing` + ); + } + } catch (error) { + logger.warn("Failed to load AI model pricing file", { error }); + } + + modelsById = byId; + return byId; +} + +function stripVendorPrefix(modelId: string): string | null { + const idx = modelId.indexOf("/"); + if (idx === -1 || idx === modelId.length - 1) { + return null; + } + return modelId.slice(idx + 1); +} + +function toPricing( + entry: AiModelPricingEntry, + approximate: boolean +): AiModelPricing { + return { + inputCostPerToken: entry.input_cost_per_token, + outputCostPerToken: entry.output_cost_per_token, + cacheReadInputTokenCost: entry.cache_read_input_token_cost, + outputCostPerReasoningToken: entry.output_cost_per_reasoning_token, + approximate + }; +} + +function findInBuckets( + byId: Map, + modelId: string, + buckets: string[] | null +): AiModelPricingEntry | null { + const candidates = [modelId, stripVendorPrefix(modelId)].filter( + (v): v is string => v != null + ); + + for (const key of candidates) { + const entries = byId.get(key); + if (!entries) continue; + const match = buckets + ? entries.find((e) => buckets.includes(e.provider)) + : entries[0]; + if (match) { + return match; + } + } + return null; +} + +/** + * Looks up per-token pricing for a model, scoped first to the litellm + * provider bucket(s) that correspond to our provider type, then falling + * back to a global search across all buckets (marked `approximate`) for + * provider types that proxy arbitrary underlying models. + */ +export function getModelPricing( + providerType: AiProviderType, + modelId: string | undefined +): AiModelPricing | null { + if (!modelId) { + return null; + } + + const byId = loadModels(); + const buckets = + providerType === "custom" + ? [] + : PROVIDER_PRICING_BUCKETS[providerType]; + + if (buckets && buckets.length > 0) { + const scoped = findInBuckets(byId, modelId, buckets); + if (scoped) { + return toPricing(scoped, false); + } + } + + const fallback = findInBuckets(byId, modelId, null); + if (fallback) { + return toPricing(fallback, true); + } + + return null; +} + +export type AiCostBreakdown = { + promptCost: number; + cacheReadCost: number; + cacheWriteCost: number; + completionCost: number; + reasoningCost: number; + totalCost: number; +}; + +/** + * Computes a $ cost breakdown for a usage record given a model's pricing. + * Cache writes and reasoning tokens fall back to the normal input/output + * rate respectively when the pricing data has no dedicated rate for them + * (the models.json schema here has no cache-write field at all, and only + * some models report a distinct reasoning rate). + */ +export function calculateAiCost( + pricing: AiModelPricing | null, + usage: AiUsage +): AiCostBreakdown | null { + if (!pricing) { + return null; + } + + const inputRate = pricing.inputCostPerToken ?? 0; + const outputRate = pricing.outputCostPerToken ?? 0; + const cacheReadRate = pricing.cacheReadInputTokenCost ?? inputRate; + const reasoningRate = pricing.outputCostPerReasoningToken ?? outputRate; + + const promptCost = usage.promptTokens * inputRate; + const cacheReadCost = usage.cacheReadTokens * cacheReadRate; + const cacheWriteCost = usage.cacheWriteTokens * inputRate; + const completionCost = usage.completionTokens * outputRate; + const reasoningCost = usage.reasoningTokens * reasoningRate; + + return { + promptCost, + cacheReadCost, + cacheWriteCost, + completionCost, + reasoningCost, + totalCost: + promptCost + + cacheReadCost + + cacheWriteCost + + completionCost + + reasoningCost + }; +} diff --git a/server/lib/aiUsageExtraction.ts b/server/lib/aiUsageExtraction.ts new file mode 100644 index 000000000..f3fc92580 --- /dev/null +++ b/server/lib/aiUsageExtraction.ts @@ -0,0 +1,468 @@ +import { encode } from "gpt-tokenizer"; +import type { AiCapability } from "@server/lib/aiCapabilities"; +import logger from "@server/logger"; + +export type AiUsage = { + // Input tokens billed at the normal input rate (i.e. NOT already + // covered by cacheReadTokens/cacheWriteTokens below). + promptTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + // Output tokens billed at the normal output rate (i.e. NOT already + // covered by reasoningTokens below). + completionTokens: number; + reasoningTokens: number; + // True when these numbers are our own best-guess estimate (the upstream + // response didn't report usage), rather than provider-reported figures. + estimated: boolean; +}; + +function emptyUsage(): AiUsage { + return { + promptTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + completionTokens: 0, + reasoningTokens: 0, + estimated: false + }; +} + +/** + * Scans raw (possibly binary-framed, e.g. Bedrock's vnd.amazon.eventstream) + * text for `"fieldName":123` occurrences and returns the last value seen for + * each field. Used as a best-effort fallback for response shapes we can't + * fully parse as JSON/SSE (streaming Bedrock, raw predict passthroughs). + */ +function scanNumericFields( + text: string, + fields: string[] +): Record { + const out: Record = {}; + for (const field of fields) { + const re = new RegExp(`"${field}"\\s*:\\s*(\\d+)`, "g"); + let match: RegExpExecArray | null; + while ((match = re.exec(text)) !== null) { + out[field] = Number(match[1]); + } + } + return out; +} + +function sseDataFrames(text: string): string[] { + const frames: string[] = []; + for (const rawFrame of text.split(/\r?\n\r?\n/)) { + for (const line of rawFrame.split(/\r?\n/)) { + if (!line.startsWith("data:")) continue; + const data = line.slice("data:".length).trim(); + if (data && data !== "[DONE]") { + frames.push(data); + } + } + } + return frames; +} + +function tryParseJson(text: string): any | null { + try { + return JSON.parse(text); + } catch { + return null; + } +} + +function extractOpenAiChat(text: string, isStream: boolean): AiUsage | null { + let usage: any = null; + + if (isStream) { + for (const frame of sseDataFrames(text)) { + const parsed = tryParseJson(frame); + if (parsed?.usage) { + usage = parsed.usage; + } + } + } else { + usage = tryParseJson(text)?.usage ?? null; + } + + if (!usage) { + return null; + } + + const cacheReadTokens = usage.prompt_tokens_details?.cached_tokens ?? 0; + const reasoningTokens = + usage.completion_tokens_details?.reasoning_tokens ?? 0; + + return { + promptTokens: Math.max(0, (usage.prompt_tokens ?? 0) - cacheReadTokens), + cacheReadTokens, + cacheWriteTokens: 0, + completionTokens: Math.max( + 0, + (usage.completion_tokens ?? 0) - reasoningTokens + ), + reasoningTokens, + estimated: false + }; +} + +function extractOpenAiResponses( + text: string, + isStream: boolean +): AiUsage | null { + let usage: any = null; + + if (isStream) { + for (const frame of sseDataFrames(text)) { + const parsed = tryParseJson(frame); + if (parsed?.type === "response.completed" && parsed?.response?.usage) { + usage = parsed.response.usage; + } else if (parsed?.usage) { + usage = parsed.usage; + } + } + } else { + const parsed = tryParseJson(text); + usage = parsed?.usage ?? parsed?.response?.usage ?? null; + } + + if (!usage) { + return null; + } + + const cacheReadTokens = usage.input_tokens_details?.cached_tokens ?? 0; + const reasoningTokens = usage.output_tokens_details?.reasoning_tokens ?? 0; + + return { + promptTokens: Math.max(0, (usage.input_tokens ?? 0) - cacheReadTokens), + cacheReadTokens, + cacheWriteTokens: 0, + completionTokens: Math.max( + 0, + (usage.output_tokens ?? 0) - reasoningTokens + ), + reasoningTokens, + estimated: false + }; +} + +function extractAnthropicMessages( + text: string, + isStream: boolean +): AiUsage | null { + let inputTokens = 0; + let cacheReadTokens = 0; + let cacheWriteTokens = 0; + let outputTokens = 0; + let found = false; + + const applyUsage = (usage: any) => { + if (!usage) return; + found = true; + if (typeof usage.input_tokens === "number") { + inputTokens = usage.input_tokens; + } + if (typeof usage.cache_read_input_tokens === "number") { + cacheReadTokens = usage.cache_read_input_tokens; + } + if (typeof usage.cache_creation_input_tokens === "number") { + cacheWriteTokens = usage.cache_creation_input_tokens; + } + if (typeof usage.output_tokens === "number") { + outputTokens = usage.output_tokens; + } + }; + + if (isStream) { + for (const frame of sseDataFrames(text)) { + const parsed = tryParseJson(frame); + if (!parsed) continue; + applyUsage(parsed.message?.usage); + applyUsage(parsed.usage); + } + } else { + applyUsage(tryParseJson(text)?.usage); + } + + if (!found) { + return null; + } + + return { + promptTokens: inputTokens, + cacheReadTokens, + cacheWriteTokens, + completionTokens: outputTokens, + // Anthropic bills extended-thinking output at the normal output + // rate, so there's no separate reasoning bucket to report. + reasoningTokens: 0, + estimated: false + }; +} + +function extractGoogleGenerateContent( + text: string, + _isStream: boolean +): AiUsage | null { + // Both the plain-JSON-array stream format and the SSE (?alt=sse) format + // repeat a cumulative `usageMetadata` object per chunk; the regex scan + // below naturally picks up the last (most complete) one either way. + const fields = scanNumericFields(text, [ + "promptTokenCount", + "candidatesTokenCount", + "cachedContentTokenCount", + "thoughtsTokenCount" + ]); + + if (fields.promptTokenCount === undefined) { + return null; + } + + const cacheReadTokens = fields.cachedContentTokenCount ?? 0; + const reasoningTokens = fields.thoughtsTokenCount ?? 0; + + return { + promptTokens: Math.max(0, fields.promptTokenCount - cacheReadTokens), + cacheReadTokens, + cacheWriteTokens: 0, + completionTokens: fields.candidatesTokenCount ?? 0, + reasoningTokens, + estimated: false + }; +} + +function extractBedrockConverse( + text: string, + _isStream: boolean +): AiUsage | null { + // Non-streaming responses are plain JSON; converse-stream frames the + // final `metadata` event's usage object inside binary event-stream + // framing, but the JSON text survives intact inside that binary + // envelope, so the same field scan works for both. + const parsed = tryParseJson(text); + const usage = parsed?.usage; + if (usage) { + const cacheReadTokens = usage.cacheReadInputTokens ?? 0; + return { + promptTokens: Math.max(0, (usage.inputTokens ?? 0) - cacheReadTokens), + cacheReadTokens, + cacheWriteTokens: usage.cacheWriteInputTokens ?? 0, + completionTokens: usage.outputTokens ?? 0, + reasoningTokens: 0, + estimated: false + }; + } + + const fields = scanNumericFields(text, [ + "inputTokens", + "outputTokens", + "cacheReadInputTokens", + "cacheWriteInputTokens" + ]); + if (fields.inputTokens === undefined) { + return null; + } + const cacheReadTokens = fields.cacheReadInputTokens ?? 0; + return { + promptTokens: Math.max(0, fields.inputTokens - cacheReadTokens), + cacheReadTokens, + cacheWriteTokens: fields.cacheWriteInputTokens ?? 0, + completionTokens: fields.outputTokens ?? 0, + reasoningTokens: 0, + estimated: false + }; +} + +function extractBedrockModelInvoke( + text: string, + _isStream: boolean, + headers: Headers +): AiUsage | null { + // Non-streaming invoke reports counts via response headers regardless + // of the underlying model's payload format. + const headerInput = headers.get("x-amzn-bedrock-input-token-count"); + const headerOutput = headers.get("x-amzn-bedrock-output-token-count"); + if (headerInput !== null || headerOutput !== null) { + return { + promptTokens: Number(headerInput ?? 0), + cacheReadTokens: 0, + cacheWriteTokens: 0, + completionTokens: Number(headerOutput ?? 0), + reasoningTokens: 0, + estimated: false + }; + } + + // invoke-with-response-stream has no equivalent headers; the model's + // own usage shape (frequently Anthropic-style on Bedrock) is embedded + // inside binary event-stream framing, so fall back to a couple of + // known field-name shapes via regex. + const anthropicStyle = extractAnthropicMessages(text, true); + if (anthropicStyle) { + return anthropicStyle; + } + + const fields = scanNumericFields(text, [ + "inputTokenCount", + "outputTokenCount" + ]); + if (fields.inputTokenCount === undefined) { + return null; + } + return { + promptTokens: fields.inputTokenCount, + cacheReadTokens: 0, + cacheWriteTokens: 0, + completionTokens: fields.outputTokenCount ?? 0, + reasoningTokens: 0, + estimated: false + }; +} + +const EXTRACTORS: Record< + AiCapability, + (text: string, isStream: boolean, headers: Headers) => AiUsage | null +> = { + openai_chat: extractOpenAiChat, + openai_responses: extractOpenAiResponses, + anthropic_messages: extractAnthropicMessages, + gemini_generate_content: extractGoogleGenerateContent, + google_generate_content: extractGoogleGenerateContent, + // rawPredict is a passthrough to whatever the underlying publisher + // model speaks (often Anthropic-shaped on Vertex); try that, then give + // up to the token-count estimate. + google_raw_predict: (text, isStream) => + extractAnthropicMessages(text, isStream), + bedrock_model_invoke: extractBedrockModelInvoke, + bedrock_converse: extractBedrockConverse +}; + +/** + * Attempts to pull provider-reported token usage out of an upstream AI + * gateway response. Returns null if the response didn't contain (or we + * couldn't find) usage data, in which case callers should fall back to + * `estimateUsage`. + */ +export function extractUsage( + capability: AiCapability, + responseText: string, + isStream: boolean, + headers: Headers +): AiUsage | null { + try { + return EXTRACTORS[capability](responseText, isStream, headers); + } catch (error) { + logger.debug("Failed to extract AI usage from response", { + capability, + error + }); + return null; + } +} + +/** + * Best-guess token estimate for when the provider doesn't report usage. + * Uses OpenAI's BPE tokenizer as a stand-in for whatever tokenizer the + * actual model uses - close enough for an approximate cost figure, not + * exact for non-OpenAI models. + */ +export function estimateUsage( + promptText: string, + completionText: string +): AiUsage { + const usage = emptyUsage(); + usage.estimated = true; + try { + usage.promptTokens = promptText ? encode(promptText).length : 0; + } catch (error) { + logger.debug("Failed to estimate prompt tokens", { error }); + } + try { + usage.completionTokens = completionText + ? encode(completionText).length + : 0; + } catch (error) { + logger.debug("Failed to estimate completion tokens", { error }); + } + return usage; +} + +/** + * OpenAI's Chat Completions API only includes a `usage` field in a + * streaming response when the request opts in via `stream_options: + * {include_usage: true}` - unlike the Responses API, Anthropic, Gemini and + * Bedrock, which report usage in a streaming response by default. Returns + * whether we need to inject that option ourselves to be able to track cost. + */ +export function needsStreamUsageInjection( + capability: AiCapability, + body: any +): boolean { + return ( + capability === "openai_chat" && + body?.stream === true && + body?.stream_options?.include_usage !== true + ); +} + +/** + * Returns a shallow-cloned body with `stream_options.include_usage` + * injected, for capabilities/requests where `needsStreamUsageInjection` + * is true. Leaves the original body untouched. + */ +export function withStreamUsageOption(body: any): any { + return { + ...body, + stream_options: { ...body.stream_options, include_usage: true } + }; +} + +/** + * When we injected stream_options.include_usage ourselves (the caller + * didn't ask for it), OpenAI appends an extra terminal SSE frame with an + * empty `choices: []` array carrying only the usage data. Callers that + * don't expect that shape (most minimal SSE parsers assume a non-empty + * choices array) shouldn't see it, so it's stripped back out of the bytes + * forwarded to the client. + */ +export function stripInjectedUsageFrame(sseText: string): string { + const parts = sseText.split(/(\r?\n\r?\n)/); + let out = ""; + for (let i = 0; i < parts.length; i += 2) { + const frame = parts[i]; + const separator = parts[i + 1] ?? ""; + const dataLine = frame + .split(/\r?\n/) + .find((line) => line.startsWith("data:")); + if (dataLine) { + const data = dataLine.slice("data:".length).trim(); + const parsed = data !== "[DONE]" ? tryParseJson(data) : null; + if (parsed && Array.isArray(parsed.choices) && parsed.choices.length === 0 && parsed.usage) { + continue; + } + } + out += frame + separator; + } + return out; +} + +/** + * Best-effort extraction of the model the upstream provider actually + * served, which some gateways/routers echo back and which may differ from + * the model the caller requested (e.g. an alias resolving to a dated + * snapshot). Falls back to the caller's requested model when absent. + */ +export function extractResponseModel(responseText: string): string | null { + const match = responseText.match(/"model"\s*:\s*"([^"]+)"/); + return match ? match[1] : null; +} + +export function isUsageEmpty(usage: AiUsage): boolean { + return ( + usage.promptTokens === 0 && + usage.cacheReadTokens === 0 && + usage.cacheWriteTokens === 0 && + usage.completionTokens === 0 && + usage.reasoningTokens === 0 + ); +} diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index d881b27ca..dbc3c22be 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -19,6 +19,7 @@ import config from "@server/lib/config"; import { decrypt } from "@server/lib/crypto"; import { AiProviderAuthType, + AiProviderType, applyAiProviderAuthHeaders, applyAiProviderCustomHeaders, authTypeRequiresApiKey @@ -49,6 +50,17 @@ import { mostSpecificMatchingAllow } from "@server/lib/aiModelKeyMatch"; import { aiGatewayUpstreamFetch } from "@server/lib/aiGatewayUpstreamFetch"; +import { getModelPricing, calculateAiCost } from "@server/lib/aiModelPricing"; +import { + extractUsage, + estimateUsage, + isUsageEmpty, + needsStreamUsageInjection, + withStreamUsageOption, + stripInjectedUsageFrame, + extractResponseModel, + type AiUsage +} from "@server/lib/aiUsageExtraction"; // Short-lived local caches so a burst of requests from the same IP/user // doesn't hit the database on every single request. None of this is @@ -500,6 +512,48 @@ async function selectProvider( }; } +function logAiUsageAndCost(args: { + capability: AiCapability; + provider: AiProvider; + requestedModel: string | undefined; + requestBody: unknown; + responseText: string; + isStream: boolean; + headers: Headers; +}): void { + const { capability, provider, requestedModel, requestBody, responseText, isStream, headers } = + args; + + let usage: AiUsage | null = extractUsage( + capability, + responseText, + isStream, + headers + ); + if (!usage || isUsageEmpty(usage)) { + usage = estimateUsage(JSON.stringify(requestBody ?? ""), responseText); + } + + const model = extractResponseModel(responseText) ?? requestedModel; + const pricing = getModelPricing(provider.type as AiProviderType, model); + const cost = calculateAiCost(pricing, usage); + + logger.info("AI gateway request usage", { + capability, + providerId: provider.providerId, + providerType: provider.type, + model, + estimated: usage.estimated, + promptTokens: usage.promptTokens, + cacheReadTokens: usage.cacheReadTokens, + cacheWriteTokens: usage.cacheWriteTokens, + completionTokens: usage.completionTokens, + reasoningTokens: usage.reasoningTokens, + pricingApproximate: pricing?.approximate ?? null, + totalCostUsd: cost?.totalCost ?? null + }); +} + export async function handleAiGatewayProxy( req: Request, res: Response, @@ -636,14 +690,25 @@ export async function handleAiGatewayProxy( applyAiProviderAuthHeaders(headers, authType, apiKey); applyRequestUserHeaders(headers, requestUser); - const body = JSON.stringify(req.body); + // OpenAI's Chat Completions API only reports usage in a streaming + // response when asked to via stream_options.include_usage - inject + // it ourselves when the caller didn't, so we can still track cost, + // and strip the extra frame it adds back out of what we forward. + const injectedUsageOurselves = needsStreamUsageInjection( + capability, + req.body + ); + const outboundBody = injectedUsageOurselves + ? withStreamUsageOption(req.body) + : req.body; + const body = JSON.stringify(outboundBody); logger.debug("AI gateway upstream request", { capability, url: targetUrl, method: "POST", headers, - body: req.body, + body: outboundBody, skipTlsVerification: provider.skipTlsVerification }); @@ -701,11 +766,31 @@ export async function handleAiGatewayProxy( if (isStream && upstreamRes.body) { res.flushHeaders(); const reader = upstreamRes.body.getReader(); + const decoder = new TextDecoder(); + let fullText = ""; + // Frame-boundary buffer, only used when we need to filter the + // usage-only frame we injected out of what reaches the client. + let sseCarry = ""; try { while (!abortController.signal.aborted) { const { done, value } = await reader.read(); if (done) break; - res.write(value); + const chunkText = decoder.decode(value, { stream: true }); + fullText += chunkText; + if (injectedUsageOurselves) { + sseCarry += chunkText; + const lastBoundary = sseCarry.lastIndexOf("\n\n"); + if (lastBoundary !== -1) { + const toEmit = sseCarry.slice(0, lastBoundary + 2); + sseCarry = sseCarry.slice(lastBoundary + 2); + res.write(stripInjectedUsageFrame(toEmit)); + } + } else { + res.write(value); + } + } + if (injectedUsageOurselves && sseCarry) { + res.write(stripInjectedUsageFrame(sseCarry)); } } finally { await reader.cancel().catch(() => {}); @@ -714,11 +799,31 @@ export async function handleAiGatewayProxy( if (!res.writableEnded) { res.end(); } + if (!abortController.signal.aborted) { + logAiUsageAndCost({ + capability, + provider, + requestedModel, + requestBody: req.body, + responseText: fullText, + isStream: true, + headers: upstreamRes.headers + }); + } return; } res.off("close", onClientClose); const text = await upstreamRes.text(); + logAiUsageAndCost({ + capability, + provider, + requestedModel, + requestBody: req.body, + responseText: text, + isStream: false, + headers: upstreamRes.headers + }); return res.send(text); } catch (error) { logger.error(error); From bc7a883f6ce3fc252d3fd1caf569ae5af42c9ffb Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 7 Aug 2026 14:21:13 -0400 Subject: [PATCH 080/296] add some parallelization to ai gateway pipeline --- server/routers/aiGateway/pipeline.ts | 238 ++++++++++++++------------- 1 file changed, 125 insertions(+), 113 deletions(-) diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index dbc3c22be..724102f07 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -242,137 +242,146 @@ async function resolveRequestUser( } async function resolveTarget(host: string): Promise { - const [resourceRow] = await db - .select({ - resourceId: resources.resourceId, - orgId: resources.orgId - }) - .from(resources) - .where( - and( - eq(resources.fullDomain, host), - eq(resources.mode, "inference"), - eq(resources.enabled, true) - ) - ) - .limit(1); - - if (resourceRow) { - const attachmentRows = await db + const [[resourceRow], [siteResourceRow]] = await Promise.all([ + db .select({ - provider: aiProviders, - accessMode: resourceAiProviders.accessMode + resourceId: resources.resourceId, + orgId: resources.orgId }) - .from(resourceAiProviders) - .innerJoin( - aiProviders, - eq(resourceAiProviders.providerId, aiProviders.providerId) - ) + .from(resources) .where( and( - eq(resourceAiProviders.resourceId, resourceRow.resourceId), - eq(aiProviders.enabled, true) + eq(resources.fullDomain, host), + eq(resources.mode, "inference"), + eq(resources.enabled, true) ) - ); + ) + .limit(1), + db + .select({ + siteResourceId: siteResources.siteResourceId, + orgId: siteResources.orgId + }) + .from(siteResources) + .where( + and( + eq(siteResources.fullDomain, host), + eq(siteResources.mode, "inference"), + eq(siteResources.enabled, true) + ) + ) + .limit(1) + ]); + + // Prefer public inference resources when both match the same host. + if (resourceRow) { + const [attachmentRows, resourcePatterns] = await Promise.all([ + db + .select({ + provider: aiProviders, + accessMode: resourceAiProviders.accessMode + }) + .from(resourceAiProviders) + .innerJoin( + aiProviders, + eq(resourceAiProviders.providerId, aiProviders.providerId) + ) + .where( + and( + eq( + resourceAiProviders.resourceId, + resourceRow.resourceId + ), + eq(aiProviders.enabled, true) + ) + ), + db + .select({ + providerId: aiModels.providerId, + modelKey: aiModels.modelKey, + listType: resourceAiModels.listType, + enabled: aiModels.enabled + }) + .from(resourceAiModels) + .innerJoin( + aiModels, + eq(resourceAiModels.modelId, aiModels.modelId) + ) + .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)) + ]); if (attachmentRows.length === 0) { return null; } - const attachments: ProviderAttachment[] = attachmentRows.map((a) => ({ - provider: a.provider, - accessMode: a.accessMode - })); - - const resourcePatterns = await db - .select({ - providerId: aiModels.providerId, - modelKey: aiModels.modelKey, - listType: resourceAiModels.listType, - enabled: aiModels.enabled - }) - .from(resourceAiModels) - .innerJoin(aiModels, eq(resourceAiModels.modelId, aiModels.modelId)) - .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)); - return { resourceId: resourceRow.resourceId, siteResourceId: null, orgId: resourceRow.orgId, - attachments, + attachments: attachmentRows.map((a) => ({ + provider: a.provider, + accessMode: a.accessMode + })), resourceListsByProvider: groupPatternsByProvider(resourcePatterns) }; } - const [siteResourceRow] = await db - .select({ - siteResourceId: siteResources.siteResourceId, - orgId: siteResources.orgId - }) - .from(siteResources) - .where( - and( - eq(siteResources.fullDomain, host), - eq(siteResources.mode, "inference"), - eq(siteResources.enabled, true) - ) - ) - .limit(1); - if (siteResourceRow) { - const attachmentRows = await db - .select({ - provider: aiProviders, - accessMode: siteResourceAiProviders.accessMode - }) - .from(siteResourceAiProviders) - .innerJoin( - aiProviders, - eq(siteResourceAiProviders.providerId, aiProviders.providerId) - ) - .where( - and( + const [attachmentRows, resourcePatterns] = await Promise.all([ + db + .select({ + provider: aiProviders, + accessMode: siteResourceAiProviders.accessMode + }) + .from(siteResourceAiProviders) + .innerJoin( + aiProviders, eq( - siteResourceAiProviders.siteResourceId, - siteResourceRow.siteResourceId - ), - eq(aiProviders.enabled, true) + siteResourceAiProviders.providerId, + aiProviders.providerId + ) ) - ); + .where( + and( + eq( + siteResourceAiProviders.siteResourceId, + siteResourceRow.siteResourceId + ), + eq(aiProviders.enabled, true) + ) + ), + db + .select({ + providerId: aiModels.providerId, + modelKey: aiModels.modelKey, + listType: siteResourceAiModels.listType, + enabled: aiModels.enabled + }) + .from(siteResourceAiModels) + .innerJoin( + aiModels, + eq(siteResourceAiModels.modelId, aiModels.modelId) + ) + .where( + eq( + siteResourceAiModels.siteResourceId, + siteResourceRow.siteResourceId + ) + ) + ]); if (attachmentRows.length === 0) { return null; } - const attachments: ProviderAttachment[] = attachmentRows.map((a) => ({ - provider: a.provider, - accessMode: a.accessMode - })); - - const resourcePatterns = await db - .select({ - providerId: aiModels.providerId, - modelKey: aiModels.modelKey, - listType: siteResourceAiModels.listType, - enabled: aiModels.enabled - }) - .from(siteResourceAiModels) - .innerJoin( - aiModels, - eq(siteResourceAiModels.modelId, aiModels.modelId) - ) - .where( - eq( - siteResourceAiModels.siteResourceId, - siteResourceRow.siteResourceId - ) - ); - return { resourceId: null, siteResourceId: siteResourceRow.siteResourceId, orgId: siteResourceRow.orgId, - attachments, + attachments: attachmentRows.map((a) => ({ + provider: a.provider, + accessMode: a.accessMode + })), resourceListsByProvider: groupPatternsByProvider(resourcePatterns) }; } @@ -587,13 +596,6 @@ export async function handleAiGatewayProxy( const { attachments, resourceListsByProvider, resourceId, orgId } = target; - const requestUser = await resolveRequestUser(req, resourceId, orgId); - if (requestUser) { - logger.debug( - `AI gateway request from user ${requestUser.userId} (${requestUser.username})` - ); - } - const capableAttachments = attachments.filter((a) => providerHasCapability(a.provider.capabilities, capability) ); @@ -608,11 +610,21 @@ export async function handleAiGatewayProxy( const requestedModel = def.extractModel(req); - const selection = await selectProvider( - capableAttachments, - resourceListsByProvider, - requestedModel - ); + const [requestUser, selection] = await Promise.all([ + resolveRequestUser(req, resourceId, orgId), + selectProvider( + capableAttachments, + resourceListsByProvider, + requestedModel + ) + ]); + + if (requestUser) { + logger.debug( + `AI gateway request from user ${requestUser.userId} (${requestUser.username})` + ); + } + if (!selection.ok) { return res.status(selection.status).json({ error: { message: selection.message } From 5e5e172d3959a279c5662441abd5763e54b3f199 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 7 Aug 2026 14:36:15 -0400 Subject: [PATCH 081/296] add streaming function to capability --- server/lib/aiCapabilities.ts | 68 ++++++++++++++++++----- server/routers/aiGateway/pipeline.ts | 20 +------ server/routers/aiGateway/targetRouting.ts | 21 +++---- 3 files changed, 68 insertions(+), 41 deletions(-) diff --git a/server/lib/aiCapabilities.ts b/server/lib/aiCapabilities.ts index 2b3691493..89fbd3d75 100644 --- a/server/lib/aiCapabilities.ts +++ b/server/lib/aiCapabilities.ts @@ -27,6 +27,7 @@ export type AiCapabilityDefinition = { req: Request, model: string ) => string; + isStreaming: (req: Request, contentType: string) => boolean; }; function bodyModel(req: Request): string | undefined { @@ -38,9 +39,6 @@ function paramModel(req: Request): string | undefined { return typeof model === "string" && model.length > 0 ? model : undefined; } -/** - * Join a provider base URL with an inbound request path. - */ export function joinUpstreamUrl(baseUrl: string, path: string): string { const base = baseUrl.replace(/\/+$/, ""); let suffix = path.startsWith("/") ? path : `/${path}`; @@ -80,13 +78,38 @@ export function joinUpstreamUrl(baseUrl: string, path: string): string { } function pathFromRequest(req: Request): string { - // Prefer originalUrl (includes mounted path) over req.url when available. - // Query string is preserved - some providers use it to select the - // streaming response format (e.g. Gemini's `?alt=sse`). const raw = req.originalUrl || req.url || req.path; return raw.startsWith("/") ? raw : `/${raw}`; } +function bodyRequestsStream(req: Request): boolean { + return req.body?.stream === true; +} + +function contentTypeIsSse(contentType: string): boolean { + return contentType.includes("text/event-stream"); +} + +function contentTypeIsAmazonEventStream(contentType: string): boolean { + return contentType.includes("application/vnd.amazon.eventstream"); +} + +function pathIncludes(req: Request, fragment: string): boolean { + return pathFromRequest(req).includes(fragment); +} + +function isBodyOrSseStreaming(req: Request, contentType: string): boolean { + return bodyRequestsStream(req) || contentTypeIsSse(contentType); +} + +function isGeminiStyleStreaming(req: Request, contentType: string): boolean { + return ( + pathIncludes(req, "streamGenerateContent") || + pathIncludes(req, "alt=sse") || + contentTypeIsSse(contentType) + ); +} + export const AI_CAPABILITY_DEFS: Record = { openai_chat: { @@ -97,21 +120,24 @@ export const AI_CAPABILITY_DEFS: Record = ], extractModel: bodyModel, resolveUpstreamUrl: (base, req) => - joinUpstreamUrl(base, pathFromRequest(req)) + joinUpstreamUrl(base, pathFromRequest(req)), + isStreaming: isBodyOrSseStreaming }, openai_responses: { id: "openai_responses", routes: [{ method: "POST", path: "/v1/responses" }], extractModel: bodyModel, resolveUpstreamUrl: (base, req) => - joinUpstreamUrl(base, pathFromRequest(req)) + joinUpstreamUrl(base, pathFromRequest(req)), + isStreaming: isBodyOrSseStreaming }, anthropic_messages: { id: "anthropic_messages", routes: [{ method: "POST", path: "/v1/messages" }], extractModel: bodyModel, resolveUpstreamUrl: (base, req) => - joinUpstreamUrl(base, pathFromRequest(req)) + joinUpstreamUrl(base, pathFromRequest(req)), + isStreaming: isBodyOrSseStreaming }, gemini_generate_content: { id: "gemini_generate_content", @@ -127,7 +153,8 @@ export const AI_CAPABILITY_DEFS: Record = ], extractModel: paramModel, resolveUpstreamUrl: (base, req) => - joinUpstreamUrl(base, pathFromRequest(req)) + joinUpstreamUrl(base, pathFromRequest(req)), + isStreaming: isGeminiStyleStreaming }, google_generate_content: { id: "google_generate_content", @@ -144,7 +171,8 @@ export const AI_CAPABILITY_DEFS: Record = ], extractModel: paramModel, resolveUpstreamUrl: (base, req) => - joinUpstreamUrl(base, pathFromRequest(req)) + joinUpstreamUrl(base, pathFromRequest(req)), + isStreaming: isGeminiStyleStreaming }, google_raw_predict: { id: "google_raw_predict", @@ -160,7 +188,11 @@ export const AI_CAPABILITY_DEFS: Record = ], extractModel: paramModel, resolveUpstreamUrl: (base, req) => - joinUpstreamUrl(base, pathFromRequest(req)) + joinUpstreamUrl(base, pathFromRequest(req)), + isStreaming: (req, contentType) => + pathIncludes(req, "streamRawPredict") || + pathIncludes(req, "alt=sse") || + contentTypeIsSse(contentType) }, bedrock_model_invoke: { id: "bedrock_model_invoke", @@ -173,7 +205,11 @@ export const AI_CAPABILITY_DEFS: Record = ], extractModel: paramModel, resolveUpstreamUrl: (base, req) => - joinUpstreamUrl(base, pathFromRequest(req)) + joinUpstreamUrl(base, pathFromRequest(req)), + isStreaming: (req, contentType) => + pathIncludes(req, "invoke-with-response-stream") || + contentTypeIsAmazonEventStream(contentType) || + contentTypeIsSse(contentType) }, bedrock_converse: { id: "bedrock_converse", @@ -183,7 +219,11 @@ export const AI_CAPABILITY_DEFS: Record = ], extractModel: paramModel, resolveUpstreamUrl: (base, req) => - joinUpstreamUrl(base, pathFromRequest(req)) + joinUpstreamUrl(base, pathFromRequest(req)), + isStreaming: (req, contentType) => + pathIncludes(req, "converse-stream") || + contentTypeIsAmazonEventStream(contentType) || + contentTypeIsSse(contentType) } }; diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 724102f07..e84165620 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -62,11 +62,6 @@ import { type AiUsage } from "@server/lib/aiUsageExtraction"; -// Short-lived local caches so a burst of requests from the same IP/user -// doesn't hit the database on every single request. None of this is -// security-critical to cache aggressively (identity is re-derived from the -// session cookie or from a client's exit-node-scoped subnet each time), so -// a small TTL is just an efficiency win, not a trust boundary. const EXIT_NODE_RANGES_CACHE_KEY = "aiGateway:exitNodeRanges"; const EXIT_NODE_RANGES_TTL_SEC = 6000; const CLIENT_BY_IP_TTL_SEC = 30; @@ -638,7 +633,8 @@ export async function handleAiGatewayProxy( req, res, provider, - requestUser + requestUser, + capability ); } @@ -724,10 +720,6 @@ export async function handleAiGatewayProxy( skipTlsVerification: provider.skipTlsVerification }); - // Cancel the upstream request (and, transitively, anything it fans - // out to) if the client goes away before we're done - otherwise a - // client-cancelled streaming chat completion keeps running upstream - // to completion, wasting the connection and any per-token billing. const abortController = new AbortController(); const onClientClose = () => { if (!res.writableEnded) { @@ -764,13 +756,7 @@ export async function handleAiGatewayProxy( } const contentType = upstreamRes.headers.get("content-type") || ""; - const isStream = - req.body?.stream === true || - contentType.includes("text/event-stream") || - req.path.includes("streamGenerateContent") || - req.path.includes("streamRawPredict") || - req.path.includes("converse-stream") || - req.path.includes("invoke-with-response-stream"); + const isStream = def.isStreaming(req, contentType); res.status(upstreamRes.status); res.setHeader("Content-Type", contentType || "application/json"); diff --git a/server/routers/aiGateway/targetRouting.ts b/server/routers/aiGateway/targetRouting.ts index b091bf8f5..a6d08d925 100644 --- a/server/routers/aiGateway/targetRouting.ts +++ b/server/routers/aiGateway/targetRouting.ts @@ -10,6 +10,10 @@ import { applyAiProviderCustomHeaders, authTypeRequiresApiKey } from "@server/lib/aiProviderDefaults"; +import { + AI_CAPABILITY_DEFS, + type AiCapability +} from "@server/lib/aiCapabilities"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; import { @@ -152,14 +156,14 @@ export async function proxyAiGatewayToSiteTarget( req: Request, res: Response, provider: AiProvider, - requestUser: RequestUser | null + requestUser: RequestUser | null, + capability: AiCapability ): Promise { const providerTargets = await getProviderTargets(provider.providerId); if (providerTargets.length === 0) { res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ error: { - message: - "AI provider has no reachable site targets configured" + message: "AI provider has no reachable site targets configured" } }); return; @@ -256,13 +260,10 @@ export async function proxyAiGatewayToSiteTarget( } const contentType = upstreamRes.headers.get("content-type") || ""; - const isStream = - req.body?.stream === true || - contentType.includes("text/event-stream") || - pathFromRequest(req).includes("streamGenerateContent") || - pathFromRequest(req).includes("streamRawPredict") || - pathFromRequest(req).includes("converse-stream") || - pathFromRequest(req).includes("invoke-with-response-stream"); + const isStream = AI_CAPABILITY_DEFS[capability].isStreaming( + req, + contentType + ); res.status(upstreamRes.status); res.setHeader("Content-Type", contentType || "application/json"); From 3f305e4d5cb7b30bbeb386584e3040239f31c05f Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 7 Aug 2026 20:52:25 +0200 Subject: [PATCH 082/296] =?UTF-8?q?=F0=9F=9A=A7=20process=20test=20alert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../private/lib/alerts/processTestAlerts.ts | 22 +++++++++ server/private/routers/alertRule/index.ts | 2 +- ...{testSiteAlertRule.ts => testAlertRule.ts} | 48 ++++++++++++++++--- server/private/routers/external.ts | 6 +-- server/routers/alertRule/types.ts | 22 +++++++++ 5 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 server/private/lib/alerts/processTestAlerts.ts rename server/private/routers/alertRule/{testSiteAlertRule.ts => testAlertRule.ts} (60%) diff --git a/server/private/lib/alerts/processTestAlerts.ts b/server/private/lib/alerts/processTestAlerts.ts new file mode 100644 index 000000000..acbd6fa5e --- /dev/null +++ b/server/private/lib/alerts/processTestAlerts.ts @@ -0,0 +1,22 @@ +import logger from "@server/logger"; +import type { TestAlertContext } from "@server/routers/alertRule/types"; +import { sendAlertEmail } from "./sendAlertEmail"; + +export async function processTestAlerts(context: TestAlertContext) { + const emailActions = context.actions.filter( + (action) => action.type === "email" + ); + // Process email actions + for (const action of emailActions) { + try { + const recipients = await resolveEmailRecipients( + action.emailActionId + ); + if (recipients.length > 0) { + await sendAlertEmail(recipients, context); + } + } catch (err) { + logger.error(`processAlerts: failed to send alert email`, err); + } + } +} diff --git a/server/private/routers/alertRule/index.ts b/server/private/routers/alertRule/index.ts index e80a9ba16..762f707f8 100644 --- a/server/private/routers/alertRule/index.ts +++ b/server/private/routers/alertRule/index.ts @@ -16,4 +16,4 @@ export * from "./updateAlertRule"; export * from "./deleteAlertRule"; export * from "./listAlertRules"; export * from "./getAlertRule"; -export * from "./testSiteAlertRule"; +export * from "./testAlertRule"; diff --git a/server/private/routers/alertRule/testSiteAlertRule.ts b/server/private/routers/alertRule/testAlertRule.ts similarity index 60% rename from server/private/routers/alertRule/testSiteAlertRule.ts rename to server/private/routers/alertRule/testAlertRule.ts index 149ac9313..39a5c28a1 100644 --- a/server/private/routers/alertRule/testSiteAlertRule.ts +++ b/server/private/routers/alertRule/testAlertRule.ts @@ -33,11 +33,44 @@ const paramsSchema = z.strictObject({ orgId: z.string().nonempty() }); -const querySchema = z.strictObject({ - event: z.enum(["site_offline", "site_online", "site_toggle"]) +export const SITE_EVENT_TYPES = [ + "site_online", + "site_offline", + "site_toggle" +] as const; +export const HC_EVENT_TYPES = [ + "health_check_healthy", + "health_check_unhealthy", + "health_check_toggle" +] as const; +export const RESOURCE_EVENT_TYPES = [ + "resource_healthy", + "resource_unhealthy", + "resource_degraded", + "resource_toggle" +] as const; + +const webhookActionSchema = z.strictObject({ + webhookUrl: z.string().url(), + config: z.string().optional(), + enabled: z.boolean().optional().default(true) }); -export async function testSiteAlertRule( +const bodySchema = z.strictObject({ + eventType: z.enum([ + ...HC_EVENT_TYPES, + ...SITE_EVENT_TYPES, + ...RESOURCE_EVENT_TYPES + ]), + // Email recipients (flat) + userIds: z.array(z.string().nonempty()).optional().default([]), + roleIds: z.array(z.number()).optional().default([]), + emails: z.array(z.email()).optional().default([]), + // Webhook actions + webhookActions: z.array(webhookActionSchema).optional().default([]) +}); + +export async function testAlertRule( req: Request, res: Response, next: NextFunction @@ -54,16 +87,17 @@ export async function testSiteAlertRule( } const { orgId } = parsedParams.data; - const parsedQuery = querySchema.safeParse(req.query); - if (!parsedQuery.success) { + const parsedBody = bodySchema.safeParse(req.body); + if (!parsedBody.success) { return next( createHttpError( HttpCode.BAD_REQUEST, - fromError(parsedQuery.error).toString() + fromError(parsedBody.error).toString() ) ); } - const { event } = parsedQuery.data; + + // TODO: process alert rule } catch (error) { logger.error(error); return next( diff --git a/server/private/routers/external.ts b/server/private/routers/external.ts index c94bce1df..0fd4cc023 100644 --- a/server/private/routers/external.ts +++ b/server/private/routers/external.ts @@ -808,12 +808,12 @@ authenticated.get( alertRule.listAlertRules ); -authenticated.get( - "/org/:orgId/test-site-alert-rule/:alertRuleId", +authenticated.post( + "/org/:orgId/alert-rule/test", verifyValidLicense, verifyOrgAccess, verifyUserHasAction(ActionsEnum.testAlertRule), - alertRule.testSiteAlertRule + alertRule.testAlertRule ); authenticated.get( diff --git a/server/routers/alertRule/types.ts b/server/routers/alertRule/types.ts index ebffd3c5b..90c4e3163 100644 --- a/server/routers/alertRule/types.ts +++ b/server/routers/alertRule/types.ts @@ -124,3 +124,25 @@ export interface AlertContext { /** Human-readable context data included in emails and webhook payloads */ data: Record; } + +type EmailAlertAction = { + type: "email"; + userIds?: string[]; + roleIds?: string[]; + emails?: string[]; +}; + +type WebhookAlertAction = { + type: "webhook"; + webhookUrl: string; + enabled: boolean; + config?: string | undefined; +}; + +type AlertAction = EmailAlertAction | WebhookAlertAction; +export interface TestAlertContext { + eventType: AlertEventType; + actions: AlertAction[]; + /** Human-readable context data included in emails and webhook payloads */ + data: Record; +} From 403b8a12e4c6f5205d32b10f77d5269de109898b Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 7 Aug 2026 20:57:54 +0200 Subject: [PATCH 083/296] =?UTF-8?q?=F0=9F=9A=A7=20wip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../private/lib/alerts/processTestAlerts.ts | 59 +++++++++++++++++-- server/routers/alertRule/types.ts | 5 +- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/server/private/lib/alerts/processTestAlerts.ts b/server/private/lib/alerts/processTestAlerts.ts index acbd6fa5e..f598dc633 100644 --- a/server/private/lib/alerts/processTestAlerts.ts +++ b/server/private/lib/alerts/processTestAlerts.ts @@ -1,6 +1,11 @@ import logger from "@server/logger"; -import type { TestAlertContext } from "@server/routers/alertRule/types"; +import type { + EmailAlertAction, + TestAlertContext +} from "@server/routers/alertRule/types"; import { sendAlertEmail } from "./sendAlertEmail"; +import type { db, alertEmailRecipients, users, userOrgRoles } from "@server/db"; +import type { eq } from "drizzle-orm"; export async function processTestAlerts(context: TestAlertContext) { const emailActions = context.actions.filter( @@ -9,9 +14,7 @@ export async function processTestAlerts(context: TestAlertContext) { // Process email actions for (const action of emailActions) { try { - const recipients = await resolveEmailRecipients( - action.emailActionId - ); + const recipients = await resolveEmailRecipients(action); if (recipients.length > 0) { await sendAlertEmail(recipients, context); } @@ -20,3 +23,51 @@ export async function processTestAlerts(context: TestAlertContext) { } } } + +/** + * Resolves all email addresses for a given `emailActionId`. + * + * Recipients may be: + * - Direct users (by `userId`) + * - All users in a role (by `roleId`, resolved via `userOrgRoles`) + * - Direct external email addresses + */ +async function resolveEmailRecipients( + action: EmailAlertAction +): Promise { + const emailSet = new Set(); + + // for (const row of rows) { + // if (row.email) { + // emailSet.add(row.email); + // } + + // if (row.userId) { + // const [user] = await db + // .select({ email: users.email }) + // .from(users) + // .where(eq(users.userId, row.userId)) + // .limit(1); + // if (user?.email) { + // emailSet.add(user.email); + // } + // } + + // if (row.roleId) { + // // Find all users with this role via userOrgRoles + // const roleUsers = await db + // .select({ email: users.email }) + // .from(userOrgRoles) + // .innerJoin(users, eq(userOrgRoles.userId, users.userId)) + // .where(eq(userOrgRoles.roleId, Number(row.roleId))); + + // for (const u of roleUsers) { + // if (u.email) { + // emailSet.add(u.email); + // } + // } + // } + // } + + return Array.from(emailSet); +} diff --git a/server/routers/alertRule/types.ts b/server/routers/alertRule/types.ts index 90c4e3163..99057b312 100644 --- a/server/routers/alertRule/types.ts +++ b/server/routers/alertRule/types.ts @@ -125,14 +125,14 @@ export interface AlertContext { data: Record; } -type EmailAlertAction = { +export type EmailAlertAction = { type: "email"; userIds?: string[]; roleIds?: string[]; emails?: string[]; }; -type WebhookAlertAction = { +export type WebhookAlertAction = { type: "webhook"; webhookUrl: string; enabled: boolean; @@ -143,6 +143,7 @@ type AlertAction = EmailAlertAction | WebhookAlertAction; export interface TestAlertContext { eventType: AlertEventType; actions: AlertAction[]; + orgId: string; /** Human-readable context data included in emails and webhook payloads */ data: Record; } From 9eafa067b9db4abf1f55f2150ebee146a3d735b5 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 7 Aug 2026 16:13:51 -0400 Subject: [PATCH 084/296] improved provider picker ui on resource --- messages/en-US.json | 19 +- server/db/pg/schema/schema.ts | 6 +- server/db/sqlite/schema/schema.ts | 6 +- server/lib/aiInferenceResource.ts | 51 +- server/routers/aiGateway/pipeline.ts | 17 +- .../resource/addAiProviderToResource.ts | 9 +- server/routers/resource/createResource.ts | 3 +- .../resource/removeAiProviderFromResource.ts | 3 +- .../addAiProviderToSiteResource.ts | 9 +- .../siteResource/createSiteResource.ts | 3 +- .../removeAiProviderFromSiteResource.ts | 3 +- .../private/[niceId]/inference/page.tsx | 139 +++-- .../resources/private/create/page.tsx | 5 - .../public/[niceId]/inference/page.tsx | 140 +++-- .../settings/resources/public/create/page.tsx | 5 - src/components/AiProviderAttachments.tsx | 581 ++++++++++++++++++ src/lib/queries.ts | 24 + 17 files changed, 874 insertions(+), 149 deletions(-) create mode 100644 src/components/AiProviderAttachments.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 2d77bb365..f18f99133 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1772,11 +1772,28 @@ "aiProviderModelsErrorUpdate": "Failed to update models", "aiResourceProviders": "Providers", "aiResourceProvidersDescription": "Choose which AI providers this inference resource can use", - "aiResourceProvidersHelp": "Each attached provider uses its own allow and block lists. Allow patterns that conflict across attached providers are not allowed.", + "aiResourceProvidersHelp": "Attach providers and choose inherit (use each provider's lists) or select (pick an allow list for this resource). Allow patterns that conflict across attached providers are not allowed.", "aiResourceProvidersSelect": "Select providers", "aiResourceProvidersEmpty": "No AI providers found", + "aiResourceProvidersNoneAttached": "No providers attached yet.", + "aiResourceProvidersAdd": "Add provider", + "aiResourceProvidersRemove": "Remove provider", + "aiResourceProviderToggleEnabled": "Enable or disable this provider on the resource", + "aiResourceProviderDisabled": "Disabled", "aiResourceProvidersUpdated": "Providers updated", "aiResourceProvidersErrorUpdate": "Failed to update providers", + "aiResourceProviderEditDescription": "Choose how this provider's models are exposed on this resource.", + "aiResourceProviderMode": "Access mode", + "aiResourceProviderModeInherit": "Inherit", + "aiResourceProviderModeSelect": "Select", + "aiResourceProviderModeSelectSummary": "Select · {count} models", + "aiResourceProviderModeInheritHelp": "Use this provider's allow and block lists as configured on the provider.", + "aiResourceProviderModeSelectHelp": "Choose a subset of this provider's allow-list models for this resource.", + "aiResourceProviderAllowModels": "Allow list", + "aiResourceProviderAllowModelsSelect": "Select models", + "aiResourceProviderAllowModelsSearch": "Search models...", + "aiResourceProviderAllowModelsEmpty": "No models found", + "aiResourceProviderAllowModelsHelp": "Only models from this provider's allow list can be selected.", "aiResourceAliasRequired": "Alias is required for inference resources", "aiResourceDomainConfiguration": "Domain configuration", "aiResourceDomainConfigurationDescription": "Choose the domain clients will use to reach this inference resource.", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 90dd5abd9..1c3d21b10 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -234,7 +234,8 @@ export const resourceAiProviders = pgTable( accessMode: varchar("accessMode") .$type<"inherit" | "select">() .notNull() - .default("inherit") + .default("inherit"), + enabled: boolean("enabled").notNull().default(true) }, (t) => [primaryKey({ columns: [t.resourceId, t.providerId] })] ); @@ -529,7 +530,8 @@ export const siteResourceAiProviders = pgTable( accessMode: varchar("accessMode") .$type<"inherit" | "select">() .notNull() - .default("inherit") + .default("inherit"), + enabled: boolean("enabled").notNull().default(true) }, (t) => [primaryKey({ columns: [t.siteResourceId, t.providerId] })] ); diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index dab445282..71fd197e7 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -231,7 +231,8 @@ export const resourceAiProviders = sqliteTable( accessMode: text("accessMode") .$type<"inherit" | "select">() .notNull() - .default("inherit") + .default("inherit"), + enabled: integer("enabled", { mode: "boolean" }).notNull().default(true) }, (t) => [primaryKey({ columns: [t.resourceId, t.providerId] })] ); @@ -514,7 +515,8 @@ export const siteResourceAiProviders = sqliteTable( accessMode: text("accessMode") .$type<"inherit" | "select">() .notNull() - .default("inherit") + .default("inherit"), + enabled: integer("enabled", { mode: "boolean" }).notNull().default(true) }, (t) => [primaryKey({ columns: [t.siteResourceId, t.providerId] })] ); diff --git a/server/lib/aiInferenceResource.ts b/server/lib/aiInferenceResource.ts index 51333473a..92a438ac2 100644 --- a/server/lib/aiInferenceResource.ts +++ b/server/lib/aiInferenceResource.ts @@ -24,7 +24,8 @@ export type AccessMode = z.infer; export const resourceAiProviderAttachmentSchema = z.strictObject({ providerId: z.number().int().positive(), - accessMode: accessModeSchema.optional().default("inherit") + accessMode: accessModeSchema.optional().default("inherit"), + enabled: z.boolean().optional().default(true) }); export type ResourceAiProviderInput = z.infer< @@ -34,6 +35,7 @@ export type ResourceAiProviderInput = z.infer< export type ResourceAiProviderAttachment = { providerId: number; accessMode: AccessMode; + enabled: boolean; }; export const resourceAiModelEntrySchema = z.strictObject({ @@ -79,14 +81,23 @@ export function resolveEffectiveLists(input: { function normalizeAttachments( inputs: ResourceAiProviderInput[] ): ResourceAiProviderAttachment[] { - const byProviderId = new Map(); + const byProviderId = new Map< + number, + { accessMode: AccessMode; enabled: boolean } + >(); for (const input of inputs) { - byProviderId.set(input.providerId, input.accessMode ?? "inherit"); + byProviderId.set(input.providerId, { + accessMode: input.accessMode ?? "inherit", + enabled: input.enabled ?? true + }); } - return [...byProviderId.entries()].map(([providerId, accessMode]) => ({ - providerId, - accessMode - })); + return [...byProviderId.entries()].map( + ([providerId, { accessMode, enabled }]) => ({ + providerId, + accessMode, + enabled + }) + ); } type EffectiveAllowRow = { @@ -110,14 +121,16 @@ export async function assertNoOverlappingModelKeys( ): Promise { const trx = options.trx ?? db; - if (attachments.length < 2) { + const activeAttachments = attachments.filter((a) => a.enabled); + + if (activeAttachments.length < 2) { return null; } - const inheritProviderIds = attachments + const inheritProviderIds = activeAttachments .filter((a) => a.accessMode === "inherit") .map((a) => a.providerId); - const selectProviderIds = attachments + const selectProviderIds = activeAttachments .filter((a) => a.accessMode === "select") .map((a) => a.providerId); @@ -318,7 +331,8 @@ export async function setPublicResourceAiProviders( attachments.map((a) => ({ resourceId, providerId: a.providerId, - accessMode: a.accessMode + accessMode: a.accessMode, + enabled: a.enabled })) ); } @@ -344,7 +358,8 @@ export async function setSiteResourceAiProviders( attachments.map((a) => ({ siteResourceId, providerId: a.providerId, - accessMode: a.accessMode + accessMode: a.accessMode, + enabled: a.enabled })) ); } @@ -473,7 +488,8 @@ export async function listPublicResourceAiProviders(resourceId: number) { providerId: resourceAiProviders.providerId, name: aiProviders.name, type: aiProviders.type, - enabled: aiProviders.enabled, + enabled: resourceAiProviders.enabled, + providerEnabled: aiProviders.enabled, accessMode: resourceAiProviders.accessMode }) .from(resourceAiProviders) @@ -490,7 +506,8 @@ export async function listSiteResourceAiProviders(siteResourceId: number) { providerId: siteResourceAiProviders.providerId, name: aiProviders.name, type: aiProviders.type, - enabled: aiProviders.enabled, + enabled: siteResourceAiProviders.enabled, + providerEnabled: aiProviders.enabled, accessMode: siteResourceAiProviders.accessMode }) .from(siteResourceAiProviders) @@ -575,7 +592,8 @@ export async function assertPublicResourceModelEntriesValid(input: { const attachments = await db .select({ providerId: resourceAiProviders.providerId, - accessMode: resourceAiProviders.accessMode + accessMode: resourceAiProviders.accessMode, + enabled: resourceAiProviders.enabled }) .from(resourceAiProviders) .innerJoin( @@ -610,7 +628,8 @@ export async function assertSiteResourceModelEntriesValid(input: { const attachments = await db .select({ providerId: siteResourceAiProviders.providerId, - accessMode: siteResourceAiProviders.accessMode + accessMode: siteResourceAiProviders.accessMode, + enabled: siteResourceAiProviders.enabled }) .from(siteResourceAiProviders) .innerJoin( diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index e84165620..bcff1a501 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -287,7 +287,8 @@ async function resolveTarget(host: string): Promise { resourceAiProviders.resourceId, resourceRow.resourceId ), - eq(aiProviders.enabled, true) + eq(aiProviders.enabled, true), + eq(resourceAiProviders.enabled, true) ) ), db @@ -342,7 +343,8 @@ async function resolveTarget(host: string): Promise { siteResourceAiProviders.siteResourceId, siteResourceRow.siteResourceId ), - eq(aiProviders.enabled, true) + eq(aiProviders.enabled, true), + eq(siteResourceAiProviders.enabled, true) ) ), db @@ -525,8 +527,15 @@ function logAiUsageAndCost(args: { isStream: boolean; headers: Headers; }): void { - const { capability, provider, requestedModel, requestBody, responseText, isStream, headers } = - args; + const { + capability, + provider, + requestedModel, + requestBody, + responseText, + isStream, + headers + } = args; let usage: AiUsage | null = extractUsage( capability, diff --git a/server/routers/resource/addAiProviderToResource.ts b/server/routers/resource/addAiProviderToResource.ts index bcc175c18..f2477d89d 100644 --- a/server/routers/resource/addAiProviderToResource.ts +++ b/server/routers/resource/addAiProviderToResource.ts @@ -118,9 +118,14 @@ export async function addAiProviderToResource( .filter((a) => a.providerId !== providerId) .map((a) => ({ providerId: a.providerId, - accessMode: a.accessMode + accessMode: a.accessMode, + enabled: a.enabled })), - { providerId, accessMode: "inherit" as const } + { + providerId, + accessMode: "inherit" as const, + enabled: true as const + } ]; const attachments = await resolveProviderAttachments({ diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index e225458d1..45e66c2c3 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -397,7 +397,8 @@ async function createHttpResource( orgId, attachments: (aiProviderInputs ?? []).map((p) => ({ providerId: p.providerId, - accessMode: "inherit" as const + accessMode: "inherit" as const, + enabled: true as const })), requireAtLeastOne: false }); diff --git a/server/routers/resource/removeAiProviderFromResource.ts b/server/routers/resource/removeAiProviderFromResource.ts index a7f9298ee..5b99ff089 100644 --- a/server/routers/resource/removeAiProviderFromResource.ts +++ b/server/routers/resource/removeAiProviderFromResource.ts @@ -127,7 +127,8 @@ export async function removeAiProviderFromResource( .filter((a) => a.providerId !== providerId) .map((a) => ({ providerId: a.providerId, - accessMode: a.accessMode + accessMode: a.accessMode, + enabled: a.enabled })); const attachments = await resolveProviderAttachments({ diff --git a/server/routers/siteResource/addAiProviderToSiteResource.ts b/server/routers/siteResource/addAiProviderToSiteResource.ts index 79a0ae093..18e8cd74d 100644 --- a/server/routers/siteResource/addAiProviderToSiteResource.ts +++ b/server/routers/siteResource/addAiProviderToSiteResource.ts @@ -118,9 +118,14 @@ export async function addAiProviderToSiteResource( .filter((a) => a.providerId !== providerId) .map((a) => ({ providerId: a.providerId, - accessMode: a.accessMode + accessMode: a.accessMode, + enabled: a.enabled })), - { providerId, accessMode: "inherit" as const } + { + providerId, + accessMode: "inherit" as const, + enabled: true as const + } ]; const attachments = await resolveProviderAttachments({ diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 15b82cb56..2038f9ce7 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -356,7 +356,8 @@ export async function createSiteResource( orgId, attachments: (aiProviderInputs ?? []).map((p) => ({ providerId: p.providerId, - accessMode: "inherit" as const + accessMode: "inherit" as const, + enabled: true as const })), requireAtLeastOne: false }); diff --git a/server/routers/siteResource/removeAiProviderFromSiteResource.ts b/server/routers/siteResource/removeAiProviderFromSiteResource.ts index ffb4f1ff8..29facaece 100644 --- a/server/routers/siteResource/removeAiProviderFromSiteResource.ts +++ b/server/routers/siteResource/removeAiProviderFromSiteResource.ts @@ -126,7 +126,8 @@ export async function removeAiProviderFromSiteResource( .filter((a) => a.providerId !== providerId) .map((a) => ({ providerId: a.providerId, - accessMode: a.accessMode + accessMode: a.accessMode, + enabled: a.enabled })); const attachments = await resolveProviderAttachments({ diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx index f9ad664a0..e824cc105 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/inference/page.tsx @@ -16,9 +16,9 @@ import { SettingsSubsectionTitle } from "@app/components/Settings"; import { - AiProvidersSelector, - type SelectedAiProvider -} from "@app/components/AiProvidersSelector"; + AiProviderAttachments, + type AiProviderAttachmentValue +} from "@app/components/AiProviderAttachments"; import DomainPicker from "@app/components/DomainPicker"; import { SwitchInput } from "@app/components/SwitchInput"; import { Button } from "@app/components/ui/button"; @@ -41,7 +41,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useTranslations } from "next-intl"; import { useRouter } from "next/navigation"; -import { useActionState, useEffect, useMemo, useState } from "react"; +import { useActionState, useEffect, useMemo } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod"; @@ -65,7 +65,15 @@ export default function PrivateResourceInferencePage() { const formSchema = useMemo( () => z.object({ - providerIds: z.array(z.number().int().positive()), + providers: z.array( + z.object({ + providerId: z.number().int().positive(), + name: z.string(), + accessMode: z.enum(["inherit", "select"]), + enabled: z.boolean(), + selectedModelIds: z.array(z.number().int().positive()) + }) + ), httpConfigSubdomain: z.string().nullish(), httpConfigDomainId: z.string().nullish(), httpConfigFullDomain: z.string().nullish(), @@ -75,10 +83,6 @@ export default function PrivateResourceInferencePage() { ); type FormValues = z.infer; - const [selectedProviders, setSelectedProviders] = useState< - SelectedAiProvider[] - >([]); - const attachedQuery = useQuery({ ...resourceQueries.siteResourceAiProviders({ siteResourceId: siteResource.id @@ -86,10 +90,17 @@ export default function PrivateResourceInferencePage() { enabled: siteResource.mode === "inference" }); + const modelsQuery = useQuery({ + ...resourceQueries.siteResourceAiModels({ + siteResourceId: siteResource.id + }), + enabled: siteResource.mode === "inference" + }); + const form = useForm({ resolver: zodResolver(formSchema), defaultValues: { - providerIds: [], + providers: [], httpConfigSubdomain: siteResource.subdomain ?? null, httpConfigDomainId: siteResource.domainId ?? null, httpConfigFullDomain: siteResource.fullDomain ?? null, @@ -103,16 +114,33 @@ export default function PrivateResourceInferencePage() { useEffect(() => { if (!attachedQuery.data) return; - const providers = attachedQuery.data.map((provider) => ({ - id: String(provider.providerId), - text: provider.name - })); - setSelectedProviders(providers); - form.setValue( - "providerIds", - attachedQuery.data.map((p) => p.providerId) + const hasSelect = attachedQuery.data.some( + (provider) => provider.accessMode === "select" ); - }, [attachedQuery.data, form]); + if (hasSelect && modelsQuery.isLoading) return; + + const modelsByProvider = new Map(); + for (const model of modelsQuery.data ?? []) { + if (model.listType !== "allow") continue; + const existing = modelsByProvider.get(model.providerId) ?? []; + existing.push(model.modelId); + modelsByProvider.set(model.providerId, existing); + } + + form.setValue( + "providers", + attachedQuery.data.map((provider) => ({ + providerId: provider.providerId, + name: provider.name, + accessMode: provider.accessMode, + enabled: provider.enabled, + selectedModelIds: + provider.accessMode === "select" + ? (modelsByProvider.get(provider.providerId) ?? []) + : [] + })) + ); + }, [attachedQuery.data, modelsQuery.data, modelsQuery.isLoading, form]); const [, formAction, saveLoading] = useActionState(async () => { const isValid = await form.trigger(); @@ -129,16 +157,37 @@ export default function PrivateResourceInferencePage() { }); await api.post(`/site-resource/${siteResource.id}/ai-providers`, { - providers: data.providerIds.map((providerId) => ({ - providerId + providers: data.providers.map((provider) => ({ + providerId: provider.providerId, + accessMode: provider.accessMode, + enabled: provider.enabled })) }); + const selectProviders = data.providers.filter( + (provider) => provider.accessMode === "select" + ); + if (selectProviders.length > 0) { + await api.post(`/site-resource/${siteResource.id}/ai-models`, { + models: selectProviders.flatMap((provider) => + provider.selectedModelIds.map((modelId) => ({ + modelId, + listType: "allow" as const + })) + ) + }); + } + await queryClient.invalidateQueries( resourceQueries.siteResourceAiProviders({ siteResourceId: siteResource.id }) ); + await queryClient.invalidateQueries( + resourceQueries.siteResourceAiModels({ + siteResourceId: siteResource.id + }) + ); toast({ title: t("success"), @@ -160,6 +209,11 @@ export default function PrivateResourceInferencePage() { return null; } + const providersLoading = + attachedQuery.isLoading || + (attachedQuery.data?.some((p) => p.accessMode === "select") && + modelsQuery.isLoading); + return ( @@ -183,8 +237,8 @@ export default function PrivateResourceInferencePage() { ( + name="providers" + render={({ field }) => ( {t( @@ -192,44 +246,21 @@ export default function PrivateResourceInferencePage() { )} - { - setSelectedProviders( - providers - ); - form.setValue( - "providerIds", - providers.map( - (p) => - parseInt( - p.id, - 10 - ) - ), - { - shouldValidate: true - } - ); - }} /> - - {t( - "aiResourceProvidersHelp" - )} - )} @@ -335,7 +366,7 @@ export default function PrivateResourceInferencePage() { type="submit" form="private-resource-providers-form" loading={saveLoading} - disabled={attachedQuery.isLoading} + disabled={providersLoading || saveLoading} > {t("saveSettings")} diff --git a/src/app/[orgId]/settings/resources/private/create/page.tsx b/src/app/[orgId]/settings/resources/private/create/page.tsx index da3c968b6..a1b04f5e4 100644 --- a/src/app/[orgId]/settings/resources/private/create/page.tsx +++ b/src/app/[orgId]/settings/resources/private/create/page.tsx @@ -710,11 +710,6 @@ export default function CreatePrivateResourcePage() { }} /> - - {t( - "aiResourceProvidersHelp" - )} - )} diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx index ad7230591..7de441995 100644 --- a/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx +++ b/src/app/[orgId]/settings/resources/public/[niceId]/inference/page.tsx @@ -13,9 +13,9 @@ import { SettingsSectionTitle } from "@app/components/Settings"; import { - AiProvidersSelector, - type SelectedAiProvider -} from "@app/components/AiProvidersSelector"; + AiProviderAttachments, + type AiProviderAttachmentValue +} from "@app/components/AiProviderAttachments"; import { Button } from "@app/components/ui/button"; import { Form, @@ -35,7 +35,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useTranslations } from "next-intl"; import { useRouter } from "next/navigation"; -import { useActionState, useEffect, useMemo, useState } from "react"; +import { useActionState, useEffect, useMemo } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod"; @@ -58,16 +58,20 @@ export default function PublicResourceInferencePage() { const formSchema = useMemo( () => z.object({ - providerIds: z.array(z.number().int().positive()) + providers: z.array( + z.object({ + providerId: z.number().int().positive(), + name: z.string(), + accessMode: z.enum(["inherit", "select"]), + enabled: z.boolean(), + selectedModelIds: z.array(z.number().int().positive()) + }) + ) }), [] ); type FormValues = z.infer; - const [selectedProviders, setSelectedProviders] = useState< - SelectedAiProvider[] - >([]); - const attachedQuery = useQuery({ ...resourceQueries.resourceAiProviders({ resourceId: resource.resourceId @@ -75,24 +79,53 @@ export default function PublicResourceInferencePage() { enabled: resource.mode === "inference" }); + const modelsQuery = useQuery({ + ...resourceQueries.resourceAiModels({ + resourceId: resource.resourceId + }), + enabled: resource.mode === "inference" + }); + const form = useForm({ resolver: zodResolver(formSchema), defaultValues: { - providerIds: [] + providers: [] } }); useEffect(() => { if (!attachedQuery.data) return; - const providers = attachedQuery.data.map((provider) => ({ - id: String(provider.providerId), - text: provider.name - })); - setSelectedProviders(providers); + const hasSelect = attachedQuery.data.some( + (provider) => provider.accessMode === "select" + ); + if (hasSelect && modelsQuery.isLoading) return; + + const modelsByProvider = new Map(); + for (const model of modelsQuery.data ?? []) { + if (model.listType !== "allow") continue; + const existing = modelsByProvider.get(model.providerId) ?? []; + existing.push(model.modelId); + modelsByProvider.set(model.providerId, existing); + } + form.reset({ - providerIds: attachedQuery.data.map((p) => p.providerId) + providers: attachedQuery.data.map((provider) => ({ + providerId: provider.providerId, + name: provider.name, + accessMode: provider.accessMode, + enabled: provider.enabled, + selectedModelIds: + provider.accessMode === "select" + ? (modelsByProvider.get(provider.providerId) ?? []) + : [] + })) }); - }, [attachedQuery.data, form]); + }, [ + attachedQuery.data, + modelsQuery.data, + modelsQuery.isLoading, + form + ]); const [, formAction, saveLoading] = useActionState(async () => { const isValid = await form.trigger(); @@ -101,16 +134,37 @@ export default function PublicResourceInferencePage() { const data = form.getValues(); try { await api.post(`/resource/${resource.resourceId}/ai-providers`, { - providers: data.providerIds.map((providerId) => ({ - providerId + providers: data.providers.map((provider) => ({ + providerId: provider.providerId, + accessMode: provider.accessMode, + enabled: provider.enabled })) }); + const selectProviders = data.providers.filter( + (provider) => provider.accessMode === "select" + ); + if (selectProviders.length > 0) { + await api.post(`/resource/${resource.resourceId}/ai-models`, { + models: selectProviders.flatMap((provider) => + provider.selectedModelIds.map((modelId) => ({ + modelId, + listType: "allow" as const + })) + ) + }); + } + await queryClient.invalidateQueries( resourceQueries.resourceAiProviders({ resourceId: resource.resourceId }) ); + await queryClient.invalidateQueries( + resourceQueries.resourceAiModels({ + resourceId: resource.resourceId + }) + ); toast({ title: t("success"), @@ -132,6 +186,11 @@ export default function PublicResourceInferencePage() { return null; } + const providersLoading = + attachedQuery.isLoading || + (attachedQuery.data?.some((p) => p.accessMode === "select") && + modelsQuery.isLoading); + return ( @@ -155,8 +214,8 @@ export default function PublicResourceInferencePage() { ( + name="providers" + render={({ field }) => ( {t( @@ -164,44 +223,21 @@ export default function PublicResourceInferencePage() { )} - { - setSelectedProviders( - providers - ); - form.setValue( - "providerIds", - providers.map( - (p) => - parseInt( - p.id, - 10 - ) - ), - { - shouldValidate: true - } - ); - }} /> - - {t( - "aiResourceProvidersHelp" - )} - )} @@ -218,7 +254,7 @@ export default function PublicResourceInferencePage() { type="submit" form="public-resource-providers-form" loading={saveLoading} - disabled={attachedQuery.isLoading} + disabled={providersLoading || saveLoading} > {t("saveSettings")} diff --git a/src/app/[orgId]/settings/resources/public/create/page.tsx b/src/app/[orgId]/settings/resources/public/create/page.tsx index 4d1c24efb..65529f74a 100644 --- a/src/app/[orgId]/settings/resources/public/create/page.tsx +++ b/src/app/[orgId]/settings/resources/public/create/page.tsx @@ -1437,11 +1437,6 @@ export default function Page() { ); }} /> -

- {t( - "aiResourceProvidersHelp" - )} -

diff --git a/src/components/AiProviderAttachments.tsx b/src/components/AiProviderAttachments.tsx new file mode 100644 index 000000000..09a4243e1 --- /dev/null +++ b/src/components/AiProviderAttachments.tsx @@ -0,0 +1,581 @@ +"use client"; + +import { + Credenza, + CredenzaBody, + CredenzaClose, + CredenzaContent, + CredenzaDescription, + CredenzaFooter, + CredenzaHeader, + CredenzaTitle +} from "@app/components/Credenza"; +import { type TagValue } from "@app/components/multi-select/multi-select-content"; +import { MultiSelectTagInput } from "@app/components/multi-select/multi-select-tag-input"; +import { Button } from "@app/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger +} from "@app/components/ui/dropdown-menu"; +import { Switch } from "@app/components/ui/switch"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue +} from "@app/components/ui/select"; +import { cn } from "@app/lib/cn"; +import { aiProviderQueries } from "@app/lib/queries"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useQuery } from "@tanstack/react-query"; +import { Plus, XIcon } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; + +export type AiProviderAttachmentValue = { + providerId: number; + name: string; + accessMode: "inherit" | "select"; + enabled: boolean; + selectedModelIds: number[]; +}; + +export type AiProviderAttachmentsProps = { + orgId: string; + value: AiProviderAttachmentValue[]; + onChange: (value: AiProviderAttachmentValue[]) => void; + disabled?: boolean; +}; + +export function AiProviderAttachments({ + orgId, + value, + onChange, + disabled +}: AiProviderAttachmentsProps) { + const t = useTranslations(); + const [editingProviderId, setEditingProviderId] = useState( + null + ); + + const { data: providers = [] } = useQuery( + aiProviderQueries.orgProviders({ orgId }) + ); + + const attachedIds = useMemo( + () => new Set(value.map((v) => v.providerId)), + [value] + ); + + const availableProviders = providers + .filter((provider) => provider.enabled) + .filter((provider) => !attachedIds.has(provider.providerId)); + + const editing = value.find((v) => v.providerId === editingProviderId); + + function addProvider(providerId: number, name: string) { + if (value.some((v) => v.providerId === providerId)) { + return; + } + onChange([ + ...value, + { + providerId, + name, + accessMode: "inherit", + enabled: true, + selectedModelIds: [] + } + ]); + } + + function removeProvider(providerId: number) { + onChange(value.filter((v) => v.providerId !== providerId)); + } + + function updateProvider(updated: AiProviderAttachmentValue) { + onChange( + value.map((v) => + v.providerId === updated.providerId ? updated : v + ) + ); + setEditingProviderId(null); + } + + return ( +
+ {value.length === 0 ? ( +

+ {t("aiResourceProvidersNoneAttached")} +

+ ) : ( +
+ {value.map((attachment) => ( + + setEditingProviderId(attachment.providerId) + } + onRemove={() => + removeProvider(attachment.providerId) + } + onToggleEnabled={(enabled) => { + onChange( + value.map((v) => + v.providerId === attachment.providerId + ? { ...v, enabled } + : v + ) + ); + }} + /> + ))} +
+ )} + + + + + + + {availableProviders.map((provider) => ( + + addProvider(provider.providerId, provider.name) + } + > + {provider.name} + + ))} + + + + {editing && ( + { + if (!open) setEditingProviderId(null); + }} + onSave={updateProvider} + /> + )} +
+ ); +} + +function AttachmentRow({ + attachment, + disabled, + onEdit, + onRemove, + onToggleEnabled +}: { + attachment: AiProviderAttachmentValue; + disabled?: boolean; + onEdit: () => void; + onRemove: () => void; + onToggleEnabled: (enabled: boolean) => void; +}) { + const t = useTranslations(); + const summary = + attachment.accessMode === "inherit" + ? t("aiResourceProviderModeInherit") + : t("aiResourceProviderModeSelectSummary", { + count: attachment.selectedModelIds.length + }); + + return ( +
{ + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + onEdit(); + } + } + } + role={disabled ? undefined : "button"} + tabIndex={disabled ? undefined : 0} + > +
+ + {attachment.name} + +

+ {attachment.enabled + ? summary + : t("aiResourceProviderDisabled")} +

+
+
e.stopPropagation()} + onKeyDown={(e) => e.stopPropagation()} + > + + + +
+
+ ); +} + +type EditFormValues = { + accessMode: "inherit" | "select"; + selectedModels: TagValue[]; +}; + +function EditAttachmentCredenza({ + attachment, + open, + onOpenChange, + onSave +}: { + attachment: AiProviderAttachmentValue; + open: boolean; + onOpenChange: (open: boolean) => void; + onSave: (value: AiProviderAttachmentValue) => void; +}) { + const t = useTranslations(); + const [modelSearch, setModelSearch] = useState(""); + + const editSchema = useMemo( + () => + z.object({ + accessMode: z.enum(["inherit", "select"]), + selectedModels: z.array( + z.object({ + id: z.string(), + text: z.string() + }) + ) + }), + [] + ); + + const modelsQuery = useQuery({ + ...aiProviderQueries.providerModels({ + providerId: attachment.providerId + }), + enabled: open + }); + + const allowCatalog = useMemo(() => { + const models = modelsQuery.data ?? []; + return models.filter( + (model) => model.enabled && (model.listType ?? "allow") === "allow" + ); + }, [modelsQuery.data]); + + const allowOptions: TagValue[] = useMemo(() => { + const query = modelSearch.trim().toLowerCase(); + return allowCatalog + .filter((model) => { + if (!query) return true; + return ( + model.modelKey.toLowerCase().includes(query) || + model.name.toLowerCase().includes(query) + ); + }) + .map((model) => ({ + id: String(model.modelId), + text: model.modelKey + })); + }, [allowCatalog, modelSearch]); + + const form = useForm({ + resolver: zodResolver(editSchema), + defaultValues: { + accessMode: attachment.accessMode, + selectedModels: [] + } + }); + + const accessMode = form.watch("accessMode"); + const pendingSeedRef = useRef(false); + + useEffect(() => { + if (!open) return; + form.reset({ + accessMode: attachment.accessMode, + selectedModels: attachment.selectedModelIds.map((modelId) => { + const catalog = (modelsQuery.data ?? []).find( + (model) => model.modelId === modelId + ); + return { + id: String(modelId), + text: catalog?.modelKey ?? String(modelId) + }; + }) + }); + setModelSearch(""); + pendingSeedRef.current = false; + // Only re-init when opening or switching which attachment is edited. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [open, attachment.providerId]); + + useEffect(() => { + if (!open || allowCatalog.length === 0) return; + + const current = form.getValues("selectedModels"); + const upgraded = current.map((model) => { + const catalog = allowCatalog.find( + (entry) => String(entry.modelId) === model.id + ); + return catalog ? { id: model.id, text: catalog.modelKey } : model; + }); + const changed = upgraded.some( + (model, index) => model.text !== current[index]?.text + ); + if (changed) { + form.setValue("selectedModels", upgraded); + } + + if (pendingSeedRef.current) { + form.setValue( + "selectedModels", + allowCatalog.map((model) => ({ + id: String(model.modelId), + text: model.modelKey + })) + ); + pendingSeedRef.current = false; + } + }, [open, allowCatalog, form]); + + function handleAccessModeChange(next: "inherit" | "select") { + form.setValue("accessMode", next); + if (next === "inherit") { + form.setValue("selectedModels", []); + pendingSeedRef.current = false; + return; + } + if (attachment.accessMode === "select") { + form.setValue( + "selectedModels", + attachment.selectedModelIds.map((modelId) => { + const catalog = allowCatalog.find( + (model) => model.modelId === modelId + ); + return { + id: String(modelId), + text: catalog?.modelKey ?? String(modelId) + }; + }) + ); + pendingSeedRef.current = false; + return; + } + if (allowCatalog.length > 0) { + form.setValue( + "selectedModels", + allowCatalog.map((model) => ({ + id: String(model.modelId), + text: model.modelKey + })) + ); + pendingSeedRef.current = false; + return; + } + form.setValue("selectedModels", []); + pendingSeedRef.current = true; + } + + function onSubmit(values: EditFormValues) { + onSave({ + providerId: attachment.providerId, + name: attachment.name, + accessMode: values.accessMode, + enabled: attachment.enabled, + selectedModelIds: + values.accessMode === "select" + ? values.selectedModels.map((model) => + parseInt(model.id, 10) + ) + : [] + }); + } + + return ( + + + + {attachment.name} + + {t("aiResourceProviderEditDescription")} + + + + + + ( + + + {t("aiResourceProviderMode")} + + + + {field.value === "inherit" + ? t( + "aiResourceProviderModeInheritHelp" + ) + : t( + "aiResourceProviderModeSelectHelp" + )} + + + + )} + /> + + {accessMode === "select" && ( + ( + + + {t( + "aiResourceProviderAllowModels" + )} + + + + + + {t( + "aiResourceProviderAllowModelsHelp" + )} + + + + )} + /> + )} + + + + + + + + + + + + ); +} diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 6b7070f5b..e1e06f928 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -38,6 +38,7 @@ import type { GetResourcePolicyResponse } from "@server/routers/policy"; import type { GetResourcePoliciesResponse, GetResourceWhitelistResponse, + ListResourceAiModelsResponse, ListResourceNamesResponse, ListResourceRolesResponse, ListResourceRulesResponse, @@ -51,6 +52,7 @@ import type { ListRolesResponse } from "@server/routers/role"; import type { ListSitesResponse } from "@server/routers/site"; import type { ListAllSiteResourcesByOrgResponse, + ListSiteResourceAiModelsResponse, ListSiteResourceClientsResponse, ListSiteResourceRolesResponse, ListSiteResourceUsersResponse @@ -1291,6 +1293,7 @@ export const resourceQueries = { name: string; type: string; enabled: boolean; + providerEnabled: boolean; accessMode: "inherit" | "select"; }>; }> @@ -1311,6 +1314,7 @@ export const resourceQueries = { name: string; type: string; enabled: boolean; + providerEnabled: boolean; accessMode: "inherit" | "select"; }>; }> @@ -1320,6 +1324,26 @@ export const resourceQueries = { return res.data.data.providers; } }), + resourceAiModels: ({ resourceId }: { resourceId: number }) => + queryOptions({ + queryKey: ["RESOURCES", resourceId, "AI_MODELS"] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/resource/${resourceId}/ai-models`, { signal }); + return res.data.data.models; + } + }), + siteResourceAiModels: ({ siteResourceId }: { siteResourceId: number }) => + queryOptions({ + queryKey: ["SITE_RESOURCES", siteResourceId, "AI_MODELS"] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/site-resource/${siteResourceId}/ai-models`, { signal }); + return res.data.data.models; + } + }), resourceTargets: ({ resourceId }: { resourceId: number }) => queryOptions({ queryKey: ["RESOURCES", resourceId, "TARGETS"] as const, From ca79abc9d4c6f8fc9fe5a2122172a7ba93fb4b3a Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 7 Aug 2026 14:29:48 -0400 Subject: [PATCH 085/296] Update structure --- server/lib/aiModelPricing.ts | 160 ++++++++++++++++------------------- 1 file changed, 72 insertions(+), 88 deletions(-) diff --git a/server/lib/aiModelPricing.ts b/server/lib/aiModelPricing.ts index 3edf4e9f4..8c0eb6739 100644 --- a/server/lib/aiModelPricing.ts +++ b/server/lib/aiModelPricing.ts @@ -5,22 +5,25 @@ import type { AiProviderType } from "@server/lib/aiProviderDefaults"; import type { AiUsage } from "@server/lib/aiUsageExtraction"; import logger from "@server/logger"; -// config/models.json is a runtime asset (same category as config.yml or the -// MaxMind DBs) - not part of the source tree. Its shape mirrors litellm's -// public model_prices_and_context_window.json: a flat list of -// { id, name, provider, input_cost_per_token, output_cost_per_token, -// cache_read_input_token_cost, output_cost_per_reasoning_token }, where -// `provider` is litellm's provider bucket, not our AiProviderType. const MODELS_JSON_PATH = path.join(APP_PATH, "models.json"); -export type AiModelPricingEntry = { - id: string; - name: string; - provider: string; - input_cost_per_token: number | null; - output_cost_per_token: number | null; - cache_read_input_token_cost: number | null; - output_cost_per_reasoning_token: number | null; +export type CatalogProvider = + | "openai" + | "anthropic" + | "gemini" + | "vertex" + | "azure" + | "bedrock"; + +export type AiModelCatalogEntry = { + provider: CatalogProvider; + model: string; + pricing: { + input: number | null; + output: number | null; + cacheRead: number | null; + reasoningOutput: number | null; + }; }; export type AiModelPricing = { @@ -28,65 +31,48 @@ export type AiModelPricing = { outputCostPerToken: number | null; cacheReadInputTokenCost: number | null; outputCostPerReasoningToken: number | null; - // True when the match came from a different provider bucket than the one - // mapped to this provider's type (e.g. an openRouter/custom model id that - // only matched by stripping a "vendor/" prefix against the whole table). - // Costs found this way are a best-effort approximation, not a guarantee - // the upstream provider bills at the same rate. + // True when the match came from a different catalog provider than the + // one mapped to this provider's type (e.g. an openRouter/custom model + // id that only matched a global search across every provider). Costs + // found this way are a best-effort approximation, not a guarantee the + // upstream provider bills at the same rate. approximate: boolean; }; -// Which litellm provider buckets to search for each of our provider types. -// Several of our provider types (openRouter, vercelAiGateway, custom) proxy -// arbitrary underlying models and have no dedicated bucket in the pricing -// data, so they fall back to a global search across all buckets. -const PROVIDER_PRICING_BUCKETS: Record< +// Each of our provider types maps to at most one catalog provider. Provider types that proxy +// arbitrary underlying models (openRouter, vercelAiGateway, custom) have no +// mapping and always fall back to a global search. +const PROVIDER_CATALOG_MAP: Record< Exclude, - string[] + CatalogProvider | null > = { - openai: ["openai"], - anthropic: ["anthropic"], - googleGemini: ["gemini"], - vertexAi: [ - "vertex_ai-language-models", - "vertex_ai", - "vertex_ai-anthropic_models", - "vertex_ai-mistral_models", - "vertex_ai-deepseek_models", - "vertex_ai-ai21_models", - "vertex_ai-llama_models", - "vertex_ai-minimax_models", - "vertex_ai-moonshot_models", - "vertex_ai-zai_models", - "vertex_ai-openai_models", - "vertex_ai-qwen_models", - "vertex_ai-text-models" - ], - bedrock: ["bedrock_converse", "bedrock", "bedrock_mantle"], - microsoftFoundry: ["azure", "azure_ai", "azure_text"], - openRouter: [], - vercelAiGateway: [] + openai: "openai", + anthropic: "anthropic", + googleGemini: "gemini", + vertexAi: "vertex", + bedrock: "bedrock", + microsoftFoundry: "azure", + openRouter: null, + vercelAiGateway: null }; -let modelsById: Map | null = null; +let modelsByName: Map | null = null; -function loadModels(): Map { - if (modelsById) { - return modelsById; +function loadModels(): Map { + if (modelsByName) { + return modelsByName; } - const byId = new Map(); + const byName = new Map(); try { if (fs.existsSync(MODELS_JSON_PATH)) { const raw = fs.readFileSync(MODELS_JSON_PATH, "utf-8"); - const parsed = JSON.parse(raw) as { data: AiModelPricingEntry[] }; + const parsed = JSON.parse(raw) as { data: AiModelCatalogEntry[] }; for (const entry of parsed.data ?? []) { - for (const key of [entry.id, entry.name]) { - if (!key) continue; - const list = byId.get(key) ?? []; - list.push(entry); - byId.set(key, list); - } + if (!entry.model) continue; + const list = byName.get(entry.model) ?? []; + list.push(entry); + byName.set(entry.model, list); } } else { logger.debug( @@ -97,8 +83,8 @@ function loadModels(): Map { logger.warn("Failed to load AI model pricing file", { error }); } - modelsById = byId; - return byId; + modelsByName = byName; + return byName; } function stripVendorPrefix(modelId: string): string | null { @@ -110,32 +96,32 @@ function stripVendorPrefix(modelId: string): string | null { } function toPricing( - entry: AiModelPricingEntry, + entry: AiModelCatalogEntry, approximate: boolean ): AiModelPricing { return { - inputCostPerToken: entry.input_cost_per_token, - outputCostPerToken: entry.output_cost_per_token, - cacheReadInputTokenCost: entry.cache_read_input_token_cost, - outputCostPerReasoningToken: entry.output_cost_per_reasoning_token, + inputCostPerToken: entry.pricing.input, + outputCostPerToken: entry.pricing.output, + cacheReadInputTokenCost: entry.pricing.cacheRead, + outputCostPerReasoningToken: entry.pricing.reasoningOutput, approximate }; } -function findInBuckets( - byId: Map, +function findEntry( + byName: Map, modelId: string, - buckets: string[] | null -): AiModelPricingEntry | null { + provider: CatalogProvider | null +): AiModelCatalogEntry | null { const candidates = [modelId, stripVendorPrefix(modelId)].filter( (v): v is string => v != null ); for (const key of candidates) { - const entries = byId.get(key); + const entries = byName.get(key); if (!entries) continue; - const match = buckets - ? entries.find((e) => buckets.includes(e.provider)) + const match = provider + ? entries.find((e) => e.provider === provider) : entries[0]; if (match) { return match; @@ -145,10 +131,10 @@ function findInBuckets( } /** - * Looks up per-token pricing for a model, scoped first to the litellm - * provider bucket(s) that correspond to our provider type, then falling - * back to a global search across all buckets (marked `approximate`) for - * provider types that proxy arbitrary underlying models. + * Looks up per-token pricing for a model, scoped first to the catalog + * provider that corresponds to our provider type, then falling back to a + * global search across every provider (marked `approximate`) for provider + * types that proxy arbitrary underlying models. */ export function getModelPricing( providerType: AiProviderType, @@ -158,20 +144,18 @@ export function getModelPricing( return null; } - const byId = loadModels(); - const buckets = - providerType === "custom" - ? [] - : PROVIDER_PRICING_BUCKETS[providerType]; + const byName = loadModels(); + const catalogProvider = + providerType === "custom" ? null : PROVIDER_CATALOG_MAP[providerType]; - if (buckets && buckets.length > 0) { - const scoped = findInBuckets(byId, modelId, buckets); + if (catalogProvider) { + const scoped = findEntry(byName, modelId, catalogProvider); if (scoped) { return toPricing(scoped, false); } } - const fallback = findInBuckets(byId, modelId, null); + const fallback = findEntry(byName, modelId, null); if (fallback) { return toPricing(fallback, true); } @@ -191,9 +175,9 @@ export type AiCostBreakdown = { /** * Computes a $ cost breakdown for a usage record given a model's pricing. * Cache writes and reasoning tokens fall back to the normal input/output - * rate respectively when the pricing data has no dedicated rate for them - * (the models.json schema here has no cache-write field at all, and only - * some models report a distinct reasoning rate). + * rate respectively when the catalog has no dedicated rate for them (the + * catalog has no cache-write field at all, and only some models report a + * distinct reasoning rate). */ export function calculateAiCost( pricing: AiModelPricing | null, From fe5831eb48087442fbafc266c67c0ec943b38a35 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 7 Aug 2026 14:36:46 -0400 Subject: [PATCH 086/296] pull the models from the upstream server --- server/index.ts | 2 + server/lib/aiModelCatalog.ts | 111 +++++++++++++++++++++++++++++++++++ server/lib/aiModelPricing.ts | 70 +++++++--------------- server/lib/readConfigFile.ts | 30 ++++++++++ 4 files changed, 165 insertions(+), 48 deletions(-) create mode 100644 server/lib/aiModelCatalog.ts diff --git a/server/index.ts b/server/index.ts index 3d9306d2b..62ee28b0b 100644 --- a/server/index.ts +++ b/server/index.ts @@ -28,6 +28,7 @@ import { initLogCleanupInterval } from "@server/lib/cleanupLogs"; import { initAcmeCertSync } from "#dynamic/lib/acmeCertSync"; import { fetchServerIp } from "@server/lib/serverIpService"; import { startRebuildQueueProcessor } from "@server/lib/rebuildClientAssociations"; +import { initAiModelCatalog } from "@server/lib/aiModelCatalog"; async function startServers() { await setHostMeta(); @@ -46,6 +47,7 @@ async function startServers() { initLogCleanupInterval(); initAcmeCertSync(); startRebuildQueueProcessor(); + await initAiModelCatalog(); // Start all servers const apiServer = createApiServer(); diff --git a/server/lib/aiModelCatalog.ts b/server/lib/aiModelCatalog.ts new file mode 100644 index 000000000..39b3a70c9 --- /dev/null +++ b/server/lib/aiModelCatalog.ts @@ -0,0 +1,111 @@ +import fs from "node:fs"; +import axios from "axios"; +import config from "@server/lib/config"; +import logger from "@server/logger"; + +export type CatalogProvider = + | "openai" + | "anthropic" + | "gemini" + | "vertex" + | "azure" + | "bedrock"; + +export type AiModelCatalogEntry = { + provider: CatalogProvider; + model: string; + pricing: { + input: number | null; + output: number | null; + cacheRead: number | null; + reasoningOutput: number | null; + }; +}; + +let catalog: AiModelCatalogEntry[] = []; +let refreshTimer: NodeJS.Timeout | null = null; + +async function fetchFromFile(filePath: string): Promise { + try { + if (!fs.existsSync(filePath)) { + logger.warn( + `AI model catalog file not found at ${filePath}; cost calculation will fall back to unknown pricing` + ); + return null; + } + const raw = fs.readFileSync(filePath, "utf-8"); + const parsed = JSON.parse(raw) as { data: AiModelCatalogEntry[] }; + return parsed.data ?? []; + } catch (error) { + logger.warn("Failed to read AI model catalog file", { error }); + return null; + } +} + +async function fetchFromUpstream( + upstreamUrl: string +): Promise { + try { + const res = await axios.get<{ data: AiModelCatalogEntry[] }>( + upstreamUrl, + { timeout: 15_000 } + ); + return res.data?.data ?? []; + } catch (error: any) { + logger.warn( + `Failed to fetch AI model catalog from ${upstreamUrl}: ${error.message || error}` + ); + return null; + } +} + +async function refreshCatalog(): Promise { + const { file, upstream_url } = config.getRawConfig().ai.model_catalog; + + const fetched = file + ? await fetchFromFile(file) + : await fetchFromUpstream(upstream_url); + + if (fetched) { + catalog = fetched; + logger.debug( + `AI model catalog refreshed: ${catalog.length} models loaded` + ); + } else { + logger.debug( + "AI model catalog refresh failed; keeping previously loaded catalog in memory" + ); + } +} + +function scheduleNextRefresh(): void { + const { refresh_interval_min_hours, refresh_interval_max_hours } = + config.getRawConfig().ai.model_catalog; + + // Jittered rather than fixed so that many self-hosted instances don't + // all hit the upstream catalog endpoint at the same moment. + const minMs = refresh_interval_min_hours * 60 * 60 * 1000; + const maxMs = refresh_interval_max_hours * 60 * 60 * 1000; + const delayMs = minMs + Math.random() * Math.max(0, maxMs - minMs); + + if (refreshTimer) { + clearTimeout(refreshTimer); + } + refreshTimer = setTimeout(async () => { + await refreshCatalog(); + scheduleNextRefresh(); + }, delayMs); +} + +/** + * Loads the AI model pricing catalog into memory and schedules periodic + * background refreshes. Call once at server startup. + */ +export async function initAiModelCatalog(): Promise { + await refreshCatalog(); + scheduleNextRefresh(); +} + +export function getAiModelCatalog(): AiModelCatalogEntry[] { + return catalog; +} diff --git a/server/lib/aiModelPricing.ts b/server/lib/aiModelPricing.ts index 8c0eb6739..873e4213a 100644 --- a/server/lib/aiModelPricing.ts +++ b/server/lib/aiModelPricing.ts @@ -1,30 +1,10 @@ -import fs from "node:fs"; -import path from "node:path"; -import { APP_PATH } from "@server/lib/consts"; import type { AiProviderType } from "@server/lib/aiProviderDefaults"; import type { AiUsage } from "@server/lib/aiUsageExtraction"; -import logger from "@server/logger"; - -const MODELS_JSON_PATH = path.join(APP_PATH, "models.json"); - -export type CatalogProvider = - | "openai" - | "anthropic" - | "gemini" - | "vertex" - | "azure" - | "bedrock"; - -export type AiModelCatalogEntry = { - provider: CatalogProvider; - model: string; - pricing: { - input: number | null; - output: number | null; - cacheRead: number | null; - reasoningOutput: number | null; - }; -}; +import { + getAiModelCatalog, + type AiModelCatalogEntry, + type CatalogProvider +} from "@server/lib/aiModelCatalog"; export type AiModelPricing = { inputCostPerToken: number | null; @@ -56,34 +36,28 @@ const PROVIDER_CATALOG_MAP: Record< vercelAiGateway: null }; -let modelsByName: Map | null = null; +// Indexed view over the in-memory catalog, rebuilt only when +// getAiModelCatalog() returns a different array instance (i.e. after a +// background refresh swaps it out), not on every lookup. +let indexedCatalog: AiModelCatalogEntry[] | null = null; +let indexedByName: Map = new Map(); -function loadModels(): Map { - if (modelsByName) { - return modelsByName; +function getIndexedCatalog(): Map { + const catalog = getAiModelCatalog(); + if (catalog === indexedCatalog) { + return indexedByName; } const byName = new Map(); - try { - if (fs.existsSync(MODELS_JSON_PATH)) { - const raw = fs.readFileSync(MODELS_JSON_PATH, "utf-8"); - const parsed = JSON.parse(raw) as { data: AiModelCatalogEntry[] }; - for (const entry of parsed.data ?? []) { - if (!entry.model) continue; - const list = byName.get(entry.model) ?? []; - list.push(entry); - byName.set(entry.model, list); - } - } else { - logger.debug( - `AI model pricing file not found at ${MODELS_JSON_PATH}; cost calculation will fall back to unknown pricing` - ); - } - } catch (error) { - logger.warn("Failed to load AI model pricing file", { error }); + for (const entry of catalog) { + if (!entry.model) continue; + const list = byName.get(entry.model) ?? []; + list.push(entry); + byName.set(entry.model, list); } - modelsByName = byName; + indexedCatalog = catalog; + indexedByName = byName; return byName; } @@ -144,7 +118,7 @@ export function getModelPricing( return null; } - const byName = loadModels(); + const byName = getIndexedCatalog(); const catalogProvider = providerType === "custom" ? null : PROVIDER_CATALOG_MAP[providerType]; diff --git a/server/lib/readConfigFile.ts b/server/lib/readConfigFile.ts index 37418ed2f..d5c52f13a 100644 --- a/server/lib/readConfigFile.ts +++ b/server/lib/readConfigFile.ts @@ -401,6 +401,36 @@ export const configSchema = z disable_enterprise_features: z.boolean().optional() }) .optional(), + ai: z + .object({ + model_catalog: z + .object({ + upstream_url: z + .url() + .optional() + .default("https://api.fossorial.io/api/v1/models"), + // No default - only used when an operator wants to + // pin the catalog to a local file instead of + // fetching it from upstream_url. + file: z.string().optional(), + refresh_interval_min_hours: z + .number() + .positive() + .gt(0) + .optional() + .default(6), + refresh_interval_max_hours: z + .number() + .positive() + .gt(0) + .optional() + .default(12) + }) + .optional() + .prefault({}) + }) + .optional() + .prefault({}), dns: z .object({ nameservers: z From dc1923ab1fc603c30df4142e3a01c8fdc74f76cb Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 09:56:31 -0400 Subject: [PATCH 087/296] Remove budget periods --- server/db/pg/schema/schema.ts | 28 ++++++++++++---------------- server/db/sqlite/schema/schema.ts | 30 ++++++++++++++---------------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 1c3d21b10..aa8b004ce 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1,3 +1,4 @@ +import { aiBudgetPeriods } from "@server/db/sqlite"; import { randomUUID } from "crypto"; import { InferSelectModel, sql } from "drizzle-orm"; import { @@ -1721,10 +1722,20 @@ export const aiBudgets = pgTable( () => siteResources.siteResourceId, { onDelete: "cascade" } ), + roleId: integer("roleId").references(() => roles.roleId, { + onDelete: "cascade" + }), amount: real("amount").notNull(), unit: varchar("unit").$type<"usd" | "tokens">().notNull(), period: varchar("period") - .$type<"monthly">() + .$type< + | "monthly" + | "yearly" + | "lifetime" + | "daily" + | "hourly" + | "weekly" + >() .notNull() .default("monthly"), enforcement: varchar("enforcement") @@ -1752,20 +1763,6 @@ export const aiBudgets = pgTable( ] ); -export const aiBudgetPeriods = pgTable( - "aiBudgetPeriods", - { - periodId: serial("periodId").primaryKey(), - budgetId: integer("budgetId") - .notNull() - .references(() => aiBudgets.budgetId, { onDelete: "cascade" }), - periodStart: bigint("periodStart", { mode: "number" }).notNull(), - periodEnd: bigint("periodEnd", { mode: "number" }).notNull(), - usedAmount: real("usedAmount").notNull().default(0) - }, - (t) => [unique("ai_budget_period_start_uniq").on(t.budgetId, t.periodStart)] -); - export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1853,7 +1850,6 @@ export type ResourcePolicyRule = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; export type AiBudget = InferSelectModel; -export type AiBudgetPeriod = InferSelectModel; export type ResourceAiProvider = InferSelectModel; export type SiteResourceAiProvider = InferSelectModel< typeof siteResourceAiProviders diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 71fd197e7..c8d0042df 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1705,9 +1705,22 @@ export const aiBudgets = sqliteTable( () => siteResources.siteResourceId, { onDelete: "cascade" } ), + roleId: integer("roleId").references(() => roles.roleId, { + onDelete: "cascade" + }), amount: real("amount").notNull(), unit: text("unit").$type<"usd" | "tokens">().notNull(), - period: text("period").$type<"monthly">().notNull().default("monthly"), + period: text("period") + .$type< + | "monthly" + | "yearly" + | "lifetime" + | "daily" + | "hourly" + | "weekly" + >() + .notNull() + .default("monthly"), enforcement: text("enforcement") .$type<"hard" | "soft">() .notNull() @@ -1726,20 +1739,6 @@ export const aiBudgets = sqliteTable( ] ); -export const aiBudgetPeriods = sqliteTable( - "aiBudgetPeriods", - { - periodId: integer("periodId").primaryKey({ autoIncrement: true }), - budgetId: integer("budgetId") - .notNull() - .references(() => aiBudgets.budgetId, { onDelete: "cascade" }), - periodStart: integer("periodStart").notNull(), - periodEnd: integer("periodEnd").notNull(), - usedAmount: real("usedAmount").notNull().default(0) - }, - (t) => [unique("ai_budget_period_start_uniq").on(t.budgetId, t.periodStart)] -); - export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1825,7 +1824,6 @@ export type UserPolicy = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; export type AiBudget = InferSelectModel; -export type AiBudgetPeriod = InferSelectModel; export type ResourceAiProvider = InferSelectModel; export type SiteResourceAiProvider = InferSelectModel< typeof siteResourceAiProviders From 591caab45a9902df40cd0c35d8ce2573f48791ad Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 10:47:18 -0400 Subject: [PATCH 088/296] Add documentation for creating new crud endpoints --- docs/crud-endpoints.md | 347 ++++++++++++++++++++++++++++++++++ server/db/pg/schema/schema.ts | 10 - 2 files changed, 347 insertions(+), 10 deletions(-) create mode 100644 docs/crud-endpoints.md diff --git a/docs/crud-endpoints.md b/docs/crud-endpoints.md new file mode 100644 index 000000000..0436780cf --- /dev/null +++ b/docs/crud-endpoints.md @@ -0,0 +1,347 @@ +# How to build a CRUD endpoint in this repo + +Reference for adding a new CRUD entity to the server. Based on two real +examples already in the codebase — read them side by side with this doc: + +- **Public / open-source (Community Edition) pattern**: `server/routers/aiProvider/` +- **Enterprise-only pattern**: `server/private/routers/alertRule/` + +The two are structurally identical. The only difference is *where the files +live* and *which router they get wired into*. + +## 1. Decide: public or private? + +- `server/routers//` — ships in the open-source Community Edition. + Anyone running Pangolin gets this. +- `server/private/routers//` — Enterprise/SaaS only. Gated behind + `verifyValidLicense` (and often `verifyValidSubscription(tierMatrix.x)`). + Every file here starts with the Fossorial Commercial License header block + (copy it verbatim from an existing private file). + +Everything below applies to both — swap `@server/...` for `#private/...` +import paths and add license headers when building the private version. + +## 2. Directory layout + +One folder per entity, one file per operation, a barrel `index.ts`: + +``` +server/routers// + index.ts # export * from each operation file + ./types + types.ts # response payload types + row->public mapper + validation.ts # zod schemas/refinements shared by create + update (optional) + create.ts + list.ts + get.ts + update.ts + delete.ts +``` + +`index.ts` is a flat barrel: + +```ts +export * from "./createAiProvider"; +export * from "./listAiProviders"; +export * from "./getAiProvider"; +export * from "./updateAiProvider"; +export * from "./deleteAiProvider"; +export * from "./types"; +``` + +## 3. Anatomy of a single handler + +Every handler file (`create.ts`, etc.) follows the same shape: + +```ts +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { , db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { eq } from "drizzle-orm"; +import type { GetXResponse } from "@server/routers//types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() // or entityId: z.coerce.number().int().positive() +}); + +const bodySchema = z.strictObject({ /* ... */ }); // create/update only + +registry.registerPath({ + method: "get", // put | post | delete + path: "/org/{orgId}/x", + description: "...", + tags: [OpenAPITags.], + request: { params: paramsSchema, /* body: {...} for write ops, query: for list */ }, + responses: { 200: { description: "Successful response" } } +}); + +export async function getX(req: Request, res: Response, next: NextFunction): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next(createHttpError(HttpCode.BAD_REQUEST, fromError(parsedParams.error).toString())); + } + // parse body too, if present, same pattern + + // ...business logic against db... + + if (!row) { + return next(createHttpError(HttpCode.NOT_FOUND, `X with ID ${id} not found`)); + } + + return response(res, { + data: { /* ... */ }, + success: true, + error: false, + message: "X retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next(createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")); + } +} +``` + +Rules to keep consistent with the rest of the codebase: + +- `z.strictObject` for params/body — rejects unknown keys. +- Params parsed first, then body; each on its own `safeParse` + early + `next(createHttpError(...))` — never throw raw errors. +- Every handler registers itself with the OpenAPI `registry` even if nobody + reads the spec directly — it's how `/api/v1/docs` stays accurate. +- Catch-all `try/catch` at the bottom: `logger.error(error)` + + generic `500` message. Never leak internal error details to the client. +- Use `response(res, { data, success, error, message, status })` from + `@server/lib/response` for every response, success or otherwise (errors go + through `next(createHttpError(...))` instead, not through `response`). +- If the route already ran an access-control middleware that fetched the row + (see §5), reuse it instead of re-querying: + `req.aiProvider && req.aiProvider.providerId === providerId ? [req.aiProvider] : await db.select()...` + +### List handler specifics + +Pagination is a fixed shape (`page`, `pageSize`, optional `query` for +search). See `listAiProviders.ts`: + +```ts +const listSchema = z.object({ + pageSize: z.coerce.number().int().positive().optional().catch(20).default(20), + page: z.coerce.number().int().min(0).optional().catch(1).default(1), + query: z.string().optional() +}); +``` + +Run the count query and the page query in `Promise.all`, and return +`PaginatedResponse<{ items: T[] }>` (`@server/types/Pagination`) with +`{ total, pageSize, page }`. + +### types.ts specifics + +- Define one response type per operation: `ListResponse`, + `GetResponse`, `CreateOrEditResponse` (create and update + commonly share a response shape). +- If the raw DB row needs to be shaped for clients (decrypting secrets, + parsing a serialized column, hiding a column), put a `toPublic()` + mapper here — see `toPublicAiProvider` for the pattern of stripping + `apiKey`/serialized columns and re-adding decrypted/parsed versions. + +### validation.ts specifics + +Only needed when create and update share non-trivial zod pieces (enums, +`superRefine` cross-field rules). Export the raw schemas (`z.enum([...])`) +and refinement functions, and import them into both `createX.ts` and +`updateX.ts` — see `aiProvider/validation.ts`'s +`refineProviderUpstreamFields`. + +## 4. Wire up an access-control middleware (for id-scoped routes) + +For routes scoped to a single row (`/x/:xId`, as opposed to +`/org/:orgId/x` create/list), add a `verifyAccess` middleware in +`server/middlewares/` (or `server/private/middlewares/` for enterprise-only +entities) and export it from that directory's `index.ts`. + +Pattern (`verifyAiProviderAccess.ts`): + +1. Read the id param, `Number.parseInt`/validate it. +2. Load the row by id. +3. `404` if it doesn't exist. +4. Resolve the row's `orgId`, then check/attach `req.userOrg` (query + `userOrgs` if not already on the request), `403` if the user isn't in + that org. +5. Run `checkOrgAccessPolicy` if `req.orgPolicyAllowed` hasn't been resolved + yet. +6. Set `req.userOrgId`, `req.userOrgRoleIds`, and stash the row on the + request (e.g. `req.aiProvider = provider`) so downstream handlers and + `verifyUserHasAction` don't have to refetch it. + +Org-scoped create/list routes (`/org/:orgId/x`) don't need a bespoke +middleware — they use the existing generic `verifyOrgAccess` from +`@server/middlewares`. + +## 5. Register an action + permission check + +Add one `ActionsEnum` entry per operation in `server/auth/actions.ts`, +grouped near the entity's other actions, named `create`, +`get`, `update`, `delete`, `list`: + +```ts +createAiProvider = "createAiProvider", +deleteAiProvider = "deleteAiProvider", +getAiProvider = "getAiProvider", +listAiProviders = "listAiProviders", +updateAiProvider = "updateAiProvider", +``` + +Every route uses `verifyUserHasAction(ActionsEnum.x)` to check the caller's +role/permissions for that action, and mutating routes (create/update/delete) +follow it with `logActionAudit(ActionsEnum.x)` to record the action in the +audit log. + +## 6. Register the routes + +There are four router files; which one(s) you touch depends on public vs. +private and user-facing vs. service-to-service: + +| File | Purpose | +|---|---| +| `server/routers/external.ts` | Public, user-facing API. Exports `authenticated`, `unauthenticated`, `authRouter` Express routers. | +| `server/routers/internal.ts` | Public, internal service-to-service API (gerbil, badger, traefik-config) — no user auth, exports `internalRouter`. | +| `server/private/routers/external.ts` | Enterprise-only, user-facing. Imports `authenticated`/`unauthenticated`/`authRouter` **from the public `external.ts`** and re-exports them, then adds more routes on top. | +| `server/private/routers/internal.ts` | Enterprise-only, service-to-service. Same re-export trick with `internalRouter`. | + +Private router files always start: + +```ts +import { + unauthenticated as ua, + authenticated as a, + authRouter as aa +} from "@server/routers/external"; + +export const authenticated = a; +export const unauthenticated = ua; +export const authRouter = aa; +``` + +...and then call `authenticated.get/put/post/delete(...)` to bolt on +additional, enterprise-only routes on the *same* router instances the public +build uses. This is why the private build has strictly more routes than the +public build, not a divergent copy. + +### Route registration order (mutating vs read) + +Standard middleware chain per verb, using `alertRule`'s registrations as the +template: + +```ts +// Create — org-scoped, no row exists yet +authenticated.put( + "/org/:orgId/x", + verifyValidLicense, // private/enterprise routes only + verifyOrgAccess, + verifyLimits, // if the entity counts against a plan limit + verifyUserHasAction(ActionsEnum.createX), + logActionAudit(ActionsEnum.createX), + x.createX +); + +// Update — row-scoped +authenticated.post( + "/org/:orgId/x/:xId", // or "/x/:xId" if id is globally unique + verifyValidLicense, + verifyOrgAccess, // or verifyXAccess if globally-keyed + verifyUserHasAction(ActionsEnum.updateX), + logActionAudit(ActionsEnum.updateX), + x.updateX +); + +// Delete — row-scoped +authenticated.delete( + "/org/:orgId/x/:xId", + verifyValidLicense, + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.deleteX), + logActionAudit(ActionsEnum.deleteX), + x.deleteX +); + +// List — org-scoped, read-only, no audit log +authenticated.get( + "/org/:orgId/xs", + verifyValidLicense, + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.listXs), + x.listXs +); + +// Get one — row-scoped, read-only, no audit log +authenticated.get( + "/org/:orgId/x/:xId", + verifyValidLicense, + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.getX), + x.getX +); +``` + +Notes: + +- HTTP verbs: `PUT` = create, `POST` = update, `GET` = read, `DELETE` = + delete. This repo does not use `PATCH` for entity updates (site + provisioning keys are the one exception, using `PATCH`). +- `verifyValidLicense` is only needed on private/enterprise routes; public + OSS routes skip it. +- Use `verifyValidSubscription(tierMatrix.someFeature)` right after + `verifyValidLicense` when a feature is gated to specific SaaS tiers (see + `tierMatrix` usages in `server/private/routers/external.ts`). +- `verifyLimits` goes on create routes for entities that count against a + plan/seat limit. +- For entities keyed by a globally-unique id (not nested under `/org/:orgId`), + use the dedicated `verifyAccess` middleware from §4 instead of + `verifyOrgAccess` on the row-scoped routes (see how `/ai-provider/:providerId` + uses `verifyAiProviderAccess`, while `/org/:orgId/ai-provider` create/list + use plain `verifyOrgAccess`). +- Read-only routes (`get`, `list`) skip `logActionAudit` — only mutations are + audited. +- `internal*.ts` routes are for trusted internal callers (gerbil/badger + sidecars) and generally skip user-facing auth entirely, using + `verifySessionUserMiddleware` / `verifyUserFromResourceSessionMiddleware` + instead of `verifyOrgAccess`/`verifyUserHasAction`. CRUD entities almost + never need internal router entries — only add one if a sidecar process + needs direct access to the resource. + +## 7. The `#dynamic` alias (advanced — most CRUD work can ignore this) + +Some middleware (e.g. `logActionAudit`) needs a real implementation in the +enterprise/SaaS build but a no-op stub in the open-source build, while +being imported by identical code in `server/routers/external.ts` in both +builds. That's done via the `#dynamic/*` import alias, which +`tsconfig.oss.json` points at `./server/*` and `tsconfig.enterprise.json` / +`tsconfig.saas.json` point at `./server/private/*`. You only need this +pattern if you're adding a genuinely dual-implementation hook; a normal +private-only CRUD entity (like `alertRule`) never touches `#dynamic` — it +just lives entirely under `server/private/` and is imported with `#private/*` +directly from `server/private/routers/external.ts`. + +## 8. Checklist for a new entity + +1. Add the DB table to `server/db/pg/schema/schema.ts` (and sqlite schema if + applicable). +2. Add `ActionsEnum` entries in `server/auth/actions.ts`. +3. Create `server/routers//` (or `server/private/routers//`): + `types.ts`, optional `validation.ts`, one file per operation, `index.ts` + barrel. +4. If routes are row-scoped by a global id, add + `verifyAccess.ts` to `server/middlewares/` or + `server/private/middlewares/`, and export it from that directory's + `index.ts`. +5. Wire routes into `external.ts` (public or private) following the verb/ + middleware table in §6. Add to `internal.ts` only if a sidecar needs + direct access. +6. Add license header block to every new file if it's under `server/private/`. diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index aa8b004ce..e2eb50205 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1,4 +1,3 @@ -import { aiBudgetPeriods } from "@server/db/sqlite"; import { randomUUID } from "crypto"; import { InferSelectModel, sql } from "drizzle-orm"; import { @@ -1747,15 +1746,6 @@ export const aiBudgets = pgTable( updatedAt: bigint("updatedAt", { mode: "number" }).notNull() }, (t) => [ - check( - "ai_budget_one_scope", - sql`( - (CASE WHEN ${t.providerId} IS NOT NULL THEN 1 ELSE 0 END) + - (CASE WHEN ${t.modelId} IS NOT NULL THEN 1 ELSE 0 END) + - (CASE WHEN ${t.resourceId} IS NOT NULL THEN 1 ELSE 0 END) + - (CASE WHEN ${t.siteResourceId} IS NOT NULL THEN 1 ELSE 0 END) - ) = 1` - ), unique("ai_budget_provider_uniq").on(t.providerId), unique("ai_budget_model_uniq").on(t.modelId), unique("ai_budget_resource_uniq").on(t.resourceId), From 969e7c9296c32f46a7e9a7bbfad97087f4aa8259 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 10:58:11 -0400 Subject: [PATCH 089/296] add crud endpoint skill --- .claude/skills/crud-endpoints/SKILL.md | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .claude/skills/crud-endpoints/SKILL.md diff --git a/.claude/skills/crud-endpoints/SKILL.md b/.claude/skills/crud-endpoints/SKILL.md new file mode 100644 index 000000000..6536ea234 --- /dev/null +++ b/.claude/skills/crud-endpoints/SKILL.md @@ -0,0 +1,31 @@ +--- +name: crud-endpoints +description: Use whenever asked to add, create, or scaffold a CRUD endpoint, router, or entity in this repo's server (create/list/get/update/delete handlers, new `server/routers//` or `server/private/routers//` folder). Points to the established file layout, middleware, ActionsEnum, and route-registration conventions before writing any code. +--- + +Before writing any router/handler/middleware code for a new entity, read +`docs/crud-endpoints.md` in full. It documents, with real examples from +`server/routers/aiProvider/` (public) and `server/private/routers/alertRule/` +(enterprise-only), how this repo structures CRUD endpoints: + +- Directory/file layout per entity (`index.ts`, `types.ts`, `validation.ts`, + one file per operation). +- The standard handler anatomy (zod parsing, OpenAPI registry, response + envelope, error handling). +- Where access-control middleware (`verifyAccess`) lives and when + it's needed vs. plain `verifyOrgAccess`. +- How to wire up `ActionsEnum` entries, `verifyUserHasAction`, and + `logActionAudit`. +- Which of the four router files (`server/routers/external.ts`, + `server/routers/internal.ts`, `server/private/routers/external.ts`, + `server/private/routers/internal.ts`) to register routes in, and the + middleware chain template per HTTP verb. +- The repo's non-standard verb convention: **`PUT` = create, `POST` = + update** (backwards from typical REST) — don't "fix" this to standard + REST verbs, match the existing convention. +- The `#dynamic` import alias, for the rare case of a hook needing different + implementations in OSS vs. enterprise builds. + +Follow that doc's checklist (§8) step by step rather than improvising a +structure. If the doc and the actual code in `aiProvider`/`alertRule` ever +disagree, trust the code and flag the doc as stale. From 0050fad70dbd6c74d0e564971f3b906423b57630 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 11:05:10 -0400 Subject: [PATCH 090/296] add basic crud for ai budgets --- server/auth/actions.ts | 7 +- server/index.ts | 2 + server/middlewares/index.ts | 1 + server/middlewares/verifyAiBudgetAccess.ts | 99 ++++++ server/openApi.ts | 3 +- server/routers/aiBudget/createAiBudget.ts | 252 +++++++++++++++ server/routers/aiBudget/deleteAiBudget.ts | 79 +++++ server/routers/aiBudget/getAiBudget.ts | 81 +++++ server/routers/aiBudget/index.ts | 6 + server/routers/aiBudget/listAiBudgets.ts | 130 ++++++++ server/routers/aiBudget/types.ts | 14 + server/routers/aiBudget/updateAiBudget.ts | 339 +++++++++++++++++++++ server/routers/aiBudget/validation.ts | 46 +++ server/routers/external.ts | 42 ++- 14 files changed, 1098 insertions(+), 3 deletions(-) create mode 100644 server/middlewares/verifyAiBudgetAccess.ts create mode 100644 server/routers/aiBudget/createAiBudget.ts create mode 100644 server/routers/aiBudget/deleteAiBudget.ts create mode 100644 server/routers/aiBudget/getAiBudget.ts create mode 100644 server/routers/aiBudget/index.ts create mode 100644 server/routers/aiBudget/listAiBudgets.ts create mode 100644 server/routers/aiBudget/types.ts create mode 100644 server/routers/aiBudget/updateAiBudget.ts create mode 100644 server/routers/aiBudget/validation.ts diff --git a/server/auth/actions.ts b/server/auth/actions.ts index 1b4a33e89..f9d781169 100644 --- a/server/auth/actions.ts +++ b/server/auth/actions.ts @@ -194,7 +194,12 @@ export enum ActionsEnum { deleteAiModel = "deleteAiModel", getAiModel = "getAiModel", listAiModels = "listAiModels", - updateAiModel = "updateAiModel" + updateAiModel = "updateAiModel", + createAiBudget = "createAiBudget", + deleteAiBudget = "deleteAiBudget", + getAiBudget = "getAiBudget", + listAiBudgets = "listAiBudgets", + updateAiBudget = "updateAiBudget" } export async function checkUserActionPermission( diff --git a/server/index.ts b/server/index.ts index 62ee28b0b..3eb16092a 100644 --- a/server/index.ts +++ b/server/index.ts @@ -10,6 +10,7 @@ import { createIntegrationApiServer } from "./integrationApiServer"; import { ApiKey, ApiKeyOrg, + AiBudget, AiModel, AiProvider, RemoteExitNode, @@ -92,6 +93,7 @@ declare global { siteResource?: SiteResource; aiProvider?: AiProvider; aiModel?: AiModel; + aiBudget?: AiBudget; orgPolicyAllowed?: boolean; } } diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts index 8f6b50e33..7242ea0cf 100644 --- a/server/middlewares/index.ts +++ b/server/middlewares/index.ts @@ -29,6 +29,7 @@ export * from "./verifySiteProvisioningKeyAccess"; export * from "./verifyDomainAccess"; export * from "./verifyAiProviderAccess"; export * from "./verifyAiModelAccess"; +export * from "./verifyAiBudgetAccess"; export * from "./verifyUserIsOrgOwner"; export * from "./verifyUserFromResourceSession"; export * from "./verifySiteResourceAccess"; diff --git a/server/middlewares/verifyAiBudgetAccess.ts b/server/middlewares/verifyAiBudgetAccess.ts new file mode 100644 index 000000000..d0834057a --- /dev/null +++ b/server/middlewares/verifyAiBudgetAccess.ts @@ -0,0 +1,99 @@ +import { Request, Response, NextFunction } from "express"; +import { aiBudgets, db, userOrgs } from "@server/db"; +import { and, eq } from "drizzle-orm"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy"; +import { getUserOrgRoleIds } from "@server/lib/userOrgRoles"; +import { getFirstString } from "@server/lib/requestParams"; + +export async function verifyAiBudgetAccess( + req: Request, + res: Response, + next: NextFunction +) { + try { + const userId = req.user!.userId; + const budgetIdRaw = getFirstString(req.params.budgetId); + const budgetId = Number.parseInt(budgetIdRaw ?? "", 10); + + if (!userId) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "User not authenticated") + ); + } + + if (Number.isNaN(budgetId)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid budget ID") + ); + } + + const [budget] = await db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.budgetId, budgetId)) + .limit(1); + + if (!budget) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI budget with ID ${budgetId} not found` + ) + ); + } + + const orgId = budget.orgId; + + if (!req.userOrg || req.userOrg.orgId !== orgId) { + const userOrgRole = await db + .select() + .from(userOrgs) + .where( + and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)) + ) + .limit(1); + req.userOrg = userOrgRole[0]; + } + + if (!req.userOrg) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + if (req.orgPolicyAllowed === undefined && req.userOrg.orgId) { + const policyCheck = await checkOrgAccessPolicy({ + orgId: req.userOrg.orgId, + userId, + session: req.session + }); + req.orgPolicyAllowed = policyCheck.allowed; + if (!policyCheck.allowed || policyCheck.error) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "" + (policyCheck.error || "Unknown error") + ) + ); + } + } + + req.userOrgId = orgId; + req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId); + req.aiBudget = budget; + + return next(); + } catch (error) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Error verifying AI budget access" + ) + ); + } +} diff --git a/server/openApi.ts b/server/openApi.ts index 1bcaa23ab..920aaa471 100644 --- a/server/openApi.ts +++ b/server/openApi.ts @@ -30,7 +30,8 @@ export enum OpenAPITags { PublicResourceLegacy = "Public Resource (Legacy)", PrivateResourceLegacy = "Private Resource (Legacy)", AiProvider = "AI Provider", - AiModel = "AI Model" + AiModel = "AI Model", + AiBudget = "AI Budget" } // Order here controls the order tags are displayed in Swagger UI diff --git a/server/routers/aiBudget/createAiBudget.ts b/server/routers/aiBudget/createAiBudget.ts new file mode 100644 index 000000000..705167c3a --- /dev/null +++ b/server/routers/aiBudget/createAiBudget.ts @@ -0,0 +1,252 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { + aiBudgets, + aiModels, + aiProviders, + db, + resources, + roles, + siteResources +} from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { eq } from "drizzle-orm"; +import type { CreateOrEditAiBudgetResponse } from "@server/routers/aiBudget/types"; +import { + aiBudgetEnforcementSchema, + aiBudgetPeriodSchema, + aiBudgetUnitSchema, + refineBudgetScopeFields +} from "@server/routers/aiBudget/validation"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +const bodySchema = z + .strictObject({ + providerId: z.coerce.number().int().positive().optional(), + modelId: z.coerce.number().int().positive().optional(), + resourceId: z.coerce.number().int().positive().optional(), + siteResourceId: z.coerce.number().int().positive().optional(), + roleId: z.coerce.number().int().positive().optional(), + amount: z.number().positive(), + unit: aiBudgetUnitSchema, + period: aiBudgetPeriodSchema.optional().default("monthly"), + enforcement: aiBudgetEnforcementSchema.optional().default("hard"), + enabled: z.boolean().optional() + }) + .superRefine((data, ctx) => refineBudgetScopeFields(data, ctx)); + +registry.registerPath({ + method: "put", + path: "/org/{orgId}/ai-budget", + description: "Create an AI budget for an organization.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema, + body: { + content: { + "application/json": { + schema: bodySchema + } + } + } + }, + responses: { + 201: { + description: "Successful response" + } + } +}); + +export async function createAiBudget( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedBody = bodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { orgId } = parsedParams.data; + const { + providerId, + modelId, + resourceId, + siteResourceId, + roleId, + amount, + unit, + period, + enforcement, + enabled + } = parsedBody.data; + + if (providerId !== undefined) { + const [provider] = await db + .select({ orgId: aiProviders.orgId }) + .from(aiProviders) + .where(eq(aiProviders.providerId, providerId)) + .limit(1); + if (!provider || provider.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found in this organization` + ) + ); + } + } + + if (modelId !== undefined) { + const [model] = await db + .select({ orgId: aiProviders.orgId }) + .from(aiModels) + .innerJoin( + aiProviders, + eq(aiModels.providerId, aiProviders.providerId) + ) + .where(eq(aiModels.modelId, modelId)) + .limit(1); + if (!model || model.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI model with ID ${modelId} not found in this organization` + ) + ); + } + } + + if (resourceId !== undefined) { + const [resource] = await db + .select({ orgId: resources.orgId }) + .from(resources) + .where(eq(resources.resourceId, resourceId)) + .limit(1); + if (!resource || resource.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Resource with ID ${resourceId} not found in this organization` + ) + ); + } + } + + if (siteResourceId !== undefined) { + const [siteResource] = await db + .select({ orgId: siteResources.orgId }) + .from(siteResources) + .where(eq(siteResources.siteResourceId, siteResourceId)) + .limit(1); + if (!siteResource || siteResource.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Site resource with ID ${siteResourceId} not found in this organization` + ) + ); + } + } + + if (roleId !== undefined) { + const [role] = await db + .select({ orgId: roles.orgId }) + .from(roles) + .where(eq(roles.roleId, roleId)) + .limit(1); + if (!role || role.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Role with ID ${roleId} not found in this organization` + ) + ); + } + } + + const conflictCondition = + providerId !== undefined + ? eq(aiBudgets.providerId, providerId) + : modelId !== undefined + ? eq(aiBudgets.modelId, modelId) + : resourceId !== undefined + ? eq(aiBudgets.resourceId, resourceId) + : siteResourceId !== undefined + ? eq(aiBudgets.siteResourceId, siteResourceId) + : undefined; + + if (conflictCondition) { + const [existing] = await db + .select({ budgetId: aiBudgets.budgetId }) + .from(aiBudgets) + .where(conflictCondition) + .limit(1); + if (existing) { + return next( + createHttpError( + HttpCode.CONFLICT, + "A budget already exists for this scope" + ) + ); + } + } + + const now = Date.now(); + const [budget] = await db + .insert(aiBudgets) + .values({ + orgId, + providerId: providerId ?? null, + modelId: modelId ?? null, + resourceId: resourceId ?? null, + siteResourceId: siteResourceId ?? null, + roleId: roleId ?? null, + amount, + unit, + period, + enforcement, + enabled: enabled ?? true, + createdAt: now, + updatedAt: now + }) + .returning(); + + return response(res, { + data: { budget }, + success: true, + error: false, + message: "AI budget created successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/deleteAiBudget.ts b/server/routers/aiBudget/deleteAiBudget.ts new file mode 100644 index 000000000..fc49ca3d4 --- /dev/null +++ b/server/routers/aiBudget/deleteAiBudget.ts @@ -0,0 +1,79 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiBudgets, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { eq } from "drizzle-orm"; + +const paramsSchema = z.strictObject({ + budgetId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "delete", + path: "/ai-budget/{budgetId}", + description: "Delete an AI budget.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function deleteAiBudget( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { budgetId } = parsedParams.data; + + const [existing] = await db + .select({ budgetId: aiBudgets.budgetId }) + .from(aiBudgets) + .where(eq(aiBudgets.budgetId, budgetId)) + .limit(1); + + if (!existing) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI budget with ID ${budgetId} not found` + ) + ); + } + + await db.delete(aiBudgets).where(eq(aiBudgets.budgetId, budgetId)); + + return response(res, { + data: null, + success: true, + error: false, + message: "AI budget deleted successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/getAiBudget.ts b/server/routers/aiBudget/getAiBudget.ts new file mode 100644 index 000000000..e5435101f --- /dev/null +++ b/server/routers/aiBudget/getAiBudget.ts @@ -0,0 +1,81 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiBudgets, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { eq } from "drizzle-orm"; +import type { GetAiBudgetResponse } from "@server/routers/aiBudget/types"; + +const paramsSchema = z.strictObject({ + budgetId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "get", + path: "/ai-budget/{budgetId}", + description: "Get an AI budget by ID.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function getAiBudget( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { budgetId } = parsedParams.data; + + const [budget] = + req.aiBudget && req.aiBudget.budgetId === budgetId + ? [req.aiBudget] + : await db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.budgetId, budgetId)) + .limit(1); + + if (!budget) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI budget with ID ${budgetId} not found` + ) + ); + } + + return response(res, { + data: { budget }, + success: true, + error: false, + message: "AI budget retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/index.ts b/server/routers/aiBudget/index.ts new file mode 100644 index 000000000..0da53e0d2 --- /dev/null +++ b/server/routers/aiBudget/index.ts @@ -0,0 +1,6 @@ +export * from "./createAiBudget"; +export * from "./listAiBudgets"; +export * from "./getAiBudget"; +export * from "./updateAiBudget"; +export * from "./deleteAiBudget"; +export * from "./types"; diff --git a/server/routers/aiBudget/listAiBudgets.ts b/server/routers/aiBudget/listAiBudgets.ts new file mode 100644 index 000000000..f71666140 --- /dev/null +++ b/server/routers/aiBudget/listAiBudgets.ts @@ -0,0 +1,130 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiBudgets, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { asc, eq } from "drizzle-orm"; +import type { ListAiBudgetsResponse } from "@server/routers/aiBudget/types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +const listSchema = z.object({ + pageSize: z.coerce + .number() + .int() + .positive() + .optional() + .catch(20) + .default(20) + .openapi({ + type: "integer", + default: 20, + description: "Number of items per page" + }), + page: z.coerce + .number() + .int() + .min(0) + .optional() + .catch(1) + .default(1) + .openapi({ + type: "integer", + default: 1, + description: "Page number to retrieve" + }) +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/ai-budgets", + description: "List AI budgets for an organization.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema, + query: listSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listAiBudgets( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = listSchema.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error).toString() + ) + ); + } + + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { orgId } = parsedParams.data; + const { pageSize, page } = parsedQuery.data; + + const baseQuery = db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.orgId, orgId)); + + const countQuery = db.$count( + db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.orgId, orgId)) + .as("filtered_ai_budgets") + ); + + const [totalCount, rows] = await Promise.all([ + countQuery, + baseQuery + .limit(pageSize) + .offset(pageSize * (page - 1)) + .orderBy(asc(aiBudgets.budgetId)) + ]); + + return response(res, { + data: { + budgets: rows, + pagination: { + total: totalCount, + pageSize, + page + } + }, + success: true, + error: false, + message: "AI budgets retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/types.ts b/server/routers/aiBudget/types.ts new file mode 100644 index 000000000..11ffd1050 --- /dev/null +++ b/server/routers/aiBudget/types.ts @@ -0,0 +1,14 @@ +import type { AiBudget } from "@server/db"; +import type { PaginatedResponse } from "@server/types/Pagination"; + +export type ListAiBudgetsResponse = PaginatedResponse<{ + budgets: AiBudget[]; +}>; + +export type GetAiBudgetResponse = { + budget: AiBudget; +}; + +export type CreateOrEditAiBudgetResponse = { + budget: AiBudget; +}; diff --git a/server/routers/aiBudget/updateAiBudget.ts b/server/routers/aiBudget/updateAiBudget.ts new file mode 100644 index 000000000..e8bd845c5 --- /dev/null +++ b/server/routers/aiBudget/updateAiBudget.ts @@ -0,0 +1,339 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { + aiBudgets, + aiModels, + aiProviders, + db, + resources, + roles, + siteResources +} from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq, ne } from "drizzle-orm"; +import type { CreateOrEditAiBudgetResponse } from "@server/routers/aiBudget/types"; +import { + aiBudgetEnforcementSchema, + aiBudgetPeriodSchema, + aiBudgetUnitSchema, + refineBudgetScopeFields +} from "@server/routers/aiBudget/validation"; + +const paramsSchema = z.strictObject({ + budgetId: z.coerce.number().int().positive() +}); + +const bodySchema = z.strictObject({ + providerId: z.coerce.number().int().positive().nullable().optional(), + modelId: z.coerce.number().int().positive().nullable().optional(), + resourceId: z.coerce.number().int().positive().nullable().optional(), + siteResourceId: z.coerce.number().int().positive().nullable().optional(), + roleId: z.coerce.number().int().positive().nullable().optional(), + amount: z.number().positive().optional(), + unit: aiBudgetUnitSchema.optional(), + period: aiBudgetPeriodSchema.optional(), + enforcement: aiBudgetEnforcementSchema.optional(), + enabled: z.boolean().optional() +}); + +registry.registerPath({ + method: "post", + path: "/ai-budget/{budgetId}", + description: "Update an AI budget.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema, + body: { + content: { + "application/json": { + schema: bodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function updateAiBudget( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedBody = bodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { budgetId } = parsedParams.data; + const body = parsedBody.data; + + const [existing] = + req.aiBudget && req.aiBudget.budgetId === budgetId + ? [req.aiBudget] + : await db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.budgetId, budgetId)) + .limit(1); + + if (!existing) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI budget with ID ${budgetId} not found` + ) + ); + } + + const orgId = existing.orgId; + + const nextProviderId = + body.providerId !== undefined + ? body.providerId + : existing.providerId; + const nextModelId = + body.modelId !== undefined ? body.modelId : existing.modelId; + const nextResourceId = + body.resourceId !== undefined + ? body.resourceId + : existing.resourceId; + const nextSiteResourceId = + body.siteResourceId !== undefined + ? body.siteResourceId + : existing.siteResourceId; + const nextRoleId = + body.roleId !== undefined ? body.roleId : existing.roleId; + + const scopeValidation = z + .object({ + providerId: z.number().nullable().optional(), + modelId: z.number().nullable().optional(), + resourceId: z.number().nullable().optional(), + siteResourceId: z.number().nullable().optional(), + roleId: z.number().nullable().optional() + }) + .superRefine((data, ctx) => refineBudgetScopeFields(data, ctx)) + .safeParse({ + providerId: nextProviderId, + modelId: nextModelId, + resourceId: nextResourceId, + siteResourceId: nextSiteResourceId, + roleId: nextRoleId + }); + + if (!scopeValidation.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(scopeValidation.error).toString() + ) + ); + } + + if (body.providerId !== undefined && body.providerId !== null) { + const [provider] = await db + .select({ orgId: aiProviders.orgId }) + .from(aiProviders) + .where(eq(aiProviders.providerId, body.providerId)) + .limit(1); + if (!provider || provider.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${body.providerId} not found in this organization` + ) + ); + } + } + + if (body.modelId !== undefined && body.modelId !== null) { + const [model] = await db + .select({ orgId: aiProviders.orgId }) + .from(aiModels) + .innerJoin( + aiProviders, + eq(aiModels.providerId, aiProviders.providerId) + ) + .where(eq(aiModels.modelId, body.modelId)) + .limit(1); + if (!model || model.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI model with ID ${body.modelId} not found in this organization` + ) + ); + } + } + + if (body.resourceId !== undefined && body.resourceId !== null) { + const [resource] = await db + .select({ orgId: resources.orgId }) + .from(resources) + .where(eq(resources.resourceId, body.resourceId)) + .limit(1); + if (!resource || resource.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Resource with ID ${body.resourceId} not found in this organization` + ) + ); + } + } + + if ( + body.siteResourceId !== undefined && + body.siteResourceId !== null + ) { + const [siteResource] = await db + .select({ orgId: siteResources.orgId }) + .from(siteResources) + .where(eq(siteResources.siteResourceId, body.siteResourceId)) + .limit(1); + if (!siteResource || siteResource.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Site resource with ID ${body.siteResourceId} not found in this organization` + ) + ); + } + } + + if (body.roleId !== undefined && body.roleId !== null) { + const [role] = await db + .select({ orgId: roles.orgId }) + .from(roles) + .where(eq(roles.roleId, body.roleId)) + .limit(1); + if (!role || role.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Role with ID ${body.roleId} not found in this organization` + ) + ); + } + } + + const conflictCondition = + body.providerId !== undefined && body.providerId !== null + ? and( + eq(aiBudgets.providerId, body.providerId), + ne(aiBudgets.budgetId, budgetId) + ) + : body.modelId !== undefined && body.modelId !== null + ? and( + eq(aiBudgets.modelId, body.modelId), + ne(aiBudgets.budgetId, budgetId) + ) + : body.resourceId !== undefined && body.resourceId !== null + ? and( + eq(aiBudgets.resourceId, body.resourceId), + ne(aiBudgets.budgetId, budgetId) + ) + : body.siteResourceId !== undefined && + body.siteResourceId !== null + ? and( + eq(aiBudgets.siteResourceId, body.siteResourceId), + ne(aiBudgets.budgetId, budgetId) + ) + : undefined; + + if (conflictCondition) { + const [conflict] = await db + .select({ budgetId: aiBudgets.budgetId }) + .from(aiBudgets) + .where(conflictCondition) + .limit(1); + if (conflict) { + return next( + createHttpError( + HttpCode.CONFLICT, + "A budget already exists for this scope" + ) + ); + } + } + + const updateData: Partial = { + updatedAt: Date.now() + }; + + if (body.providerId !== undefined) { + updateData.providerId = body.providerId; + } + if (body.modelId !== undefined) { + updateData.modelId = body.modelId; + } + if (body.resourceId !== undefined) { + updateData.resourceId = body.resourceId; + } + if (body.siteResourceId !== undefined) { + updateData.siteResourceId = body.siteResourceId; + } + if (body.roleId !== undefined) { + updateData.roleId = body.roleId; + } + if (body.amount !== undefined) { + updateData.amount = body.amount; + } + if (body.unit !== undefined) { + updateData.unit = body.unit; + } + if (body.period !== undefined) { + updateData.period = body.period; + } + if (body.enforcement !== undefined) { + updateData.enforcement = body.enforcement; + } + if (body.enabled !== undefined) { + updateData.enabled = body.enabled; + } + + const [budget] = await db + .update(aiBudgets) + .set(updateData) + .where(eq(aiBudgets.budgetId, budgetId)) + .returning(); + + return response(res, { + data: { budget }, + success: true, + error: false, + message: "AI budget updated successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/validation.ts b/server/routers/aiBudget/validation.ts new file mode 100644 index 000000000..303039948 --- /dev/null +++ b/server/routers/aiBudget/validation.ts @@ -0,0 +1,46 @@ +import { z } from "zod"; + +export const aiBudgetUnitSchema = z.enum(["usd", "tokens"]); + +export const aiBudgetPeriodSchema = z.enum([ + "monthly", + "yearly", + "lifetime", + "daily", + "hourly", + "weekly" +]); + +export const aiBudgetEnforcementSchema = z.enum(["hard", "soft"]); + +export function refineBudgetScopeFields( + data: { + providerId?: number | null; + modelId?: number | null; + resourceId?: number | null; + siteResourceId?: number | null; + roleId?: number | null; + }, + ctx: z.RefinementCtx +) { + const scopeFields = [ + data.providerId, + data.modelId, + data.resourceId, + data.siteResourceId, + data.roleId + ]; + + const setCount = scopeFields.filter( + (value) => value !== null && value !== undefined + ).length; + + if (setCount > 1) { + ctx.addIssue({ + code: "custom", + message: + "Only one of providerId, modelId, resourceId, siteResourceId, or roleId may be set on a budget", + path: ["providerId"] + }); + } +} diff --git a/server/routers/external.ts b/server/routers/external.ts index 94165cc8d..f469707f8 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -47,7 +47,8 @@ import { verifyLimits, verifyResourcePolicyAccess, verifyAiProviderAccess, - verifyAiModelAccess + verifyAiModelAccess, + verifyAiBudgetAccess } from "@server/middlewares"; import { ActionsEnum } from "@server/auth/actions"; import rateLimit, { ipKeyGenerator } from "express-rate-limit"; @@ -58,6 +59,7 @@ import { logActionAudit } from "#dynamic/middlewares"; import { checkRoundTripMessage } from "./ws"; import * as labels from "@server/routers/labels"; import * as aiProvider from "@server/routers/aiProvider"; +import * as aiBudget from "@server/routers/aiBudget"; // Root routes export const unauthenticated = Router(); @@ -1586,6 +1588,44 @@ authenticated.delete( aiProvider.deleteAiModel ); +authenticated.put( + "/org/:orgId/ai-budget", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.createAiBudget), + logActionAudit(ActionsEnum.createAiBudget), + aiBudget.createAiBudget +); + +authenticated.get( + "/org/:orgId/ai-budgets", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.listAiBudgets), + aiBudget.listAiBudgets +); + +authenticated.get( + "/ai-budget/:budgetId", + verifyAiBudgetAccess, + verifyUserHasAction(ActionsEnum.getAiBudget), + aiBudget.getAiBudget +); + +authenticated.post( + "/ai-budget/:budgetId", + verifyAiBudgetAccess, + verifyUserHasAction(ActionsEnum.updateAiBudget), + logActionAudit(ActionsEnum.updateAiBudget), + aiBudget.updateAiBudget +); + +authenticated.delete( + "/ai-budget/:budgetId", + verifyAiBudgetAccess, + verifyUserHasAction(ActionsEnum.deleteAiBudget), + logActionAudit(ActionsEnum.deleteAiBudget), + aiBudget.deleteAiBudget +); + authenticated.get( "/org/:orgId/labels", verifyOrgAccess, From b180d064d18cf5fc63fd561699913088a8b9358c Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 11:39:34 -0400 Subject: [PATCH 091/296] adjust conflict to be on unit<>period and add list endpoints for providers --- server/db/pg/schema/schema.ts | 13 ++- server/db/sqlite/schema/schema.ts | 13 ++- server/routers/aiBudget/createAiBudget.ts | 47 +++++++---- server/routers/aiBudget/index.ts | 5 ++ .../routers/aiBudget/listAiBudgetsForModel.ts | 69 ++++++++++++++++ .../aiBudget/listAiBudgetsForProvider.ts | 69 ++++++++++++++++ .../aiBudget/listAiBudgetsForResource.ts | 69 ++++++++++++++++ .../routers/aiBudget/listAiBudgetsForRole.ts | 69 ++++++++++++++++ .../aiBudget/listAiBudgetsForSiteResource.ts | 69 ++++++++++++++++ server/routers/aiBudget/types.ts | 4 + server/routers/aiBudget/updateAiBudget.ts | 80 ++++++++++--------- server/routers/external.ts | 35 ++++++++ 12 files changed, 479 insertions(+), 63 deletions(-) create mode 100644 server/routers/aiBudget/listAiBudgetsForModel.ts create mode 100644 server/routers/aiBudget/listAiBudgetsForProvider.ts create mode 100644 server/routers/aiBudget/listAiBudgetsForResource.ts create mode 100644 server/routers/aiBudget/listAiBudgetsForRole.ts create mode 100644 server/routers/aiBudget/listAiBudgetsForSiteResource.ts diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index e2eb50205..7ded27eb7 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1746,10 +1746,15 @@ export const aiBudgets = pgTable( updatedAt: bigint("updatedAt", { mode: "number" }).notNull() }, (t) => [ - unique("ai_budget_provider_uniq").on(t.providerId), - unique("ai_budget_model_uniq").on(t.modelId), - unique("ai_budget_resource_uniq").on(t.resourceId), - unique("ai_budget_site_resource_uniq").on(t.siteResourceId) + unique("ai_budget_provider_uniq").on(t.providerId, t.unit, t.period), + unique("ai_budget_model_uniq").on(t.modelId, t.unit, t.period), + unique("ai_budget_resource_uniq").on(t.resourceId, t.unit, t.period), + unique("ai_budget_site_resource_uniq").on( + t.siteResourceId, + t.unit, + t.period + ), + unique("ai_budget_role_uniq").on(t.roleId, t.unit, t.period) ] ); diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index c8d0042df..34fe67a0f 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1732,10 +1732,15 @@ export const aiBudgets = sqliteTable( updatedAt: integer("updatedAt").notNull() }, (t) => [ - unique("ai_budget_provider_uniq").on(t.providerId), - unique("ai_budget_model_uniq").on(t.modelId), - unique("ai_budget_resource_uniq").on(t.resourceId), - unique("ai_budget_site_resource_uniq").on(t.siteResourceId) + unique("ai_budget_provider_uniq").on(t.providerId, t.unit, t.period), + unique("ai_budget_model_uniq").on(t.modelId, t.unit, t.period), + unique("ai_budget_resource_uniq").on(t.resourceId, t.unit, t.period), + unique("ai_budget_site_resource_uniq").on( + t.siteResourceId, + t.unit, + t.period + ), + unique("ai_budget_role_uniq").on(t.roleId, t.unit, t.period) ] ); diff --git a/server/routers/aiBudget/createAiBudget.ts b/server/routers/aiBudget/createAiBudget.ts index 705167c3a..3ddd76174 100644 --- a/server/routers/aiBudget/createAiBudget.ts +++ b/server/routers/aiBudget/createAiBudget.ts @@ -15,7 +15,7 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { eq } from "drizzle-orm"; +import { and, eq, isNull } from "drizzle-orm"; import type { CreateOrEditAiBudgetResponse } from "@server/routers/aiBudget/types"; import { aiBudgetEnforcementSchema, @@ -189,7 +189,7 @@ export async function createAiBudget( } } - const conflictCondition = + const scopeCondition = providerId !== undefined ? eq(aiBudgets.providerId, providerId) : modelId !== undefined @@ -198,22 +198,35 @@ export async function createAiBudget( ? eq(aiBudgets.resourceId, resourceId) : siteResourceId !== undefined ? eq(aiBudgets.siteResourceId, siteResourceId) - : undefined; + : roleId !== undefined + ? eq(aiBudgets.roleId, roleId) + : and( + eq(aiBudgets.orgId, orgId), + isNull(aiBudgets.providerId), + isNull(aiBudgets.modelId), + isNull(aiBudgets.resourceId), + isNull(aiBudgets.siteResourceId), + isNull(aiBudgets.roleId) + ); - if (conflictCondition) { - const [existing] = await db - .select({ budgetId: aiBudgets.budgetId }) - .from(aiBudgets) - .where(conflictCondition) - .limit(1); - if (existing) { - return next( - createHttpError( - HttpCode.CONFLICT, - "A budget already exists for this scope" - ) - ); - } + const [existing] = await db + .select({ budgetId: aiBudgets.budgetId }) + .from(aiBudgets) + .where( + and( + scopeCondition, + eq(aiBudgets.unit, unit), + eq(aiBudgets.period, period) + ) + ) + .limit(1); + if (existing) { + return next( + createHttpError( + HttpCode.CONFLICT, + `A ${period} ${unit} budget already exists for this scope` + ) + ); } const now = Date.now(); diff --git a/server/routers/aiBudget/index.ts b/server/routers/aiBudget/index.ts index 0da53e0d2..36eed7ba4 100644 --- a/server/routers/aiBudget/index.ts +++ b/server/routers/aiBudget/index.ts @@ -1,5 +1,10 @@ export * from "./createAiBudget"; export * from "./listAiBudgets"; +export * from "./listAiBudgetsForProvider"; +export * from "./listAiBudgetsForModel"; +export * from "./listAiBudgetsForResource"; +export * from "./listAiBudgetsForSiteResource"; +export * from "./listAiBudgetsForRole"; export * from "./getAiBudget"; export * from "./updateAiBudget"; export * from "./deleteAiBudget"; diff --git a/server/routers/aiBudget/listAiBudgetsForModel.ts b/server/routers/aiBudget/listAiBudgetsForModel.ts new file mode 100644 index 000000000..4a2277b5b --- /dev/null +++ b/server/routers/aiBudget/listAiBudgetsForModel.ts @@ -0,0 +1,69 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiBudgets, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { asc, eq } from "drizzle-orm"; +import type { ListAiBudgetsByScopeResponse } from "@server/routers/aiBudget/types"; + +const paramsSchema = z.strictObject({ + modelId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "get", + path: "/ai-model/{modelId}/ai-budgets", + description: "List AI budgets scoped to an AI model.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listAiBudgetsForModel( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { modelId } = parsedParams.data; + + const budgets = await db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.modelId, modelId)) + .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + + return response(res, { + data: { budgets }, + success: true, + error: false, + message: "AI budgets retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/listAiBudgetsForProvider.ts b/server/routers/aiBudget/listAiBudgetsForProvider.ts new file mode 100644 index 000000000..1a0625f4b --- /dev/null +++ b/server/routers/aiBudget/listAiBudgetsForProvider.ts @@ -0,0 +1,69 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiBudgets, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { asc, eq } from "drizzle-orm"; +import type { ListAiBudgetsByScopeResponse } from "@server/routers/aiBudget/types"; + +const paramsSchema = z.strictObject({ + providerId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "get", + path: "/ai-provider/{providerId}/ai-budgets", + description: "List AI budgets scoped to an AI provider.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listAiBudgetsForProvider( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { providerId } = parsedParams.data; + + const budgets = await db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.providerId, providerId)) + .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + + return response(res, { + data: { budgets }, + success: true, + error: false, + message: "AI budgets retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/listAiBudgetsForResource.ts b/server/routers/aiBudget/listAiBudgetsForResource.ts new file mode 100644 index 000000000..106b049a7 --- /dev/null +++ b/server/routers/aiBudget/listAiBudgetsForResource.ts @@ -0,0 +1,69 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiBudgets, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { asc, eq } from "drizzle-orm"; +import type { ListAiBudgetsByScopeResponse } from "@server/routers/aiBudget/types"; + +const paramsSchema = z.strictObject({ + resourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "get", + path: "/resource/{resourceId}/ai-budgets", + description: "List AI budgets scoped to a resource.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listAiBudgetsForResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { resourceId } = parsedParams.data; + + const budgets = await db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.resourceId, resourceId)) + .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + + return response(res, { + data: { budgets }, + success: true, + error: false, + message: "AI budgets retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/listAiBudgetsForRole.ts b/server/routers/aiBudget/listAiBudgetsForRole.ts new file mode 100644 index 000000000..2e8ff0e8d --- /dev/null +++ b/server/routers/aiBudget/listAiBudgetsForRole.ts @@ -0,0 +1,69 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiBudgets, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { asc, eq } from "drizzle-orm"; +import type { ListAiBudgetsByScopeResponse } from "@server/routers/aiBudget/types"; + +const paramsSchema = z.strictObject({ + roleId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "get", + path: "/role/{roleId}/ai-budgets", + description: "List AI budgets scoped to a role.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listAiBudgetsForRole( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { roleId } = parsedParams.data; + + const budgets = await db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.roleId, roleId)) + .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + + return response(res, { + data: { budgets }, + success: true, + error: false, + message: "AI budgets retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/listAiBudgetsForSiteResource.ts b/server/routers/aiBudget/listAiBudgetsForSiteResource.ts new file mode 100644 index 000000000..b2557719d --- /dev/null +++ b/server/routers/aiBudget/listAiBudgetsForSiteResource.ts @@ -0,0 +1,69 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiBudgets, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { asc, eq } from "drizzle-orm"; +import type { ListAiBudgetsByScopeResponse } from "@server/routers/aiBudget/types"; + +const paramsSchema = z.strictObject({ + siteResourceId: z.coerce.number().int().positive() +}); + +registry.registerPath({ + method: "get", + path: "/site-resource/{siteResourceId}/ai-budgets", + description: "List AI budgets scoped to a site resource.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listAiBudgetsForSiteResource( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteResourceId } = parsedParams.data; + + const budgets = await db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.siteResourceId, siteResourceId)) + .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + + return response(res, { + data: { budgets }, + success: true, + error: false, + message: "AI budgets retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/types.ts b/server/routers/aiBudget/types.ts index 11ffd1050..c689ef30e 100644 --- a/server/routers/aiBudget/types.ts +++ b/server/routers/aiBudget/types.ts @@ -5,6 +5,10 @@ export type ListAiBudgetsResponse = PaginatedResponse<{ budgets: AiBudget[]; }>; +export type ListAiBudgetsByScopeResponse = { + budgets: AiBudget[]; +}; + export type GetAiBudgetResponse = { budget: AiBudget; }; diff --git a/server/routers/aiBudget/updateAiBudget.ts b/server/routers/aiBudget/updateAiBudget.ts index e8bd845c5..723b346c7 100644 --- a/server/routers/aiBudget/updateAiBudget.ts +++ b/server/routers/aiBudget/updateAiBudget.ts @@ -15,7 +15,7 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { and, eq, ne } from "drizzle-orm"; +import { and, eq, isNull, ne } from "drizzle-orm"; import type { CreateOrEditAiBudgetResponse } from "@server/routers/aiBudget/types"; import { aiBudgetEnforcementSchema, @@ -128,6 +128,9 @@ export async function updateAiBudget( : existing.siteResourceId; const nextRoleId = body.roleId !== undefined ? body.roleId : existing.roleId; + const nextUnit = body.unit !== undefined ? body.unit : existing.unit; + const nextPeriod = + body.period !== undefined ? body.period : existing.period; const scopeValidation = z .object({ @@ -242,44 +245,45 @@ export async function updateAiBudget( } } - const conflictCondition = - body.providerId !== undefined && body.providerId !== null - ? and( - eq(aiBudgets.providerId, body.providerId), - ne(aiBudgets.budgetId, budgetId) - ) - : body.modelId !== undefined && body.modelId !== null - ? and( - eq(aiBudgets.modelId, body.modelId), - ne(aiBudgets.budgetId, budgetId) - ) - : body.resourceId !== undefined && body.resourceId !== null - ? and( - eq(aiBudgets.resourceId, body.resourceId), - ne(aiBudgets.budgetId, budgetId) - ) - : body.siteResourceId !== undefined && - body.siteResourceId !== null - ? and( - eq(aiBudgets.siteResourceId, body.siteResourceId), - ne(aiBudgets.budgetId, budgetId) - ) - : undefined; + const scopeCondition = + nextProviderId !== null + ? eq(aiBudgets.providerId, nextProviderId) + : nextModelId !== null + ? eq(aiBudgets.modelId, nextModelId) + : nextResourceId !== null + ? eq(aiBudgets.resourceId, nextResourceId) + : nextSiteResourceId !== null + ? eq(aiBudgets.siteResourceId, nextSiteResourceId) + : nextRoleId !== null + ? eq(aiBudgets.roleId, nextRoleId) + : and( + eq(aiBudgets.orgId, orgId), + isNull(aiBudgets.providerId), + isNull(aiBudgets.modelId), + isNull(aiBudgets.resourceId), + isNull(aiBudgets.siteResourceId), + isNull(aiBudgets.roleId) + ); - if (conflictCondition) { - const [conflict] = await db - .select({ budgetId: aiBudgets.budgetId }) - .from(aiBudgets) - .where(conflictCondition) - .limit(1); - if (conflict) { - return next( - createHttpError( - HttpCode.CONFLICT, - "A budget already exists for this scope" - ) - ); - } + const [conflict] = await db + .select({ budgetId: aiBudgets.budgetId }) + .from(aiBudgets) + .where( + and( + scopeCondition, + eq(aiBudgets.unit, nextUnit), + eq(aiBudgets.period, nextPeriod), + ne(aiBudgets.budgetId, budgetId) + ) + ) + .limit(1); + if (conflict) { + return next( + createHttpError( + HttpCode.CONFLICT, + `A ${nextPeriod} ${nextUnit} budget already exists for this scope` + ) + ); } const updateData: Partial = { diff --git a/server/routers/external.ts b/server/routers/external.ts index f469707f8..41df6ea59 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1626,6 +1626,41 @@ authenticated.delete( aiBudget.deleteAiBudget ); +authenticated.get( + "/ai-provider/:providerId/ai-budgets", + verifyAiProviderAccess, + verifyUserHasAction(ActionsEnum.listAiBudgets), + aiBudget.listAiBudgetsForProvider +); + +authenticated.get( + "/ai-model/:modelId/ai-budgets", + verifyAiModelAccess, + verifyUserHasAction(ActionsEnum.listAiBudgets), + aiBudget.listAiBudgetsForModel +); + +authenticated.get( + "/resource/:resourceId/ai-budgets", + verifyResourceAccess, + verifyUserHasAction(ActionsEnum.listAiBudgets), + aiBudget.listAiBudgetsForResource +); + +authenticated.get( + "/site-resource/:siteResourceId/ai-budgets", + verifySiteResourceAccess, + verifyUserHasAction(ActionsEnum.listAiBudgets), + aiBudget.listAiBudgetsForSiteResource +); + +authenticated.get( + "/role/:roleId/ai-budgets", + verifyRoleAccess, + verifyUserHasAction(ActionsEnum.listAiBudgets), + aiBudget.listAiBudgetsForRole +); + authenticated.get( "/org/:orgId/labels", verifyOrgAccess, From 187936e5dd162617d56ddd54a386a72643b071e4 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Mon, 10 Aug 2026 11:54:37 -0400 Subject: [PATCH 092/296] use class with helper methods for model catelog --- server/lib/aiModelCatalog.ts | 295 +++++++++++++++++------ server/lib/aiModelPricing.ts | 47 +--- server/routers/site/socketIntegration.ts | 2 +- 3 files changed, 229 insertions(+), 115 deletions(-) diff --git a/server/lib/aiModelCatalog.ts b/server/lib/aiModelCatalog.ts index 39b3a70c9..faee7a345 100644 --- a/server/lib/aiModelCatalog.ts +++ b/server/lib/aiModelCatalog.ts @@ -3,13 +3,18 @@ import axios from "axios"; import config from "@server/lib/config"; import logger from "@server/logger"; -export type CatalogProvider = - | "openai" - | "anthropic" - | "gemini" - | "vertex" - | "azure" - | "bedrock"; +export const CATALOG_PROVIDERS = [ + "openai", + "anthropic", + "gemini", + "vertex", + "azure", + "bedrock" +] as const; + +export type CatalogProvider = (typeof CATALOG_PROVIDERS)[number]; + +const CATALOG_PROVIDER_SET = new Set(CATALOG_PROVIDERS); export type AiModelCatalogEntry = { provider: CatalogProvider; @@ -22,90 +27,222 @@ export type AiModelCatalogEntry = { }; }; -let catalog: AiModelCatalogEntry[] = []; -let refreshTimer: NodeJS.Timeout | null = null; +type RawCatalogEntry = { + id?: string; + name?: string; + model?: string; + provider: string; + input_cost_per_token?: number | null; + output_cost_per_token?: number | null; + cache_read_input_token_cost?: number | null; + output_cost_per_reasoning_token?: number | null; + pricing?: { + input?: number | null; + output?: number | null; + cacheRead?: number | null; + reasoningOutput?: number | null; + }; +}; -async function fetchFromFile(filePath: string): Promise { - try { - if (!fs.existsSync(filePath)) { +function normalizeCatalogProvider(raw: string): CatalogProvider | null { + if (CATALOG_PROVIDER_SET.has(raw)) { + return raw as CatalogProvider; + } + if (raw.startsWith("bedrock")) { + return "bedrock"; + } + if (raw.startsWith("vertex")) { + return "vertex"; + } + if (raw.startsWith("azure")) { + return "azure"; + } + return null; +} + +function normalizeEntry(raw: RawCatalogEntry): AiModelCatalogEntry | null { + const provider = normalizeCatalogProvider(raw.provider); + if (!provider) { + return null; + } + + const model = raw.model ?? raw.name ?? raw.id; + if (!model) { + return null; + } + + return { + provider, + model, + pricing: { + input: raw.pricing?.input ?? raw.input_cost_per_token ?? null, + output: raw.pricing?.output ?? raw.output_cost_per_token ?? null, + cacheRead: + raw.pricing?.cacheRead ?? + raw.cache_read_input_token_cost ?? + null, + reasoningOutput: + raw.pricing?.reasoningOutput ?? + raw.output_cost_per_reasoning_token ?? + null + } + }; +} + +function providerKey(provider: CatalogProvider, key: string): string { + return `${provider}\0${key}`; +} + +export class AiModelCatalog { + private entries: AiModelCatalogEntry[] = []; + private byProvider = new Map(); + private byProviderAndKey = new Map(); + private byKey = new Map(); + private refreshTimer: NodeJS.Timeout | null = null; + + /** + * Loads the catalog into memory and schedules periodic background refreshes. + * Call once at server startup. + */ + async init(): Promise { + await this.refresh(); + this.scheduleNextRefresh(); + } + + /** Exact lookup by catalog provider and model key. */ + get( + provider: CatalogProvider, + key: string + ): AiModelCatalogEntry | undefined { + return this.byProviderAndKey.get(providerKey(provider, key)); + } + + /** All models for a catalog provider. */ + list(provider: CatalogProvider): AiModelCatalogEntry[] { + return this.byProvider.get(provider) ?? []; + } + + /** All catalog entries that share a model key, across providers. */ + listByKey(key: string): AiModelCatalogEntry[] { + return this.byKey.get(key) ?? []; + } + + /** Full in-memory catalog. */ + getAll(): AiModelCatalogEntry[] { + return this.entries; + } + + private setEntries(entries: AiModelCatalogEntry[]): void { + const byProvider = new Map(); + const byProviderAndKey = new Map(); + const byKey = new Map(); + + for (const entry of entries) { + const list = byProvider.get(entry.provider) ?? []; + list.push(entry); + byProvider.set(entry.provider, list); + + const mapKey = providerKey(entry.provider, entry.model); + if (!byProviderAndKey.has(mapKey)) { + byProviderAndKey.set(mapKey, entry); + } + + const keyList = byKey.get(entry.model) ?? []; + keyList.push(entry); + byKey.set(entry.model, keyList); + } + + this.entries = entries; + this.byProvider = byProvider; + this.byProviderAndKey = byProviderAndKey; + this.byKey = byKey; + } + + private async fetchFromFile( + filePath: string + ): Promise { + try { + if (!fs.existsSync(filePath)) { + logger.warn( + `AI model catalog file not found at ${filePath}; cost calculation will fall back to unknown pricing` + ); + return null; + } + const raw = fs.readFileSync(filePath, "utf-8"); + const parsed = JSON.parse(raw) as { data: RawCatalogEntry[] }; + return (parsed.data ?? []) + .map(normalizeEntry) + .filter((e): e is AiModelCatalogEntry => e != null); + } catch (error) { + logger.warn("Failed to read AI model catalog file", { error }); + return null; + } + } + + private async fetchFromUpstream( + upstreamUrl: string + ): Promise { + try { + const res = await axios.get<{ data: RawCatalogEntry[] }>( + upstreamUrl, + { timeout: 15_000 } + ); + return (res.data?.data ?? []) + .map(normalizeEntry) + .filter((e): e is AiModelCatalogEntry => e != null); + } catch (error: any) { logger.warn( - `AI model catalog file not found at ${filePath}; cost calculation will fall back to unknown pricing` + `Failed to fetch AI model catalog from ${upstreamUrl}: ${error.message || error}` ); return null; } - const raw = fs.readFileSync(filePath, "utf-8"); - const parsed = JSON.parse(raw) as { data: AiModelCatalogEntry[] }; - return parsed.data ?? []; - } catch (error) { - logger.warn("Failed to read AI model catalog file", { error }); - return null; + } + + private async refresh(): Promise { + const { file, upstream_url } = config.getRawConfig().ai.model_catalog; + + const fetched = file + ? await this.fetchFromFile(file) + : await this.fetchFromUpstream(upstream_url); + + if (fetched) { + this.setEntries(fetched); + logger.debug( + `AI model catalog refreshed: ${this.entries.length} models loaded` + ); + } else { + logger.debug( + "AI model catalog refresh failed; keeping previously loaded catalog in memory" + ); + } + } + + private scheduleNextRefresh(): void { + const { refresh_interval_min_hours, refresh_interval_max_hours } = + config.getRawConfig().ai.model_catalog; + + // Jittered rather than fixed so that many self-hosted instances don't + // all hit the upstream catalog endpoint at the same moment. + const minMs = refresh_interval_min_hours * 60 * 60 * 1000; + const maxMs = refresh_interval_max_hours * 60 * 60 * 1000; + const delayMs = minMs + Math.random() * Math.max(0, maxMs - minMs); + + if (this.refreshTimer) { + clearTimeout(this.refreshTimer); + } + this.refreshTimer = setTimeout(async () => { + await this.refresh(); + this.scheduleNextRefresh(); + }, delayMs); } } -async function fetchFromUpstream( - upstreamUrl: string -): Promise { - try { - const res = await axios.get<{ data: AiModelCatalogEntry[] }>( - upstreamUrl, - { timeout: 15_000 } - ); - return res.data?.data ?? []; - } catch (error: any) { - logger.warn( - `Failed to fetch AI model catalog from ${upstreamUrl}: ${error.message || error}` - ); - return null; - } -} - -async function refreshCatalog(): Promise { - const { file, upstream_url } = config.getRawConfig().ai.model_catalog; - - const fetched = file - ? await fetchFromFile(file) - : await fetchFromUpstream(upstream_url); - - if (fetched) { - catalog = fetched; - logger.debug( - `AI model catalog refreshed: ${catalog.length} models loaded` - ); - } else { - logger.debug( - "AI model catalog refresh failed; keeping previously loaded catalog in memory" - ); - } -} - -function scheduleNextRefresh(): void { - const { refresh_interval_min_hours, refresh_interval_max_hours } = - config.getRawConfig().ai.model_catalog; - - // Jittered rather than fixed so that many self-hosted instances don't - // all hit the upstream catalog endpoint at the same moment. - const minMs = refresh_interval_min_hours * 60 * 60 * 1000; - const maxMs = refresh_interval_max_hours * 60 * 60 * 1000; - const delayMs = minMs + Math.random() * Math.max(0, maxMs - minMs); - - if (refreshTimer) { - clearTimeout(refreshTimer); - } - refreshTimer = setTimeout(async () => { - await refreshCatalog(); - scheduleNextRefresh(); - }, delayMs); -} +export const aiModelCatalog = new AiModelCatalog(); /** * Loads the AI model pricing catalog into memory and schedules periodic * background refreshes. Call once at server startup. */ export async function initAiModelCatalog(): Promise { - await refreshCatalog(); - scheduleNextRefresh(); -} - -export function getAiModelCatalog(): AiModelCatalogEntry[] { - return catalog; + await aiModelCatalog.init(); } diff --git a/server/lib/aiModelPricing.ts b/server/lib/aiModelPricing.ts index 873e4213a..4cc398068 100644 --- a/server/lib/aiModelPricing.ts +++ b/server/lib/aiModelPricing.ts @@ -1,7 +1,7 @@ import type { AiProviderType } from "@server/lib/aiProviderDefaults"; import type { AiUsage } from "@server/lib/aiUsageExtraction"; import { - getAiModelCatalog, + aiModelCatalog, type AiModelCatalogEntry, type CatalogProvider } from "@server/lib/aiModelCatalog"; @@ -36,31 +36,6 @@ const PROVIDER_CATALOG_MAP: Record< vercelAiGateway: null }; -// Indexed view over the in-memory catalog, rebuilt only when -// getAiModelCatalog() returns a different array instance (i.e. after a -// background refresh swaps it out), not on every lookup. -let indexedCatalog: AiModelCatalogEntry[] | null = null; -let indexedByName: Map = new Map(); - -function getIndexedCatalog(): Map { - const catalog = getAiModelCatalog(); - if (catalog === indexedCatalog) { - return indexedByName; - } - - const byName = new Map(); - for (const entry of catalog) { - if (!entry.model) continue; - const list = byName.get(entry.model) ?? []; - list.push(entry); - byName.set(entry.model, list); - } - - indexedCatalog = catalog; - indexedByName = byName; - return byName; -} - function stripVendorPrefix(modelId: string): string | null { const idx = modelId.indexOf("/"); if (idx === -1 || idx === modelId.length - 1) { @@ -83,7 +58,6 @@ function toPricing( } function findEntry( - byName: Map, modelId: string, provider: CatalogProvider | null ): AiModelCatalogEntry | null { @@ -92,11 +66,15 @@ function findEntry( ); for (const key of candidates) { - const entries = byName.get(key); - if (!entries) continue; - const match = provider - ? entries.find((e) => e.provider === provider) - : entries[0]; + if (provider) { + const match = aiModelCatalog.get(provider, key); + if (match) { + return match; + } + continue; + } + + const match = aiModelCatalog.listByKey(key)[0]; if (match) { return match; } @@ -118,18 +96,17 @@ export function getModelPricing( return null; } - const byName = getIndexedCatalog(); const catalogProvider = providerType === "custom" ? null : PROVIDER_CATALOG_MAP[providerType]; if (catalogProvider) { - const scoped = findEntry(byName, modelId, catalogProvider); + const scoped = findEntry(modelId, catalogProvider); if (scoped) { return toPricing(scoped, false); } } - const fallback = findEntry(byName, modelId, null); + const fallback = findEntry(modelId, null); if (fallback) { return toPricing(fallback, true); } diff --git a/server/routers/site/socketIntegration.ts b/server/routers/site/socketIntegration.ts index 85d5bc9dd..bf63eb686 100644 --- a/server/routers/site/socketIntegration.ts +++ b/server/routers/site/socketIntegration.ts @@ -201,7 +201,7 @@ async function checkSocket( ): Promise<{ siteId: number; newtId: string }> { const { newt } = await getSiteAndNewt(siteId); - logger.info( + logger.debug( `Checking Docker socket for site ${siteId} with Newt ${newt.newtId}` ); From 211d3a53f5940781033f1bd9ef710654b6c108f1 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 12:03:46 -0400 Subject: [PATCH 093/296] initial budget ui on the provider --- messages/en-US.json | 19 +- server/routers/aiBudget/README.md | 126 ++++++ .../ai-providers/[providerId]/budget/page.tsx | 22 + .../ai-providers/[providerId]/layout.tsx | 4 + src/components/BudgetsEditor.tsx | 418 ++++++++++++++++++ src/lib/aiBudgetScope.ts | 70 +++ src/lib/queries.ts | 18 + 7 files changed, 676 insertions(+), 1 deletion(-) create mode 100644 server/routers/aiBudget/README.md create mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/budget/page.tsx create mode 100644 src/components/BudgetsEditor.tsx create mode 100644 src/lib/aiBudgetScope.ts diff --git a/messages/en-US.json b/messages/en-US.json index f18f99133..9acb93b45 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1653,6 +1653,23 @@ "aiProviderNetworkSettingsDescription": "Choose how traffic reaches this provider", "aiProviderAuthSettings": "Authentication", "aiProviderAuthSettingsDescription": "Configure how this provider authenticates requests to its upstream URL", + "aiProviderBudgetSettings": "Budget", + "aiProviderBudgetSettingsDescription": "Configure how this provider restricts usage based on spending or token limits", + "aiBudgetAdd": "Add Budget", + "aiBudgetEmpty": "No budgets configured yet. Click Add Budget to set a spending or token limit.", + "aiBudgetAmountPlaceholder": "Maximum spend", + "aiBudgetPeriodHourly": "Hourly", + "aiBudgetPeriodDaily": "Daily", + "aiBudgetPeriodWeekly": "Weekly", + "aiBudgetPeriodMonthly": "Monthly", + "aiBudgetPeriodYearly": "Yearly", + "aiBudgetPeriodLifetime": "Lifetime", + "aiBudgetUnitUsd": "USD", + "aiBudgetUnitTokens": "Tokens", + "aiBudgetConflictError": "A budget for this reset period and spend type already exists", + "aiBudgetInvalidAmountError": "Enter a maximum spend greater than 0", + "aiBudgetUpdated": "Budgets updated", + "aiBudgetErrorSave": "Failed to update budgets", "aiProviderType": "Provider Type", "aiProviderTypeSearch": "Search providers...", "aiProviderTypeNotFound": "No provider type found", @@ -4004,4 +4021,4 @@ "sessionToolbarShow": "Show toolbar", "sessionToolbarHide": "Hide toolbar", "actionUpdateSiteApprovals": "Update Site Approvals" -} +} \ No newline at end of file diff --git a/server/routers/aiBudget/README.md b/server/routers/aiBudget/README.md new file mode 100644 index 000000000..ef13e773d --- /dev/null +++ b/server/routers/aiBudget/README.md @@ -0,0 +1,126 @@ +# AI Budget API + +Public/OSS CRUD entity (`server/routers/aiBudget/`, not enterprise-gated). +Table: `aiBudgets` in `server/db/{pg,sqlite}/schema/schema.ts`, type `AiBudget`. + +## What a budget is + +A row is a spend/usage cap of `amount` `unit` (`usd` | `tokens`) per `period` +(`hourly` | `daily` | `weekly` | `monthly` | `yearly` | `lifetime`), with +`enforcement` (`hard` | `soft`) and an `enabled` flag. + +Every budget belongs to an org (`orgId`, required) and is optionally further +scoped to **exactly one** of: + +- `providerId` → an `aiProviders` row +- `modelId` → an `aiModels` row +- `resourceId` → a `resources` row +- `siteResourceId` → a `siteResources` row +- `roleId` → a `roles` row + +If none of those five are set, the budget is **org-wide**. Setting more than +one at once is rejected by `validation.ts`'s `refineBudgetScopeFields` +(`400`, "Only one of providerId, modelId, resourceId, siteResourceId, or +roleId may be set on a budget"). + +## Uniqueness / conflict rule + +A given scope (one specific provider, or model, or resource, or site +resource, or role, or "org-wide") may have **multiple** budgets, but at most +**one per `(unit, period)` combination** — e.g. one `weekly`/`usd` budget and +one `hourly`/`usd` budget can coexist on the same provider, but two +`weekly`/`usd` budgets cannot. This is enforced at two levels: + +- DB: composite `unique` constraints in both schema files — + `ai_budget_provider_uniq (providerId, unit, period)`, + `ai_budget_model_uniq (modelId, unit, period)`, + `ai_budget_resource_uniq (resourceId, unit, period)`, + `ai_budget_site_resource_uniq (siteResourceId, unit, period)`, + `ai_budget_role_uniq (roleId, unit, period)`. (NULL scope columns never + collide under a plain unique index, so this does *not* cover the org-wide + case — see next bullet.) +- App: `createAiBudget`/`updateAiBudget` both run an explicit pre-check + query keyed on `(scopeCondition, unit, period)` before insert/update, + where `scopeCondition` is `eq(, id)` for whichever scope + field is set, or — when none is set — `orgId = X AND` all five scope + columns `IS NULL`, so org-wide budgets get the same one-per-`(unit, + period)` guarantee even though the DB constraint can't express it. + Violating this returns `409` with + `` `A ${period} ${unit} budget already exists for this scope` ``. + +Because only one row can ever exist for a given `(scope, unit, period)`, +there is no separate check needed to prevent a `hard` and a `soft` budget +from coexisting on the same `(scope, unit, period)` — the conflict check +above already blocks the second row regardless of its `enforcement` value. + +On `updateAiBudget`, the conflict/ownership checks are run against the +**merged** next-state (existing row's scope/unit/period overlaid with +whatever the request body changes), not just the fields present in the +body — so e.g. changing only `unit` on a budget that already has +`providerId` set re-validates against that provider's other budgets at the +new unit. + +## Ownership validation + +`providerId`/`modelId`/`resourceId`/`siteResourceId`/`roleId` are validated +to belong to the same `orgId` as the budget (`modelId` via an +`aiModels ⋈ aiProviders` join, since `aiModels` has no `orgId` column +directly). A mismatch returns `404`, not `403` — this matches how the +sibling `aiProvider`/`aiModel` routers report cross-org references. + +## Routes + +All under `server/routers/external.ts`, registered right after the +`aiProvider`/`aiModel` block. `PUT` = create, `POST` = update (repo +convention, not standard REST). + +| Method | Path | Middleware | Action | Handler | +|---|---|---|---|---| +| PUT | `/org/:orgId/ai-budget` | `verifyOrgAccess` | `createAiBudget` | `createAiBudget` | +| GET | `/org/:orgId/ai-budgets` | `verifyOrgAccess` | `listAiBudgets` | `listAiBudgets` (paginated) | +| GET | `/ai-budget/:budgetId` | `verifyAiBudgetAccess` | `getAiBudget` | `getAiBudget` | +| POST | `/ai-budget/:budgetId` | `verifyAiBudgetAccess` | `updateAiBudget` | `updateAiBudget` | +| DELETE | `/ai-budget/:budgetId` | `verifyAiBudgetAccess` | `deleteAiBudget` | `deleteAiBudget` | +| GET | `/ai-provider/:providerId/ai-budgets` | `verifyAiProviderAccess` | `listAiBudgets` | `listAiBudgetsForProvider` | +| GET | `/ai-model/:modelId/ai-budgets` | `verifyAiModelAccess` | `listAiBudgets` | `listAiBudgetsForModel` | +| GET | `/resource/:resourceId/ai-budgets` | `verifyResourceAccess` | `listAiBudgets` | `listAiBudgetsForResource` | +| GET | `/site-resource/:siteResourceId/ai-budgets` | `verifySiteResourceAccess` | `listAiBudgets` | `listAiBudgetsForSiteResource` | +| GET | `/role/:roleId/ai-budgets` | `verifyRoleAccess` | `listAiBudgets` | `listAiBudgetsForRole` | + +The five scope-filtered `GET .../ai-budgets` routes intentionally reuse the +single `ActionsEnum.listAiBudgets` action rather than getting one action +each — access control is already fully handled by the entity-specific +middleware (a user who can see the provider/resource/etc. can see its +budgets), so per-scope actions would just be enum bloat. They also skip +pagination (unlike the org-wide list) since a single entity realistically +has only a handful of `(unit, period)` budgets — response shape is a flat +`{ budgets: AiBudget[] }` (`ListAiBudgetsByScopeResponse`), not +`PaginatedResponse`. + +`verifyAiBudgetAccess` (`server/middlewares/verifyAiBudgetAccess.ts`) loads +the budget by `budgetId`, resolves its `orgId` directly off the row (no +join needed, unlike `verifyAiModelAccess`), and stashes it on +`req.aiBudget` so `getAiBudget`/`updateAiBudget` can skip a re-fetch. + +## Request/response shapes + +- Create body: `providerId?`, `modelId?`, `resourceId?`, `siteResourceId?`, + `roleId?` (all `number`, mutually exclusive), `amount` (positive + `number`, required), `unit` (required), `period` (default `"monthly"`), + `enforcement` (default `"hard"`), `enabled?` (default `true`). +- Update body: same fields, all optional; the five scope fields are + `nullable().optional()` so a client can explicitly send `null` to clear + a scope (turning a scoped budget into an org-wide one). +- All five CRUD responses wrap a single `budget: AiBudget` (or + `budgets: AiBudget[]` + `pagination` for the org-wide list). No public/ + private mapper exists for `AiBudget` — unlike `AiProvider`, there's no + secret field to strip, so the raw DB row is returned as-is. + +## Not yet migrated + +Schema changes here (composite unique constraints) were made directly in +`schema.ts` without hand-writing a `server/migrations/*.sql` file — this +repo's CI (`.github/workflows/test.yml`) runs `drizzle-kit generate` +against `schema.ts` fresh, and other recent schema-only commits (e.g. "Remove +budget periods") follow the same pattern of not committing a matching +migration by hand. diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/budget/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/budget/page.tsx new file mode 100644 index 000000000..f86abbfd8 --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/budget/page.tsx @@ -0,0 +1,22 @@ +"use client"; + +import { SettingsContainer } from "@app/components/Settings"; +import { BudgetsEditor } from "@app/components/BudgetsEditor"; +import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; +import { useTranslations } from "next-intl"; + +export default function AiProviderBudgetPage() { + const { provider } = useAiProviderContext(); + const t = useTranslations(); + + return ( + + + + ); +} diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx index 4647347f5..01841d925 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/layout.tsx @@ -76,6 +76,10 @@ export default async function AiProviderLayout({ children, params }: Props) { { title: t("aiProviderAuthSettings"), href: "/{orgId}/settings/ai-providers/{providerId}/authentication" + }, + { + title: t("aiProviderBudgetSettings"), + href: "/{orgId}/settings/ai-providers/{providerId}/budget" } ]; diff --git a/src/components/BudgetsEditor.tsx b/src/components/BudgetsEditor.tsx new file mode 100644 index 000000000..1327998cf --- /dev/null +++ b/src/components/BudgetsEditor.tsx @@ -0,0 +1,418 @@ +"use client"; + +import { + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionFooter, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { Button } from "@app/components/ui/button"; +import { Input } from "@app/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue +} from "@app/components/ui/select"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { + AI_BUDGET_PERIODS, + AI_BUDGET_UNITS, + getAiBudgetScopeBodyField, + type AiBudgetPeriod, + type AiBudgetScope, + type AiBudgetUnit +} from "@app/lib/aiBudgetScope"; +import { aiBudgetQueries } from "@app/lib/queries"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import type { AiBudget } from "@server/db"; +import { Plus, Trash2 } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useEffect, useMemo, useState } from "react"; + +type BudgetRow = { + key: string; + budgetId?: number; + amount: string; + unit: AiBudgetUnit; + period: AiBudgetPeriod; +}; + +function rowsFromBudgets(budgets: AiBudget[]): BudgetRow[] { + return budgets.map((budget) => ({ + key: String(budget.budgetId), + budgetId: budget.budgetId, + amount: String(budget.amount), + unit: budget.unit, + period: budget.period + })); +} + +function comboKey(unit: AiBudgetUnit, period: AiBudgetPeriod): string { + return `${unit}:${period}`; +} + +function nextAvailableCombo(rows: BudgetRow[]): { + unit: AiBudgetUnit; + period: AiBudgetPeriod; +} { + const used = new Set(rows.map((row) => comboKey(row.unit, row.period))); + for (const unit of AI_BUDGET_UNITS) { + for (const period of AI_BUDGET_PERIODS) { + if (!used.has(comboKey(unit, period))) { + return { unit, period }; + } + } + } + return { unit: "usd", period: "monthly" }; +} + +export function BudgetsEditor({ + scope, + orgId, + title, + description +}: { + scope: AiBudgetScope; + orgId: string; + title: string; + description: string; +}) { + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const queryClient = useQueryClient(); + const t = useTranslations(); + + const [rows, setRows] = useState([]); + const [saveLoading, setSaveLoading] = useState(false); + const [attemptedSave, setAttemptedSave] = useState(false); + + const budgetsQuery = useQuery(aiBudgetQueries.scoped({ scope })); + + useEffect(() => { + if (!budgetsQuery.data) return; + setRows(rowsFromBudgets(budgetsQuery.data)); + setAttemptedSave(false); + }, [budgetsQuery.data]); + + const conflictingKeys = useMemo(() => { + const counts = new Map(); + for (const row of rows) { + const key = comboKey(row.unit, row.period); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + const conflicting = new Set(); + for (const row of rows) { + const key = comboKey(row.unit, row.period); + if ((counts.get(key) ?? 0) > 1) { + conflicting.add(row.key); + } + } + return conflicting; + }, [rows]); + + const invalidAmountKeys = useMemo(() => { + const invalid = new Set(); + for (const row of rows) { + const amount = Number(row.amount); + if (!row.amount.trim() || !Number.isFinite(amount) || amount <= 0) { + invalid.add(row.key); + } + } + return invalid; + }, [rows]); + + const hasErrors = conflictingKeys.size > 0 || invalidAmountKeys.size > 0; + + function addRow() { + const combo = nextAvailableCombo(rows); + setRows((prev) => [ + ...prev, + { + key: crypto.randomUUID(), + amount: "", + unit: combo.unit, + period: combo.period + } + ]); + } + + function removeRow(key: string) { + setRows((prev) => prev.filter((row) => row.key !== key)); + } + + function updateRow(key: string, patch: Partial) { + setRows((prev) => + prev.map((row) => (row.key === key ? { ...row, ...patch } : row)) + ); + } + + async function onSave() { + setAttemptedSave(true); + if (hasErrors) { + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: conflictingKeys.size + ? t("aiBudgetConflictError") + : t("aiBudgetInvalidAmountError") + }); + return; + } + + setSaveLoading(true); + try { + const existing = budgetsQuery.data ?? []; + const existingById = new Map( + existing.map((budget) => [budget.budgetId, budget]) + ); + const currentBudgetIds = new Set( + rows + .filter((row) => row.budgetId !== undefined) + .map((row) => row.budgetId as number) + ); + const bodyField = getAiBudgetScopeBodyField(scope); + + const toDelete = existing.filter( + (budget) => !currentBudgetIds.has(budget.budgetId) + ); + const toCreate = rows.filter((row) => row.budgetId === undefined); + const toUpdate = rows.filter((row) => { + if (row.budgetId === undefined) return false; + const existingBudget = existingById.get(row.budgetId); + if (!existingBudget) return false; + return ( + existingBudget.amount !== Number(row.amount) || + existingBudget.unit !== row.unit || + existingBudget.period !== row.period + ); + }); + + await Promise.all([ + ...toDelete.map((budget) => + api.delete(`/ai-budget/${budget.budgetId}`) + ), + ...toCreate.map((row) => + api.put(`/org/${orgId}/ai-budget`, { + [bodyField]: scope.id, + amount: Number(row.amount), + unit: row.unit, + period: row.period + }) + ), + ...toUpdate.map((row) => + api.post(`/ai-budget/${row.budgetId}`, { + amount: Number(row.amount), + unit: row.unit, + period: row.period + }) + ) + ]); + + await queryClient.invalidateQueries( + aiBudgetQueries.scoped({ scope }) + ); + + toast({ + title: t("success"), + description: t("aiBudgetUpdated") + }); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: formatAxiosError(e, t("aiBudgetErrorSave")) + }); + } finally { + setSaveLoading(false); + } + } + + const periodLabels: Record = { + hourly: t("aiBudgetPeriodHourly"), + daily: t("aiBudgetPeriodDaily"), + weekly: t("aiBudgetPeriodWeekly"), + monthly: t("aiBudgetPeriodMonthly"), + yearly: t("aiBudgetPeriodYearly"), + lifetime: t("aiBudgetPeriodLifetime") + }; + + const unitLabels: Record = { + usd: t("aiBudgetUnitUsd"), + tokens: t("aiBudgetUnitTokens") + }; + + return ( + + +
+
+ {title} + + {description} + +
+ +
+ + + + {rows.length === 0 ? ( +

+ {t("aiBudgetEmpty")} +

+ ) : ( +
+ {rows.map((row) => { + const showConflict = conflictingKeys.has(row.key); + const showInvalidAmount = + attemptedSave && + invalidAmountKeys.has(row.key); + return ( +
+
+ + updateRow(row.key, { + amount: e.target.value + }) + } + className="flex-1" + /> + + + +
+ {showConflict && ( +

+ {t("aiBudgetConflictError")} +

+ )} + {showInvalidAmount && ( +

+ {t("aiBudgetInvalidAmountError")} +

+ )} +
+ ); + })} +
+ )} + + + + + + + ); +} diff --git a/src/lib/aiBudgetScope.ts b/src/lib/aiBudgetScope.ts new file mode 100644 index 000000000..beabc85a3 --- /dev/null +++ b/src/lib/aiBudgetScope.ts @@ -0,0 +1,70 @@ +import type { AiBudget } from "@server/db"; + +export type AiBudgetScopeType = + | "provider" + | "model" + | "resource" + | "siteResource" + | "role"; + +export type AiBudgetScope = { + type: AiBudgetScopeType; + id: number; +}; + +export type AiBudgetScopeBodyField = + | "providerId" + | "modelId" + | "resourceId" + | "siteResourceId" + | "roleId"; + +const scopeConfig: Record< + AiBudgetScopeType, + { listPath: (id: number) => string; bodyField: AiBudgetScopeBodyField } +> = { + provider: { + listPath: (id) => `/ai-provider/${id}/ai-budgets`, + bodyField: "providerId" + }, + model: { + listPath: (id) => `/ai-model/${id}/ai-budgets`, + bodyField: "modelId" + }, + resource: { + listPath: (id) => `/resource/${id}/ai-budgets`, + bodyField: "resourceId" + }, + siteResource: { + listPath: (id) => `/site-resource/${id}/ai-budgets`, + bodyField: "siteResourceId" + }, + role: { + listPath: (id) => `/role/${id}/ai-budgets`, + bodyField: "roleId" + } +}; + +export function getAiBudgetScopeListPath(scope: AiBudgetScope): string { + return scopeConfig[scope.type].listPath(scope.id); +} + +export function getAiBudgetScopeBodyField( + scope: AiBudgetScope +): AiBudgetScopeBodyField { + return scopeConfig[scope.type].bodyField; +} + +export type AiBudgetUnit = AiBudget["unit"]; +export type AiBudgetPeriod = AiBudget["period"]; + +export const AI_BUDGET_UNITS: AiBudgetUnit[] = ["usd", "tokens"]; + +export const AI_BUDGET_PERIODS: AiBudgetPeriod[] = [ + "hourly", + "daily", + "weekly", + "monthly", + "yearly", + "lifetime" +]; diff --git a/src/lib/queries.ts b/src/lib/queries.ts index e1e06f928..868b3aa90 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -63,6 +63,11 @@ import type { ListAiModelsResponse, ListAiProvidersResponse } from "@server/routers/aiProvider/types"; +import type { ListAiBudgetsByScopeResponse } from "@server/routers/aiBudget/types"; +import { + getAiBudgetScopeListPath, + type AiBudgetScope +} from "@app/lib/aiBudgetScope"; import type { ListUsersResponse } from "@server/routers/user"; import type ResponseT from "@server/types/Response"; import { @@ -1213,6 +1218,19 @@ export const aiProviderQueries = { }) }; +export const aiBudgetQueries = { + scoped: ({ scope }: { scope: AiBudgetScope }) => + queryOptions({ + queryKey: ["AI_BUDGETS", scope.type, scope.id] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(getAiBudgetScopeListPath(scope), { signal }); + return res.data.data.budgets; + } + }) +}; + export const resourceQueries = { resourceUsers: ({ resourceId }: { resourceId: number }) => queryOptions({ From 7e6e0a8a9c8c3ce411ef4088423ad3e70da213f4 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 12:20:47 -0400 Subject: [PATCH 094/296] make the budget match the rules --- messages/en-US.json | 3 + src/components/BudgetsEditor.tsx | 387 +++++++++++++++++++------------ 2 files changed, 242 insertions(+), 148 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 9acb93b45..6dea651c9 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1657,6 +1657,9 @@ "aiProviderBudgetSettingsDescription": "Configure how this provider restricts usage based on spending or token limits", "aiBudgetAdd": "Add Budget", "aiBudgetEmpty": "No budgets configured yet. Click Add Budget to set a spending or token limit.", + "aiBudgetUnit": "Spend Type", + "aiBudgetPeriod": "Reset Period", + "aiBudgetAmount": "Maximum Spend", "aiBudgetAmountPlaceholder": "Maximum spend", "aiBudgetPeriodHourly": "Hourly", "aiBudgetPeriodDaily": "Daily", diff --git a/src/components/BudgetsEditor.tsx b/src/components/BudgetsEditor.tsx index 1327998cf..edef2906b 100644 --- a/src/components/BudgetsEditor.tsx +++ b/src/components/BudgetsEditor.tsx @@ -1,14 +1,18 @@ "use client"; import { + SettingsFormCell, + SettingsFormGrid, SettingsSection, SettingsSectionBody, SettingsSectionDescription, SettingsSectionFooter, + SettingsSectionForm, SettingsSectionHeader, SettingsSectionTitle } from "@app/components/Settings"; import { Button } from "@app/components/ui/button"; +import { DataTableEmptyState } from "@app/components/ui/data-table-empty-state"; import { Input } from "@app/components/ui/input"; import { Select, @@ -17,6 +21,14 @@ import { SelectTrigger, SelectValue } from "@app/components/ui/select"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow +} from "@app/components/ui/table"; import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; @@ -247,160 +259,239 @@ export function BudgetsEditor({ tokens: t("aiBudgetUnitTokens") }; + const addRowButton = ( + + ); + return ( -
-
- {title} - - {description} - -
- -
+ {title} + + {description} + - {rows.length === 0 ? ( -

- {t("aiBudgetEmpty")} -

- ) : ( -
- {rows.map((row) => { - const showConflict = conflictingKeys.has(row.key); - const showInvalidAmount = - attemptedSave && - invalidAmountKeys.has(row.key); - return ( -
-
- - updateRow(row.key, { - amount: e.target.value - }) - } - className="flex-1" - /> - - - -
- {showConflict && ( -

- {t("aiBudgetConflictError")} -

- )} - {showInvalidAmount && ( -

- {t("aiBudgetInvalidAmountError")} -

- )} -
- ); - })} -
- )} + + + +
+
+ + + + {t("aiBudgetUnit")} + + + {t("aiBudgetPeriod")} + + + {t("aiBudgetAmount")} + + + + + + {rows.length === 0 ? ( + + ) : ( + rows.map((row) => { + const showConflict = + conflictingKeys.has( + row.key + ); + const showInvalidAmount = + attemptedSave && + invalidAmountKeys.has( + row.key + ); + return ( + + + + + + + + + + updateRow( + row.key, + { + amount: e + .target + .value + } + ) + } + className="w-full min-w-0" + /> + + + + + + ); + }) + )} + +
+ {(conflictingKeys.size > 0 || + (attemptedSave && + invalidAmountKeys.size > 0)) && ( +

+ {conflictingKeys.size > 0 + ? t("aiBudgetConflictError") + : t("aiBudgetInvalidAmountError")} +

+ )} + {rows.length > 0 && addRowButton} +
+ + + From 93dea0525ecbe5fd6f75ce507b4d2b69d3a48fa5 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Mon, 10 Aug 2026 12:32:00 -0400 Subject: [PATCH 095/296] basic provider model selector --- messages/en-US.json | 8 +- server/lib/aiModelCatalog.ts | 27 ++++ server/lib/aiModelPricing.ts | 21 +-- server/routers/aiProvider/index.ts | 1 + .../routers/aiProvider/listCatalogModels.ts | 131 ++++++++++++++++++ server/routers/aiProvider/types.ts | 4 + server/routers/external.ts | 7 + .../ai-providers/[providerId]/models/page.tsx | 24 +++- src/lib/queries.ts | 13 +- 9 files changed, 209 insertions(+), 27 deletions(-) create mode 100644 server/routers/aiProvider/listCatalogModels.ts diff --git a/messages/en-US.json b/messages/en-US.json index 9acb93b45..da08b4a57 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1776,14 +1776,14 @@ "aiProviderMessageRemove": "This will permanently delete the provider and its models and targets. This cannot be undone.", "aiProviderErrorNoUpdate": "AI provider is not available to update", "aiProviderModels": "Models", - "aiProviderModelsDescription": "Define allow and block patterns for this provider. Requests must match an allow pattern and must not match a block pattern. Use * and ? as wildcards (for example gpt-4* or claude-?). An empty allow list denies all models.", - "aiProviderModelsPlaceholder": "Model name or pattern (e.g. gpt-4*)", + "aiProviderModelsDescription": "Define allow and block patterns for this provider. Requests must match an allow pattern and must not match a block pattern. Pick from known models for this provider, or type a custom key or wildcard (for example gpt-4* or claude-?). An empty allow list denies all models.", + "aiProviderModelsPlaceholder": "Select a model or type a pattern (e.g. gpt-4*)", "aiProviderModelsAllow": "Allow List", "aiProviderModelsAllowDescription": "Models that may be used through this provider. Empty means deny all.", - "aiProviderModelsAllowPlaceholder": "Allowed model or pattern (e.g. gpt-4*)", + "aiProviderModelsAllowPlaceholder": "Select a model or type a pattern (e.g. gpt-4*)", "aiProviderModelsBlock": "Block List", "aiProviderModelsBlockDescription": "Models to deny even if they match an allow pattern.", - "aiProviderModelsBlockPlaceholder": "Blocked model or pattern (e.g. gpt-4o-mini)", + "aiProviderModelsBlockPlaceholder": "Select a model or type a pattern (e.g. gpt-4o-mini)", "aiProviderModelsOverlapError": "These patterns cannot be on both lists: {keys}", "aiProviderModelsUpdated": "Models updated", "aiProviderModelsErrorUpdate": "Failed to update models", diff --git a/server/lib/aiModelCatalog.ts b/server/lib/aiModelCatalog.ts index faee7a345..9986366fb 100644 --- a/server/lib/aiModelCatalog.ts +++ b/server/lib/aiModelCatalog.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import axios from "axios"; import config from "@server/lib/config"; import logger from "@server/logger"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; export const CATALOG_PROVIDERS = [ "openai", @@ -16,6 +17,32 @@ export type CatalogProvider = (typeof CATALOG_PROVIDERS)[number]; const CATALOG_PROVIDER_SET = new Set(CATALOG_PROVIDERS); +// Each of our provider types maps to at most one catalog provider. Provider +// types that proxy arbitrary underlying models (openRouter, vercelAiGateway, +// custom) have no mapping. +const PROVIDER_CATALOG_MAP: Record< + Exclude, + CatalogProvider | null +> = { + openai: "openai", + anthropic: "anthropic", + googleGemini: "gemini", + vertexAi: "vertex", + bedrock: "bedrock", + microsoftFoundry: "azure", + openRouter: null, + vercelAiGateway: null +}; + +export function getCatalogProviderForType( + type: AiProviderType +): CatalogProvider | null { + if (type === "custom") { + return null; + } + return PROVIDER_CATALOG_MAP[type]; +} + export type AiModelCatalogEntry = { provider: CatalogProvider; model: string; diff --git a/server/lib/aiModelPricing.ts b/server/lib/aiModelPricing.ts index 4cc398068..569bfa42e 100644 --- a/server/lib/aiModelPricing.ts +++ b/server/lib/aiModelPricing.ts @@ -2,6 +2,7 @@ import type { AiProviderType } from "@server/lib/aiProviderDefaults"; import type { AiUsage } from "@server/lib/aiUsageExtraction"; import { aiModelCatalog, + getCatalogProviderForType, type AiModelCatalogEntry, type CatalogProvider } from "@server/lib/aiModelCatalog"; @@ -19,23 +20,6 @@ export type AiModelPricing = { approximate: boolean; }; -// Each of our provider types maps to at most one catalog provider. Provider types that proxy -// arbitrary underlying models (openRouter, vercelAiGateway, custom) have no -// mapping and always fall back to a global search. -const PROVIDER_CATALOG_MAP: Record< - Exclude, - CatalogProvider | null -> = { - openai: "openai", - anthropic: "anthropic", - googleGemini: "gemini", - vertexAi: "vertex", - bedrock: "bedrock", - microsoftFoundry: "azure", - openRouter: null, - vercelAiGateway: null -}; - function stripVendorPrefix(modelId: string): string | null { const idx = modelId.indexOf("/"); if (idx === -1 || idx === modelId.length - 1) { @@ -96,8 +80,7 @@ export function getModelPricing( return null; } - const catalogProvider = - providerType === "custom" ? null : PROVIDER_CATALOG_MAP[providerType]; + const catalogProvider = getCatalogProviderForType(providerType); if (catalogProvider) { const scoped = findEntry(modelId, catalogProvider); diff --git a/server/routers/aiProvider/index.ts b/server/routers/aiProvider/index.ts index 34c900139..835991fbd 100644 --- a/server/routers/aiProvider/index.ts +++ b/server/routers/aiProvider/index.ts @@ -5,6 +5,7 @@ export * from "./updateAiProvider"; export * from "./deleteAiProvider"; export * from "./createAiModel"; export * from "./listAiModels"; +export * from "./listCatalogModels"; export * from "./getAiModel"; export * from "./updateAiModel"; export * from "./deleteAiModel"; diff --git a/server/routers/aiProvider/listCatalogModels.ts b/server/routers/aiProvider/listCatalogModels.ts new file mode 100644 index 000000000..9a3241276 --- /dev/null +++ b/server/routers/aiProvider/listCatalogModels.ts @@ -0,0 +1,131 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiProviders, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { eq } from "drizzle-orm"; +import { + aiModelCatalog, + getCatalogProviderForType +} from "@server/lib/aiModelCatalog"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import type { ListCatalogModelsResponse } from "@server/routers/aiProvider/types"; + +const paramsSchema = z.strictObject({ + providerId: z.coerce.number().int().positive() +}); + +const listSchema = z.object({ + query: z.string().optional() +}); + +registry.registerPath({ + method: "get", + path: "/ai-provider/{providerId}/catalog-models", + description: + "List known catalog models for an AI provider's type. Used for model key suggestions.", + tags: [OpenAPITags.AiModel], + request: { + params: paramsSchema, + query: listSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listCatalogModels( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = listSchema.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error).toString() + ) + ); + } + + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { providerId } = parsedParams.data; + + const [provider] = + req.aiProvider && req.aiProvider.providerId === providerId + ? [req.aiProvider] + : await db + .select() + .from(aiProviders) + .where(eq(aiProviders.providerId, providerId)) + .limit(1); + + if (!provider) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `AI provider with ID ${providerId} not found` + ) + ); + } + + const catalogProvider = getCatalogProviderForType( + provider.type as AiProviderType + ); + + let models = catalogProvider + ? aiModelCatalog.list(catalogProvider).map((entry) => ({ + model: entry.model + })) + : []; + + const { query } = parsedQuery.data; + if (query) { + const q = query.toLowerCase(); + models = models.filter((m) => m.model.toLowerCase().includes(q)); + } + + // Deduplicate model keys (catalog may have duplicates after provider + // normalization, e.g. bedrock + bedrock_converse). + const seen = new Set(); + models = models.filter((m) => { + if (seen.has(m.model)) { + return false; + } + seen.add(m.model); + return true; + }); + + models.sort((a, b) => a.model.localeCompare(b.model)); + + return response(res, { + data: { models }, + success: true, + error: false, + message: "Catalog models retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiProvider/types.ts b/server/routers/aiProvider/types.ts index d2f314b30..c2b8e2ed1 100644 --- a/server/routers/aiProvider/types.ts +++ b/server/routers/aiProvider/types.ts @@ -39,6 +39,10 @@ export type ListAiModelsResponse = PaginatedResponse<{ models: AiModel[]; }>; +export type ListCatalogModelsResponse = { + models: { model: string }[]; +}; + export type GetAiModelResponse = { model: AiModel; }; diff --git a/server/routers/external.ts b/server/routers/external.ts index 41df6ea59..9e1a5f210 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1565,6 +1565,13 @@ authenticated.get( aiProvider.listAiModels ); +authenticated.get( + "/ai-provider/:providerId/catalog-models", + verifyAiProviderAccess, + verifyUserHasAction(ActionsEnum.listAiModels), + aiProvider.listCatalogModels +); + authenticated.get( "/ai-model/:modelId", verifyAiModelAccess, diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx index 10db41571..cba252dc8 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx @@ -20,7 +20,7 @@ import { createApiClient, formatAxiosError } from "@app/lib/api"; import { aiProviderQueries } from "@app/lib/queries"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useTranslations } from "next-intl"; -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; type ModelListType = "allow" | "block"; @@ -43,6 +43,18 @@ export default function AiProviderModelsPage() { const modelsQuery = useQuery( aiProviderQueries.providerModels({ providerId: provider.providerId }) ); + const catalogQuery = useQuery( + aiProviderQueries.catalogModels({ providerId: provider.providerId }) + ); + + const catalogTags = useMemo( + () => + (catalogQuery.data ?? []).map((entry) => ({ + id: entry.model, + text: entry.model + })), + [catalogQuery.data] + ); useEffect(() => { if (!modelsQuery.data) return; @@ -165,6 +177,8 @@ export default function AiProviderModelsPage() { } } + const inputsDisabled = modelsQuery.isLoading || saveLoading; + return ( @@ -196,10 +210,12 @@ export default function AiProviderModelsPage() { : newTags; setAllowTags(next as Tag[]); }} + enableAutocomplete={catalogTags.length > 0} + autocompleteOptions={catalogTags} allowDuplicates={false} sortTags delimiterList={[",", "Enter"]} - disabled={modelsQuery.isLoading || saveLoading} + disabled={inputsDisabled} />

{t("aiProviderModelsAllowDescription")} @@ -223,10 +239,12 @@ export default function AiProviderModelsPage() { : newTags; setBlockTags(next as Tag[]); }} + enableAutocomplete={catalogTags.length > 0} + autocompleteOptions={catalogTags} allowDuplicates={false} sortTags delimiterList={[",", "Enter"]} - disabled={modelsQuery.isLoading || saveLoading} + disabled={inputsDisabled} />

{t("aiProviderModelsBlockDescription")} diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 868b3aa90..a7faae811 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -61,7 +61,8 @@ import type { GetSiteResourceResponse } from "@server/routers/siteResource/getSi import type { ListTargetsResponse } from "@server/routers/target"; import type { ListAiModelsResponse, - ListAiProvidersResponse + ListAiProvidersResponse, + ListCatalogModelsResponse } from "@server/routers/aiProvider/types"; import type { ListAiBudgetsByScopeResponse } from "@server/routers/aiBudget/types"; import { @@ -1199,6 +1200,16 @@ export const aiProviderQueries = { return res.data.data.models; } }), + catalogModels: ({ providerId }: { providerId: number }) => + queryOptions({ + queryKey: ["AI_PROVIDERS", providerId, "CATALOG_MODELS"] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/ai-provider/${providerId}/catalog-models`, { signal }); + return res.data.data.models; + } + }), orgProviders: ({ orgId, query }: { orgId: string; query?: string }) => queryOptions({ queryKey: ["AI_PROVIDERS", orgId, "LIST", query ?? ""] as const, From f2e7f83b3d8995178bb2ab3c0001f6480db9d593 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 13:55:05 -0400 Subject: [PATCH 096/296] match the budget picker to the rules picker --- src/components/BudgetsEditor.tsx | 370 ++++++++++++++----------------- 1 file changed, 161 insertions(+), 209 deletions(-) diff --git a/src/components/BudgetsEditor.tsx b/src/components/BudgetsEditor.tsx index edef2906b..2a8b73022 100644 --- a/src/components/BudgetsEditor.tsx +++ b/src/components/BudgetsEditor.tsx @@ -1,13 +1,10 @@ "use client"; import { - SettingsFormCell, - SettingsFormGrid, SettingsSection, SettingsSectionBody, SettingsSectionDescription, SettingsSectionFooter, - SettingsSectionForm, SettingsSectionHeader, SettingsSectionTitle } from "@app/components/Settings"; @@ -281,217 +278,172 @@ export function BudgetsEditor({ - - - -

- - - - - {t("aiBudgetUnit")} - - - {t("aiBudgetPeriod")} - - - {t("aiBudgetAmount")} - - - - - - {rows.length === 0 ? ( - - ) : ( - rows.map((row) => { - const showConflict = - conflictingKeys.has( - row.key - ); - const showInvalidAmount = - attemptedSave && - invalidAmountKeys.has( - row.key - ); - return ( - - -
+ + + {t("aiBudgetAmount")} + {t("aiBudgetUnit")} + {t("aiBudgetPeriod")} + + + + + {rows.length === 0 ? ( + + ) : ( + rows.map((row) => { + const showConflict = conflictingKeys.has( + row.key + ); + const showInvalidAmount = + attemptedSave && + invalidAmountKeys.has(row.key); + return ( + + + + updateRow(row.key, { + amount: e.target + .value + }) + } + className="w-full min-w-0" + /> + + + - - - + + + - - - - updateRow( - row.key, - { - amount: e - .target - .value - } - ) - } - className="w-full min-w-0" - /> - - - - - - ); - }) - )} - -
- {(conflictingKeys.size > 0 || - (attemptedSave && - invalidAmountKeys.size > 0)) && ( -

- {conflictingKeys.size > 0 - ? t("aiBudgetConflictError") - : t("aiBudgetInvalidAmountError")} -

- )} - {rows.length > 0 && addRowButton} -
- - - + ] + } + + ) + )} + + + + +
+ +
+ + + ); + }) + )} + + + {(conflictingKeys.size > 0 || + (attemptedSave && invalidAmountKeys.size > 0)) && ( +

+ {conflictingKeys.size > 0 + ? t("aiBudgetConflictError") + : t("aiBudgetInvalidAmountError")} +

+ )} + {rows.length > 0 && addRowButton} +
From 52de5eb539207f1ffa14ef7bad245516558b3fb6 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 14:08:17 -0400 Subject: [PATCH 097/296] add the budget config to the resources --- messages/en-US.json | 2 + .../private/[niceId]/budget/page.tsx | 42 +++++++++++++++++++ .../resources/private/[niceId]/layout.tsx | 7 ++++ .../resources/public/[niceId]/budget/page.tsx | 37 ++++++++++++++++ .../resources/public/[niceId]/layout.tsx | 7 ++++ 5 files changed, 95 insertions(+) create mode 100644 src/app/[orgId]/settings/resources/private/[niceId]/budget/page.tsx create mode 100644 src/app/[orgId]/settings/resources/public/[niceId]/budget/page.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 5b9eb0ce5..646a82664 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1817,6 +1817,8 @@ "aiResourceAliasRequired": "Alias is required for inference resources", "aiResourceDomainConfiguration": "Domain configuration", "aiResourceDomainConfigurationDescription": "Choose the domain clients will use to reach this inference resource.", + "resourceBudgetSettings": "Budget", + "resourceBudgetSettingsDescription": "Configure how this inference resource restricts usage based on spending or token limits", "sidebarApiKeys": "API Keys", "sidebarProvisioning": "Provisioning", "sidebarSettings": "Settings", diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/budget/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/budget/page.tsx new file mode 100644 index 000000000..9268d664e --- /dev/null +++ b/src/app/[orgId]/settings/resources/private/[niceId]/budget/page.tsx @@ -0,0 +1,42 @@ +"use client"; + +import { SettingsContainer } from "@app/components/Settings"; +import { BudgetsEditor } from "@app/components/BudgetsEditor"; +import { useSiteResourceContext } from "@app/hooks/useSiteResourceContext"; +import { useTranslations } from "next-intl"; +import { useRouter } from "next/navigation"; +import { useEffect } from "react"; + +export default function PrivateResourceBudgetPage() { + const { siteResource } = useSiteResourceContext(); + const router = useRouter(); + const t = useTranslations(); + + useEffect(() => { + if (siteResource.mode !== "inference") { + router.replace( + `/${siteResource.orgId}/settings/resources/private/${siteResource.niceId}/general` + ); + } + }, [ + router, + siteResource.mode, + siteResource.niceId, + siteResource.orgId + ]); + + if (siteResource.mode !== "inference") { + return null; + } + + return ( + + + + ); +} diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx index 294e8f81a..e4bb32ef7 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/layout.tsx @@ -70,6 +70,13 @@ export default async function PrivateResourceLayout( } ]; + if (siteResource.mode === "inference") { + navItems.push({ + title: t("resourceBudgetSettings"), + href: `/{orgId}/settings/resources/private/{niceId}/budget` + }); + } + return ( <> { + if (resource.mode !== "inference") { + router.replace( + `/${resource.orgId}/settings/resources/public/${resource.niceId}/general` + ); + } + }, [router, resource.mode, resource.niceId, resource.orgId]); + + if (resource.mode !== "inference") { + return null; + } + + return ( + + + + ); +} diff --git a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx index 43a3a62c0..ed69a22b0 100644 --- a/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx +++ b/src/app/[orgId]/settings/resources/public/[niceId]/layout.tsx @@ -113,6 +113,13 @@ export default async function ResourceLayout(props: ResourceLayoutProps) { } } + if (resource.mode === "inference") { + navItems.push({ + title: t("resourceBudgetSettings"), + href: `/{orgId}/settings/resources/public/{niceId}/budget` + }); + } + return ( <> Date: Mon, 10 Aug 2026 14:12:34 -0400 Subject: [PATCH 098/296] order by budget id --- server/routers/aiBudget/listAiBudgetsForModel.ts | 2 +- server/routers/aiBudget/listAiBudgetsForProvider.ts | 2 +- server/routers/aiBudget/listAiBudgetsForResource.ts | 2 +- server/routers/aiBudget/listAiBudgetsForRole.ts | 2 +- server/routers/aiBudget/listAiBudgetsForSiteResource.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/routers/aiBudget/listAiBudgetsForModel.ts b/server/routers/aiBudget/listAiBudgetsForModel.ts index 4a2277b5b..2c04368a7 100644 --- a/server/routers/aiBudget/listAiBudgetsForModel.ts +++ b/server/routers/aiBudget/listAiBudgetsForModel.ts @@ -51,7 +51,7 @@ export async function listAiBudgetsForModel( .select() .from(aiBudgets) .where(eq(aiBudgets.modelId, modelId)) - .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + .orderBy(asc(aiBudgets.budgetId)); return response(res, { data: { budgets }, diff --git a/server/routers/aiBudget/listAiBudgetsForProvider.ts b/server/routers/aiBudget/listAiBudgetsForProvider.ts index 1a0625f4b..d231f7445 100644 --- a/server/routers/aiBudget/listAiBudgetsForProvider.ts +++ b/server/routers/aiBudget/listAiBudgetsForProvider.ts @@ -51,7 +51,7 @@ export async function listAiBudgetsForProvider( .select() .from(aiBudgets) .where(eq(aiBudgets.providerId, providerId)) - .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + .orderBy(asc(aiBudgets.budgetId)); return response(res, { data: { budgets }, diff --git a/server/routers/aiBudget/listAiBudgetsForResource.ts b/server/routers/aiBudget/listAiBudgetsForResource.ts index 106b049a7..0d59ccae6 100644 --- a/server/routers/aiBudget/listAiBudgetsForResource.ts +++ b/server/routers/aiBudget/listAiBudgetsForResource.ts @@ -51,7 +51,7 @@ export async function listAiBudgetsForResource( .select() .from(aiBudgets) .where(eq(aiBudgets.resourceId, resourceId)) - .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + .orderBy(asc(aiBudgets.budgetId)); return response(res, { data: { budgets }, diff --git a/server/routers/aiBudget/listAiBudgetsForRole.ts b/server/routers/aiBudget/listAiBudgetsForRole.ts index 2e8ff0e8d..45f90a4a6 100644 --- a/server/routers/aiBudget/listAiBudgetsForRole.ts +++ b/server/routers/aiBudget/listAiBudgetsForRole.ts @@ -51,7 +51,7 @@ export async function listAiBudgetsForRole( .select() .from(aiBudgets) .where(eq(aiBudgets.roleId, roleId)) - .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + .orderBy(asc(aiBudgets.budgetId)); return response(res, { data: { budgets }, diff --git a/server/routers/aiBudget/listAiBudgetsForSiteResource.ts b/server/routers/aiBudget/listAiBudgetsForSiteResource.ts index b2557719d..a1f6e280d 100644 --- a/server/routers/aiBudget/listAiBudgetsForSiteResource.ts +++ b/server/routers/aiBudget/listAiBudgetsForSiteResource.ts @@ -51,7 +51,7 @@ export async function listAiBudgetsForSiteResource( .select() .from(aiBudgets) .where(eq(aiBudgets.siteResourceId, siteResourceId)) - .orderBy(asc(aiBudgets.unit), asc(aiBudgets.period)); + .orderBy(asc(aiBudgets.budgetId)); return response(res, { data: { budgets }, From 186eeed784bd62f9d8e36d43b0dd95f5c28a8cb0 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Mon, 10 Aug 2026 14:54:58 -0400 Subject: [PATCH 099/296] add richer model editor on provider --- messages/en-US.json | 45 +- .../ai-providers/[providerId]/models/page.tsx | 207 +++-- src/components/AiProviderModelListEditor.tsx | 875 ++++++++++++++++++ 3 files changed, 1028 insertions(+), 99 deletions(-) create mode 100644 src/components/AiProviderModelListEditor.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 646a82664..ec25fde1a 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1779,14 +1779,49 @@ "aiProviderMessageRemove": "This will permanently delete the provider and its models and targets. This cannot be undone.", "aiProviderErrorNoUpdate": "AI provider is not available to update", "aiProviderModels": "Models", - "aiProviderModelsDescription": "Define allow and block patterns for this provider. Requests must match an allow pattern and must not match a block pattern. Pick from known models for this provider, or type a custom key or wildcard (for example gpt-4* or claude-?). An empty allow list denies all models.", - "aiProviderModelsPlaceholder": "Select a model or type a pattern (e.g. gpt-4*)", + "aiProviderModelsDescription": "Define allow and block lists for this provider. Requests must match an allow entry and must not match a block entry.", + "aiProviderModelsPlaceholder": "Search models or type a custom key", "aiProviderModelsAllow": "Allow List", "aiProviderModelsAllowDescription": "Models that may be used through this provider. Empty means deny all.", - "aiProviderModelsAllowPlaceholder": "Select a model or type a pattern (e.g. gpt-4*)", + "aiProviderModelsAllowPlaceholder": "Enter model key", + "aiProviderModelsAllowEmpty": "No models allowed. All requests will be denied.", "aiProviderModelsBlock": "Block List", - "aiProviderModelsBlockDescription": "Models to deny even if they match an allow pattern.", - "aiProviderModelsBlockPlaceholder": "Select a model or type a pattern (e.g. gpt-4o-mini)", + "aiProviderModelsBlockDescription": "Models to deny even if they match an allow entry.", + "aiProviderModelsBlockPlaceholder": "Enter model key", + "aiProviderModelsBlockEmpty": "No blocked models.", + "aiProviderModelsAdd": "Add Models", + "aiProviderModelsClearAll": "Clear All", + "aiProviderModelsAddCustom": "Add \"{key}\"", + "aiProviderModelsAddCustomHint": "Press Enter to add this custom model key.", + "aiProviderModelsAddBulk": "Add {count} custom keys", + "aiProviderModelsAddBulkHint": "Press Enter to add {count} custom keys.", + "aiProviderModelsAddOne": "Add {key} now", + "aiProviderModelsAddSelected": "Add Selected", + "aiProviderModelsSelectedCount": "{count} selected", + "aiProviderModelsSelectAll": "Select all", + "aiProviderModelsClearSelected": "Clear", + "aiProviderModelsBulkHint": "Select known models or type a custom key.", + "aiProviderModelsCatalogEmpty": "No matching catalog models.", + "aiProviderModelsCatalogHeading": "Known Models", + "aiProviderModelsAllLabel": "All models", + "aiProviderModelsAllPatternHint": "Wildcard: *", + "aiProviderModelsAddAllAllow": "Allow all models", + "aiProviderModelsAddAllBlock": "Block all models", + "aiProviderModelsAddAllDescription": "Uses the * wildcard so every model key matches.", + "aiProviderModelsViewMore": "View more ({count})", + "aiProviderModelsViewLess": "View less", + "aiProviderModelsRemove": "Remove model", + "aiProviderModelsEditHint": "Click to edit model settings", + "aiProviderModelsSourceCatalog": "Known catalog model", + "aiProviderModelsSourceCustom": "Custom model key", + "aiProviderModelsSourcePattern": "Wildcard pattern", + "aiProviderModelsSourceAll": "Matches every model key", + "aiProviderModelsBudgetConfigured": "Budget configured", + "aiProviderModelsEditTitle": "Edit Model", + "aiProviderModelsEditDescription": "Update the model key. Additional settings such as budgets will appear here later.", + "aiProviderModelsKeyLabel": "Model Key", + "aiProviderModelsKeyRequired": "Enter a model key", + "aiProviderModelsKeyDuplicate": "This model key is already on a list", "aiProviderModelsOverlapError": "These patterns cannot be on both lists: {keys}", "aiProviderModelsUpdated": "Models updated", "aiProviderModelsErrorUpdate": "Failed to update models", diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx index cba252dc8..c8a1a4483 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx @@ -10,7 +10,11 @@ import { SettingsSectionHeader, SettingsSectionTitle } from "@app/components/Settings"; -import { TagInput, type Tag } from "@app/components/tags/tag-input"; +import { + AiProviderModelListEditor, + type AiProviderModelListItem, + type ModelListType +} from "@app/components/AiProviderModelListEditor"; import { Button } from "@app/components/ui/button"; import { Label } from "@app/components/ui/label"; import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; @@ -22,7 +26,22 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useTranslations } from "next-intl"; import { useEffect, useMemo, useState } from "react"; -type ModelListType = "allow" | "block"; +function toListItem( + model: { + modelId: number; + modelKey: string; + listType?: ModelListType | null; + }, + listType: ModelListType +): AiProviderModelListItem { + return { + clientId: String(model.modelId), + modelId: model.modelId, + modelKey: model.modelKey, + listType, + hasBudget: false + }; +} export default function AiProviderModelsPage() { const { provider } = useAiProviderContext(); @@ -31,14 +50,8 @@ export default function AiProviderModelsPage() { const queryClient = useQueryClient(); const t = useTranslations(); const [saveLoading, setSaveLoading] = useState(false); - const [allowTags, setAllowTags] = useState([]); - const [blockTags, setBlockTags] = useState([]); - const [activeAllowTagIndex, setActiveAllowTagIndex] = useState< - number | null - >(null); - const [activeBlockTagIndex, setActiveBlockTagIndex] = useState< - number | null - >(null); + const [allowItems, setAllowItems] = useState([]); + const [blockItems, setBlockItems] = useState([]); const modelsQuery = useQuery( aiProviderQueries.providerModels({ providerId: provider.providerId }) @@ -47,32 +60,31 @@ export default function AiProviderModelsPage() { aiProviderQueries.catalogModels({ providerId: provider.providerId }) ); - const catalogTags = useMemo( - () => - (catalogQuery.data ?? []).map((entry) => ({ - id: entry.model, - text: entry.model - })), + const catalogModels = useMemo( + () => (catalogQuery.data ?? []).map((entry) => entry.model), [catalogQuery.data] ); + const allowExcludeKeys = useMemo( + () => new Set(blockItems.map((item) => item.modelKey)), + [blockItems] + ); + const blockExcludeKeys = useMemo( + () => new Set(allowItems.map((item) => item.modelKey)), + [allowItems] + ); + useEffect(() => { if (!modelsQuery.data) return; - setAllowTags( + setAllowItems( modelsQuery.data .filter((model) => (model.listType ?? "allow") === "allow") - .map((model) => ({ - id: String(model.modelId), - text: model.modelKey - })) + .map((model) => toListItem(model, "allow")) ); - setBlockTags( + setBlockItems( modelsQuery.data .filter((model) => model.listType === "block") - .map((model) => ({ - id: String(model.modelId), - text: model.modelKey - })) + .map((model) => toListItem(model, "block")) ); }, [modelsQuery.data]); @@ -80,15 +92,23 @@ export default function AiProviderModelsPage() { setSaveLoading(true); try { const existing = modelsQuery.data ?? []; + const existingById = new Map( + existing.map((model) => [model.modelId, model]) + ); const existingByKey = new Map( existing.map((model) => [model.modelKey, model]) ); + const desiredItems = [...allowItems, ...blockItems].map((item) => ({ + ...item, + modelKey: item.modelKey.trim() + })); + const nextAllow = new Set( - allowTags.map((tag) => tag.text.trim()).filter(Boolean) + allowItems.map((item) => item.modelKey.trim()).filter(Boolean) ); const nextBlock = new Set( - blockTags.map((tag) => tag.text.trim()).filter(Boolean) + blockItems.map((item) => item.modelKey.trim()).filter(Boolean) ); const overlap = [...nextAllow].filter((key) => nextBlock.has(key)); @@ -103,41 +123,58 @@ export default function AiProviderModelsPage() { return; } - const desired = new Map(); - for (const key of nextAllow) { - desired.set(key, "allow"); - } - for (const key of nextBlock) { - desired.set(key, "block"); - } - const toCreate: { modelKey: string; listType: ModelListType }[] = []; const toUpdate: { modelId: number; + modelKey: string; listType: ModelListType; }[] = []; - const toDelete: number[] = []; + const retainedIds = new Set(); - for (const [modelKey, listType] of desired) { - const existingModel = existingByKey.get(modelKey); - if (!existingModel) { - toCreate.push({ modelKey, listType }); + for (const item of desiredItems) { + const listType = item.listType; + const modelKey = item.modelKey; + if (!modelKey) continue; + + if (item.modelId != null && existingById.has(item.modelId)) { + retainedIds.add(item.modelId); + const existingModel = existingById.get(item.modelId)!; + if ( + existingModel.modelKey !== modelKey || + (existingModel.listType ?? "allow") !== listType + ) { + toUpdate.push({ + modelId: item.modelId, + modelKey, + listType + }); + } continue; } - if ((existingModel.listType ?? "allow") !== listType) { - toUpdate.push({ - modelId: existingModel.modelId, - listType - }); + + const existingBySameKey = existingByKey.get(modelKey); + if ( + existingBySameKey && + !retainedIds.has(existingBySameKey.modelId) + ) { + retainedIds.add(existingBySameKey.modelId); + if ((existingBySameKey.listType ?? "allow") !== listType) { + toUpdate.push({ + modelId: existingBySameKey.modelId, + modelKey, + listType + }); + } + continue; } + + toCreate.push({ modelKey, listType }); } - for (const model of existing) { - if (!desired.has(model.modelKey)) { - toDelete.push(model.modelId); - } - } + const toDelete = existing + .filter((model) => !retainedIds.has(model.modelId)) + .map((model) => model.modelId); await Promise.all([ ...toCreate.map(({ modelKey, listType }) => @@ -147,8 +184,12 @@ export default function AiProviderModelsPage() { listType }) ), - ...toUpdate.map(({ modelId, listType }) => - api.post(`/ai-model/${modelId}`, { listType }) + ...toUpdate.map(({ modelId, modelKey, listType }) => + api.post(`/ai-model/${modelId}`, { + modelKey, + name: modelKey, + listType + }) ), ...toDelete.map((modelId) => api.delete(`/ai-model/${modelId}`)) ]); @@ -177,8 +218,6 @@ export default function AiProviderModelsPage() { } } - const inputsDisabled = modelsQuery.isLoading || saveLoading; - return ( @@ -192,30 +231,20 @@ export default function AiProviderModelsPage() { - +
- { - const next = - typeof newTags === "function" - ? newTags(allowTags) - : newTags; - setAllowTags(next as Tag[]); - }} - enableAutocomplete={catalogTags.length > 0} - autocompleteOptions={catalogTags} - allowDuplicates={false} - sortTags - delimiterList={[",", "Enter"]} - disabled={inputsDisabled} />

{t("aiProviderModelsAllowDescription")} @@ -224,27 +253,17 @@ export default function AiProviderModelsPage() {

- { - const next = - typeof newTags === "function" - ? newTags(blockTags) - : newTags; - setBlockTags(next as Tag[]); - }} - enableAutocomplete={catalogTags.length > 0} - autocompleteOptions={catalogTags} - allowDuplicates={false} - sortTags - delimiterList={[",", "Enter"]} - disabled={inputsDisabled} />

{t("aiProviderModelsBlockDescription")} diff --git a/src/components/AiProviderModelListEditor.tsx b/src/components/AiProviderModelListEditor.tsx new file mode 100644 index 000000000..e3bf2bf07 --- /dev/null +++ b/src/components/AiProviderModelListEditor.tsx @@ -0,0 +1,875 @@ +"use client"; + +import { + Credenza, + CredenzaBody, + CredenzaClose, + CredenzaContent, + CredenzaDescription, + CredenzaFooter, + CredenzaHeader, + CredenzaTitle +} from "@app/components/Credenza"; +import { Button } from "@app/components/ui/button"; +import { Checkbox } from "@app/components/ui/checkbox"; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList +} from "@app/components/ui/command"; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { Input } from "@app/components/ui/input"; +import { + Popover, + PopoverContent, + PopoverTrigger +} from "@app/components/ui/popover"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger +} from "@app/components/ui/tooltip"; +import { cn } from "@app/lib/cn"; +import { isModelKeyPattern } from "@server/lib/aiModelKeyMatch"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { + Asterisk, + BookMarked, + Check, + Globe, + Pencil, + Plus, + Wallet, + XIcon +} from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; + +export type ModelListType = "allow" | "block"; +export type ModelSource = "catalog" | "custom" | "pattern" | "all"; + +/** Matches every model key via the provider policy wildcard. */ +export const ALL_MODELS_KEY = "*"; + +const COLLAPSED_ROWS = 5; + +export type AiProviderModelListItem = { + clientId: string; + modelId?: number; + modelKey: string; + listType: ModelListType; + hasBudget?: boolean; +}; + +export type AiProviderModelListEditorProps = { + listType: ModelListType; + items: AiProviderModelListItem[]; + catalogModels: string[]; + /** Keys already used on this list or the sibling list. */ + excludeKeys?: ReadonlySet; + onChange: (items: AiProviderModelListItem[]) => void; + disabled?: boolean; + emptyMessage: string; + addPlaceholder: string; +}; + +export function isAllModelsKey(modelKey: string): boolean { + return modelKey.trim() === ALL_MODELS_KEY; +} + +export function resolveModelSource( + modelKey: string, + catalogModels: ReadonlySet | readonly string[] +): ModelSource { + if (isAllModelsKey(modelKey)) { + return "all"; + } + if (isModelKeyPattern(modelKey)) { + return "pattern"; + } + const set = + catalogModels instanceof Set ? catalogModels : new Set(catalogModels); + return set.has(modelKey) ? "catalog" : "custom"; +} + +function newClientId(): string { + if (typeof crypto !== "undefined" && "randomUUID" in crypto) { + return crypto.randomUUID(); + } + return `tmp-${Date.now()}-${Math.random().toString(36).slice(2)}`; +} + +function parseBulkKeys(raw: string): string[] { + const seen = new Set(); + const keys: string[] = []; + for (const part of raw.split(/[\n,]+/)) { + const key = part.trim(); + if (!key || seen.has(key)) continue; + seen.add(key); + keys.push(key); + } + return keys; +} + +function useModelGridColumns(): number { + const [columns, setColumns] = useState(1); + + useEffect(() => { + const sm = window.matchMedia("(min-width: 640px)"); + const xl = window.matchMedia("(min-width: 1280px)"); + const update = () => { + setColumns(xl.matches ? 3 : sm.matches ? 2 : 1); + }; + update(); + sm.addEventListener("change", update); + xl.addEventListener("change", update); + return () => { + sm.removeEventListener("change", update); + xl.removeEventListener("change", update); + }; + }, []); + + return columns; +} + +export function AiProviderModelListEditor({ + listType, + items, + catalogModels, + excludeKeys, + onChange, + disabled, + emptyMessage, + addPlaceholder +}: AiProviderModelListEditorProps) { + const t = useTranslations(); + const [editingClientId, setEditingClientId] = useState(null); + const [addOpen, setAddOpen] = useState(false); + const [addQuery, setAddQuery] = useState(""); + const [selectedKeys, setSelectedKeys] = useState>(new Set()); + const [listExpanded, setListExpanded] = useState(false); + const [clipHeight, setClipHeight] = useState(null); + const gridRef = useRef(null); + const columns = useModelGridColumns(); + const collapsedLimit = columns * COLLAPSED_ROWS; + const hasOverflow = items.length > collapsedLimit; + const isCollapsed = hasOverflow && !listExpanded; + + const catalogSet = useMemo(() => new Set(catalogModels), [catalogModels]); + + const blockedKeys = useMemo(() => { + const set = new Set(excludeKeys ? [...excludeKeys] : []); + for (const item of items) { + set.add(item.modelKey); + } + return set; + }, [excludeKeys, items]); + + const availableCatalog = useMemo(() => { + const q = addQuery.trim().toLowerCase(); + return catalogModels + .filter((model) => !blockedKeys.has(model)) + .filter((model) => (q ? model.toLowerCase().includes(q) : true)); + }, [addQuery, blockedKeys, catalogModels]); + + const trimmedQuery = addQuery.trim(); + const bulkKeys = useMemo( + () => + parseBulkKeys(addQuery).filter( + (key) => !blockedKeys.has(key) && !catalogSet.has(key) + ), + [addQuery, blockedKeys, catalogSet] + ); + const canAddCustom = + bulkKeys.length === 1 && + !trimmedQuery.includes("\n") && + !trimmedQuery.includes(",") && + !isAllModelsKey(trimmedQuery) && + !catalogSet.has(trimmedQuery) && + !blockedKeys.has(trimmedQuery); + const canAddBulkCustom = bulkKeys.length > 1; + + const allModelsLabel = t("aiProviderModelsAllLabel"); + const showAllModelsOption = + !blockedKeys.has(ALL_MODELS_KEY) && + (!trimmedQuery || + trimmedQuery === ALL_MODELS_KEY || + "all".includes(trimmedQuery.toLowerCase()) || + allModelsLabel.toLowerCase().includes(trimmedQuery.toLowerCase())); + + const editing = items.find((item) => item.clientId === editingClientId); + + function appendModels(modelKeys: string[]) { + if (disabled) return; + const nextBlocked = new Set(blockedKeys); + const additions: AiProviderModelListItem[] = []; + for (const raw of modelKeys) { + const key = raw.trim(); + if (!key || nextBlocked.has(key)) continue; + nextBlocked.add(key); + additions.push({ + clientId: newClientId(), + modelKey: key, + listType, + hasBudget: false + }); + } + if (additions.length === 0) return; + onChange([...items, ...additions]); + } + + function addModel(modelKey: string, options?: { keepOpen?: boolean }) { + appendModels([modelKey]); + setAddQuery(""); + setSelectedKeys(new Set()); + if (!options?.keepOpen) { + setAddOpen(false); + } + } + + function addSelected() { + appendModels([...selectedKeys]); + setSelectedKeys(new Set()); + setAddQuery(""); + // Keep open so more can be selected after filter refresh + } + + function addBulkCustom() { + appendModels(bulkKeys); + setAddQuery(""); + setSelectedKeys(new Set()); + } + + function toggleSelected(model: string) { + setSelectedKeys((prev) => { + const next = new Set(prev); + if (next.has(model)) { + next.delete(model); + } else { + next.add(model); + } + return next; + }); + } + + function selectAllVisible() { + setSelectedKeys((prev) => { + const next = new Set(prev); + for (const model of availableCatalog) { + next.add(model); + } + return next; + }); + } + + function clearSelected() { + setSelectedKeys(new Set()); + } + + function removeModel(clientId: string) { + onChange(items.filter((item) => item.clientId !== clientId)); + } + + function updateModel(updated: AiProviderModelListItem) { + onChange( + items.map((item) => + item.clientId === updated.clientId ? updated : item + ) + ); + setEditingClientId(null); + } + + // Drop selections that are no longer available (already added). + useEffect(() => { + setSelectedKeys((prev) => { + let changed = false; + const next = new Set(); + for (const key of prev) { + if (blockedKeys.has(key)) { + changed = true; + continue; + } + next.add(key); + } + return changed ? next : prev; + }); + }, [blockedKeys]); + + useEffect(() => { + if (!hasOverflow) { + setListExpanded(false); + } + }, [hasOverflow]); + + useLayoutEffect(() => { + if (!isCollapsed || !gridRef.current) { + setClipHeight(null); + return; + } + + const children = Array.from(gridRef.current.children) as HTMLElement[]; + const lastVisible = children[collapsedLimit - 1]; + if (!lastVisible) { + setClipHeight(null); + return; + } + + const gridTop = gridRef.current.getBoundingClientRect().top; + const cardBottom = lastVisible.getBoundingClientRect().bottom; + // Peek slightly into the next row so the fade has content to soften. + setClipHeight(cardBottom - gridTop + 12); + }, [isCollapsed, collapsedLimit, items]); + + return ( +

+
+
+
+ {items.length === 0 ? ( +
+ + {emptyMessage} + +
+ ) : ( + items.map((item) => ( + + setEditingClientId(item.clientId) + } + onRemove={() => removeModel(item.clientId)} + /> + )) + )} +
+ {isCollapsed ? ( +
+ ) : null} +
+ {isCollapsed ? ( +
+ +
+ ) : null} + {hasOverflow && listExpanded ? ( +
+ +
+ ) : null} +
+ +
+ { + if (disabled) return; + setAddOpen(open); + if (!open) { + setAddQuery(""); + setSelectedKeys(new Set()); + } + }} + > + + + + + + { + if (e.key !== "Enter") return; + if (canAddBulkCustom) { + e.preventDefault(); + addBulkCustom(); + return; + } + if (canAddCustom) { + e.preventDefault(); + addModel(trimmedQuery, { + keepOpen: true + }); + } + }} + /> +
+

+ {t("aiProviderModelsBulkHint")} +

+ {availableCatalog.length > 0 ? ( +
+ + {selectedKeys.size > 0 ? ( + + ) : null} +
+ ) : null} +
+ + + {canAddBulkCustom + ? t("aiProviderModelsAddBulkHint", { + count: bulkKeys.length + }) + : canAddCustom + ? t("aiProviderModelsAddCustomHint") + : t("aiProviderModelsCatalogEmpty")} + + {showAllModelsOption ? ( + + + addModel(ALL_MODELS_KEY, { + keepOpen: true + }) + } + className="items-start gap-2 py-2" + > + +
+

+ {listType === "allow" + ? t( + "aiProviderModelsAddAllAllow" + ) + : t( + "aiProviderModelsAddAllBlock" + )} +

+

+ {t( + "aiProviderModelsAddAllDescription" + )} +

+
+ + + ) : null} + {canAddBulkCustom ? ( + + + + {t("aiProviderModelsAddBulk", { + count: bulkKeys.length + })} + + + ) : null} + {canAddCustom ? ( + + + addModel(trimmedQuery, { + keepOpen: true + }) + } + > + + {t("aiProviderModelsAddCustom", { + key: trimmedQuery + })} + + + ) : null} + {availableCatalog.length > 0 ? ( + + {availableCatalog.map((model) => { + const isSelected = + selectedKeys.has(model); + return ( + { + // Toggle selection for bulk; + // double-purpose: shift-free multi-pick. + toggleSelected(model); + }} + className="gap-2" + > + + + {model} + + + ); + })} + + ) : null} + + {selectedKeys.size > 0 ? ( +
+ + {t("aiProviderModelsSelectedCount", { + count: selectedKeys.size + })} + + +
+ ) : null} + + + + {items.length > 0 ? ( + + ) : null} +
+ + {editing && ( + { + if (!open) setEditingClientId(null); + }} + existingKeys={blockedKeys} + onSave={updateModel} + /> + )} +
+ ); +} + +function ModelCard({ + item, + source, + disabled, + onEdit, + onRemove +}: { + item: AiProviderModelListItem; + source: ModelSource; + disabled?: boolean; + onEdit: () => void; + onRemove: () => void; +}) { + const t = useTranslations(); + + const sourceLabel = + source === "all" + ? t("aiProviderModelsSourceAll") + : source === "catalog" + ? t("aiProviderModelsSourceCatalog") + : source === "pattern" + ? t("aiProviderModelsSourcePattern") + : t("aiProviderModelsSourceCustom"); + + const SourceIcon = + source === "all" + ? Globe + : source === "catalog" + ? BookMarked + : source === "pattern" + ? Asterisk + : Pencil; + + return ( +
{ + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + onEdit(); + } + } + } + role={disabled ? undefined : "button"} + tabIndex={disabled ? undefined : 0} + title={t("aiProviderModelsEditHint")} + > +
+ {source === "all" ? ( + <> + + {t("aiProviderModelsAllLabel")} + + + ) : ( + + {item.modelKey} + + )} +
+ +
+ + + e.stopPropagation()} + > + + + + {sourceLabel} + + {item.hasBudget ? ( + + + e.stopPropagation()} + > + + + + + {t("aiProviderModelsBudgetConfigured")} + + + ) : null} +
+ + +
+ ); +} + +type EditFormValues = { + modelKey: string; +}; + +function EditModelCredenza({ + item, + open, + onOpenChange, + existingKeys, + onSave +}: { + item: AiProviderModelListItem; + open: boolean; + onOpenChange: (open: boolean) => void; + existingKeys: ReadonlySet; + onSave: (item: AiProviderModelListItem) => void; +}) { + const t = useTranslations(); + + const editSchema = useMemo( + () => + z.object({ + modelKey: z + .string() + .trim() + .min(1, t("aiProviderModelsKeyRequired")) + .refine( + (key) => + key === item.modelKey || !existingKeys.has(key), + t("aiProviderModelsKeyDuplicate") + ) + }), + [existingKeys, item.modelKey, t] + ); + + const form = useForm({ + resolver: zodResolver(editSchema), + defaultValues: { modelKey: item.modelKey } + }); + + useEffect(() => { + if (!open) return; + form.reset({ modelKey: item.modelKey }); + }, [form, item.clientId, item.modelKey, open]); + + function handleSubmit(values: EditFormValues) { + onSave({ + ...item, + modelKey: values.modelKey.trim() + }); + } + + return ( + + + + + {t("aiProviderModelsEditTitle")} + + + {t("aiProviderModelsEditDescription")} + + +
+ + + ( + + + {t("aiProviderModelsKeyLabel")} + + + + + + + )} + /> + +
+ + + + + + + + + + ); +} From 2becb15916ec4d545eb971c7a221d391ff781b65 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 14:27:17 -0400 Subject: [PATCH 100/296] put the budget on the role form --- messages/en-US.json | 2 ++ src/components/BudgetsEditor.tsx | 31 +++++++++++++++++++++---------- src/components/RoleForm.tsx | 29 ++++++++++++++++++++++++++++- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index ec25fde1a..f8619f7da 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1079,6 +1079,8 @@ "accessRoleErrorNewRequired": "New role is required", "accessRoleErrorRemove": "Failed to remove role", "accessRoleErrorRemoveDescription": "An error occurred while removing the role.", + "accessRoleInferenceBudget": "Inference Budget", + "accessRoleInferenceBudgetDescription": "Configure how members of this role restrict AI usage based on spending or token limits", "accessRoleName": "Role Name", "accessRoleQuestionRemove": "You're about to delete the `{name}` role. You cannot undo this action.", "accessRoleRemove": "Remove Role", diff --git a/src/components/BudgetsEditor.tsx b/src/components/BudgetsEditor.tsx index 2a8b73022..61a0595b8 100644 --- a/src/components/BudgetsEditor.tsx +++ b/src/components/BudgetsEditor.tsx @@ -85,12 +85,14 @@ export function BudgetsEditor({ scope, orgId, title, - description + description, + hideCardHeader = false }: { scope: AiBudgetScope; orgId: string; title: string; description: string; + hideCardHeader?: boolean; }) { const { env } = useEnvContext(); const api = createApiClient({ env }); @@ -268,15 +270,8 @@ export function BudgetsEditor({ ); - return ( - - - {title} - - {description} - - - + const body = ( + <>
@@ -456,6 +451,22 @@ export function BudgetsEditor({ {t("saveSettings")} + + ); + + if (hideCardHeader) { + return
{body}
; + } + + return ( + + + {title} + + {description} + + + {body} ); } diff --git a/src/components/RoleForm.tsx b/src/components/RoleForm.tsx index 102aa8e3a..eb34f6aa5 100644 --- a/src/components/RoleForm.tsx +++ b/src/components/RoleForm.tsx @@ -35,6 +35,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { HorizontalTabs } from "@app/components/HorizontalTabs"; import { PaidFeaturesAlert } from "./PaidFeaturesAlert"; import { CheckboxWithLabel } from "./ui/checkbox"; +import { BudgetsEditor } from "@app/components/BudgetsEditor"; import { tierMatrix } from "@server/lib/billing/tierMatrix"; import type { Role } from "@server/db"; @@ -333,7 +334,15 @@ export function RoleForm({ { title: t("general"), href: "#" }, ...(env.flags.disableEnterpriseFeatures ? [] - : [{ title: t("sshAccess"), href: "#" }]) + : [{ title: t("sshAccess"), href: "#" }]), + ...(variant === "edit" && role + ? [ + { + title: t("accessRoleInferenceBudget"), + href: "#" + } + ] + : []) ]} > {/* General tab */} @@ -635,6 +644,24 @@ export function RoleForm({ /> )} + + {/* Inference Budget tab - only available once the role exists */} + {variant === "edit" && role && ( +
+ +
+ )} )} From a357f42c488a570b2ae93c19460f41c65ba53714 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 14:35:35 -0400 Subject: [PATCH 101/296] allow creating role with budgets --- src/components/BudgetsEditor.tsx | 496 +++++++++++++++--------------- src/components/CreateRoleForm.tsx | 28 +- src/components/RoleForm.tsx | 85 ++++- 3 files changed, 351 insertions(+), 258 deletions(-) diff --git a/src/components/BudgetsEditor.tsx b/src/components/BudgetsEditor.tsx index 61a0595b8..7a3e15344 100644 --- a/src/components/BudgetsEditor.tsx +++ b/src/components/BudgetsEditor.tsx @@ -44,7 +44,7 @@ import { Plus, Trash2 } from "lucide-react"; import { useTranslations } from "next-intl"; import { useEffect, useMemo, useState } from "react"; -type BudgetRow = { +export type BudgetRow = { key: string; budgetId?: number; amount: string; @@ -81,6 +81,249 @@ function nextAvailableCombo(rows: BudgetRow[]): { return { unit: "usd", period: "monthly" }; } +export function newBudgetRow(rows: BudgetRow[]): BudgetRow { + const combo = nextAvailableCombo(rows); + return { + key: crypto.randomUUID(), + amount: "", + unit: combo.unit, + period: combo.period + }; +} + +export function getBudgetRowsErrors(rows: BudgetRow[]): { + conflictingKeys: Set; + invalidAmountKeys: Set; +} { + const counts = new Map(); + for (const row of rows) { + const key = comboKey(row.unit, row.period); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + const conflictingKeys = new Set(); + for (const row of rows) { + const key = comboKey(row.unit, row.period); + if ((counts.get(key) ?? 0) > 1) { + conflictingKeys.add(row.key); + } + } + + const invalidAmountKeys = new Set(); + for (const row of rows) { + const amount = Number(row.amount); + if (!row.amount.trim() || !Number.isFinite(amount) || amount <= 0) { + invalidAmountKeys.add(row.key); + } + } + + return { conflictingKeys, invalidAmountKeys }; +} + +export function BudgetRowsFields({ + rows, + onChange, + disabled = false, + attemptedSave = false +}: { + rows: BudgetRow[]; + onChange: (rows: BudgetRow[]) => void; + disabled?: boolean; + attemptedSave?: boolean; +}) { + const t = useTranslations(); + + const { conflictingKeys, invalidAmountKeys } = useMemo( + () => getBudgetRowsErrors(rows), + [rows] + ); + + function addRow() { + onChange([...rows, newBudgetRow(rows)]); + } + + function removeRow(key: string) { + onChange(rows.filter((row) => row.key !== key)); + } + + function updateRow(key: string, patch: Partial) { + onChange( + rows.map((row) => (row.key === key ? { ...row, ...patch } : row)) + ); + } + + const periodLabels: Record = { + hourly: t("aiBudgetPeriodHourly"), + daily: t("aiBudgetPeriodDaily"), + weekly: t("aiBudgetPeriodWeekly"), + monthly: t("aiBudgetPeriodMonthly"), + yearly: t("aiBudgetPeriodYearly"), + lifetime: t("aiBudgetPeriodLifetime") + }; + + const unitLabels: Record = { + usd: t("aiBudgetUnitUsd"), + tokens: t("aiBudgetUnitTokens") + }; + + const addRowButton = ( + + ); + + return ( +
+
+ + + {t("aiBudgetAmount")} + {t("aiBudgetUnit")} + {t("aiBudgetPeriod")} + + + + + {rows.length === 0 ? ( + + ) : ( + rows.map((row) => { + const showConflict = conflictingKeys.has(row.key); + const showInvalidAmount = + attemptedSave && + invalidAmountKeys.has(row.key); + return ( + + + + updateRow(row.key, { + amount: e.target.value + }) + } + className="w-full min-w-0" + /> + + + + + + + + +
+ +
+ + + ); + }) + )} + +
+ {(conflictingKeys.size > 0 || + (attemptedSave && invalidAmountKeys.size > 0)) && ( +

+ {conflictingKeys.size > 0 + ? t("aiBudgetConflictError") + : t("aiBudgetInvalidAmountError")} +

+ )} + {rows.length > 0 && addRowButton} +
+ ); +} + export function BudgetsEditor({ scope, orgId, @@ -111,58 +354,13 @@ export function BudgetsEditor({ setAttemptedSave(false); }, [budgetsQuery.data]); - const conflictingKeys = useMemo(() => { - const counts = new Map(); - for (const row of rows) { - const key = comboKey(row.unit, row.period); - counts.set(key, (counts.get(key) ?? 0) + 1); - } - const conflicting = new Set(); - for (const row of rows) { - const key = comboKey(row.unit, row.period); - if ((counts.get(key) ?? 0) > 1) { - conflicting.add(row.key); - } - } - return conflicting; - }, [rows]); - - const invalidAmountKeys = useMemo(() => { - const invalid = new Set(); - for (const row of rows) { - const amount = Number(row.amount); - if (!row.amount.trim() || !Number.isFinite(amount) || amount <= 0) { - invalid.add(row.key); - } - } - return invalid; - }, [rows]); + const { conflictingKeys, invalidAmountKeys } = useMemo( + () => getBudgetRowsErrors(rows), + [rows] + ); const hasErrors = conflictingKeys.size > 0 || invalidAmountKeys.size > 0; - function addRow() { - const combo = nextAvailableCombo(rows); - setRows((prev) => [ - ...prev, - { - key: crypto.randomUUID(), - amount: "", - unit: combo.unit, - period: combo.period - } - ]); - } - - function removeRow(key: string) { - setRows((prev) => prev.filter((row) => row.key !== key)); - } - - function updateRow(key: string, patch: Partial) { - setRows((prev) => - prev.map((row) => (row.key === key ? { ...row, ...patch } : row)) - ); - } - async function onSave() { setAttemptedSave(true); if (hasErrors) { @@ -244,201 +442,15 @@ export function BudgetsEditor({ } } - const periodLabels: Record = { - hourly: t("aiBudgetPeriodHourly"), - daily: t("aiBudgetPeriodDaily"), - weekly: t("aiBudgetPeriodWeekly"), - monthly: t("aiBudgetPeriodMonthly"), - yearly: t("aiBudgetPeriodYearly"), - lifetime: t("aiBudgetPeriodLifetime") - }; - - const unitLabels: Record = { - usd: t("aiBudgetUnitUsd"), - tokens: t("aiBudgetUnitTokens") - }; - - const addRowButton = ( - - ); - const body = ( <> -
- - - - {t("aiBudgetAmount")} - {t("aiBudgetUnit")} - {t("aiBudgetPeriod")} - - - - - {rows.length === 0 ? ( - - ) : ( - rows.map((row) => { - const showConflict = conflictingKeys.has( - row.key - ); - const showInvalidAmount = - attemptedSave && - invalidAmountKeys.has(row.key); - return ( - - - - updateRow(row.key, { - amount: e.target - .value - }) - } - className="w-full min-w-0" - /> - - - - - - - - -
- -
- - - ); - }) - )} - -
- {(conflictingKeys.size > 0 || - (attemptedSave && invalidAmountKeys.size > 0)) && ( -

- {conflictingKeys.size > 0 - ? t("aiBudgetConflictError") - : t("aiBudgetInvalidAmountError")} -

- )} - {rows.length > 0 && addRowButton} -
+ diff --git a/src/components/CreateRoleForm.tsx b/src/components/CreateRoleForm.tsx index 678a9edb5..56af53c56 100644 --- a/src/components/CreateRoleForm.tsx +++ b/src/components/CreateRoleForm.tsx @@ -80,13 +80,39 @@ export default function CreateRoleForm({ }); if (res && res.status === 201) { + const createdRole = res.data.data; + + const pendingBudgets = (values.budgets ?? []).filter( + (budget) => budget.amount.trim() !== "" + ); + if (pendingBudgets.length > 0) { + try { + await Promise.all( + pendingBudgets.map((budget) => + api.put(`/org/${org?.org.orgId}/ai-budget`, { + roleId: createdRole.roleId, + amount: Number(budget.amount), + unit: budget.unit, + period: budget.period + }) + ) + ); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: formatAxiosError(e, t("aiBudgetErrorSave")) + }); + } + } + toast({ variant: "default", title: t("accessRoleCreated"), description: t("accessRoleCreatedDescription") }); if (open) setOpen(false); - afterCreate?.(res.data.data); + afterCreate?.(createdRole); } } diff --git a/src/components/RoleForm.tsx b/src/components/RoleForm.tsx index eb34f6aa5..e55a7363b 100644 --- a/src/components/RoleForm.tsx +++ b/src/components/RoleForm.tsx @@ -35,7 +35,13 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { HorizontalTabs } from "@app/components/HorizontalTabs"; import { PaidFeaturesAlert } from "./PaidFeaturesAlert"; import { CheckboxWithLabel } from "./ui/checkbox"; -import { BudgetsEditor } from "@app/components/BudgetsEditor"; +import { + BudgetsEditor, + BudgetRowsFields, + getBudgetRowsErrors, + type BudgetRow +} from "@app/components/BudgetsEditor"; +import type { AiBudgetPeriod, AiBudgetUnit } from "@app/lib/aiBudgetScope"; import { tierMatrix } from "@server/lib/billing/tierMatrix"; import type { Role } from "@server/db"; @@ -83,6 +89,12 @@ function hasOnlyAbsoluteSudoCommands(value: string | undefined): boolean { }); } +export type PendingRoleBudget = { + amount: string; + unit: AiBudgetUnit; + period: AiBudgetPeriod; +}; + export type RoleFormValues = { name: string; description?: string; @@ -92,6 +104,7 @@ export type RoleFormValues = { sshSudoCommands?: string; sshCreateHomeDir?: boolean; sshUnixGroups?: string; + budgets?: PendingRoleBudget[]; }; type RoleFormProps = { @@ -203,6 +216,10 @@ export function RoleForm({ useState(null); const [dragOverField, setDragOverField] = useState(null); + const [pendingBudgetRows, setPendingBudgetRows] = useState( + [] + ); + const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); useEffect(() => { if (sshDisabled) { @@ -253,6 +270,35 @@ export function RoleForm({ }); } + function handleFormSubmit(values: z.infer) { + if (variant === "create") { + const { conflictingKeys, invalidAmountKeys } = + getBudgetRowsErrors(pendingBudgetRows); + if (conflictingKeys.size > 0 || invalidAmountKeys.size > 0) { + setAttemptedBudgetsSave(true); + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: conflictingKeys.size + ? t("aiBudgetConflictError") + : t("aiBudgetInvalidAmountError") + }); + return; + } + + return onSubmit({ + ...values, + budgets: pendingBudgetRows.map(({ amount, unit, period }) => ({ + amount, + unit, + period + })) + }); + } + + return onSubmit(values); + } + function getTextImportDropHandlers(field: RoleTextImportField) { return { onDragOver: (event: React.DragEvent) => { @@ -285,7 +331,7 @@ export function RoleForm({ return (
onSubmit(values))} + onSubmit={form.handleSubmit(handleFormSubmit)} className="space-y-4" id={formId} > @@ -335,14 +381,10 @@ export function RoleForm({ ...(env.flags.disableEnterpriseFeatures ? [] : [{ title: t("sshAccess"), href: "#" }]), - ...(variant === "edit" && role - ? [ - { - title: t("accessRoleInferenceBudget"), - href: "#" - } - ] - : []) + { + title: t("accessRoleInferenceBudget"), + href: "#" + } ]} > {/* General tab */} @@ -645,9 +687,9 @@ export function RoleForm({
)} - {/* Inference Budget tab - only available once the role exists */} - {variant === "edit" && role && ( -
+ {/* Inference Budget tab */} +
+ {variant === "edit" && role ? ( -
- )} + ) : ( + <> +

+ {t( + "accessRoleInferenceBudgetDescription" + )} +

+ + + )} +
)} From e76377d3c7fa747d0efe71383871be99ef44766f Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 14:55:31 -0400 Subject: [PATCH 102/296] Match the pg query logging --- server/db/sqlite/driver.ts | 38 ++++---------------------------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/server/db/sqlite/driver.ts b/server/db/sqlite/driver.ts index 6b115e4ee..a58ec1ead 100644 --- a/server/db/sqlite/driver.ts +++ b/server/db/sqlite/driver.ts @@ -5,39 +5,7 @@ import path from "path"; import fs from "fs"; import { APP_PATH } from "@server/lib/consts"; import { existsSync, mkdirSync } from "fs"; - -// Temporary diagnostic trace for the random better-sqlite3 native aborts -// (Statement::~Statement -> RemoveEnvironmentCleanupHook assertion). That -// abort is a hard SIGABRT from native code, so it bypasses uncaughtException/ -// unhandledRejection and can outrun winston's async file transport. This -// writes every statement text synchronously (fsync'd via appendFileSync) so -// the statements immediately preceding a crash survive it. better-sqlite3's -// `verbose` hook fires for BEGIN/SAVEPOINT/RELEASE/COMMIT/ROLLBACK too, since -// those are just prepared statements under the hood - so this also lets us -// see if two "transactions" ever overlap on the shared connection. -// Enable with SQL_TRACE=true; remove once the crash is root-caused. -function sqlTraceVerbose(): - | ((message: unknown, ...args: unknown[]) => void) - | undefined { - if (process.env.SQL_TRACE !== "true") { - return undefined; - } - const traceLogDir = path.join(APP_PATH, "logs"); - if (!existsSync(traceLogDir)) { - mkdirSync(traceLogDir, { recursive: true }); - } - const traceLogPath = path.join(traceLogDir, "sql-trace.log"); - let seq = 0; - return (message: unknown) => { - seq += 1; - const line = `${new Date().toISOString()} pid=${process.pid} #${seq} ${String(message).replace(/\s+/g, " ").trim()}\n`; - try { - fs.appendFileSync(traceLogPath, line); - } catch { - // best-effort diagnostic logging only - } - }; -} +import logger from "@server/logger"; export const location = path.join(APP_PATH, "db", "db.sqlite"); export const exists = checkFileExists(location); @@ -46,7 +14,9 @@ bootstrapVolume(); function createDb() { const verbose = - process.env.QUERY_LOGGING == "true" ? sqlTraceVerbose() : undefined; + process.env.QUERY_LOGGING == "true" + ? (message: unknown) => logger.debug(String(message)) + : undefined; const sqlite = new Database(location, { verbose }); if (process.env.ENABLE_SQLITE_WAL_MODE == "true") { From e012063f5b70b00e36d8e2ec106de9ab022b9be7 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 15:09:39 -0400 Subject: [PATCH 103/296] add the budget to the models on the provider --- messages/en-US.json | 5 +- .../ai-providers/[providerId]/models/page.tsx | 2 + src/components/AiProviderModelListEditor.tsx | 91 ++++++++++++++----- 3 files changed, 76 insertions(+), 22 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index f8619f7da..4ff3f9da2 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1820,7 +1820,10 @@ "aiProviderModelsSourceAll": "Matches every model key", "aiProviderModelsBudgetConfigured": "Budget configured", "aiProviderModelsEditTitle": "Edit Model", - "aiProviderModelsEditDescription": "Update the model key. Additional settings such as budgets will appear here later.", + "aiProviderModelsEditDescription": "Update the model key or configure its inference budget.", + "aiProviderModelsBudgetTab": "Inference Budget", + "aiProviderModelsBudgetDescription": "Configure how this model restricts usage based on spending or token limits", + "aiProviderModelsBudgetUnsaved": "Save this model first to configure its inference budget.", "aiProviderModelsKeyLabel": "Model Key", "aiProviderModelsKeyRequired": "Enter a model key", "aiProviderModelsKeyDuplicate": "This model key is already on a list", diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx index c8a1a4483..f8ace5cb2 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx @@ -235,6 +235,7 @@ export default function AiProviderModelsPage() {
{ @@ -772,12 +777,14 @@ type EditFormValues = { }; function EditModelCredenza({ + orgId, item, open, onOpenChange, existingKeys, onSave }: { + orgId: string; item: AiProviderModelListItem; open: boolean; onOpenChange: (open: boolean) => void; @@ -835,27 +842,69 @@ function EditModelCredenza({ id="ai-provider-model-edit-form" onSubmit={form.handleSubmit(handleSubmit)} > - - ( - - - {t("aiProviderModelsKeyLabel")} - - - - - - - )} - /> + + +
+ ( + + + {t( + "aiProviderModelsKeyLabel" + )} + + + + + + + )} + /> +
+
+ {item.modelId !== undefined ? ( + + ) : ( +

+ {t( + "aiProviderModelsBudgetUnsaved" + )} +

+ )} +
+ From 88515a7dad2f941184a0380bc55aaf7ce460f55b Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 15:31:45 -0400 Subject: [PATCH 104/296] remove duplicate save buttons --- src/components/AiProviderModelListEditor.tsx | 127 ++++++++++++++++--- src/components/BudgetsEditor.tsx | 112 +++++++++------- src/components/EditRoleForm.tsx | 32 +++++ src/components/RoleForm.tsx | 100 +++++++-------- 4 files changed, 258 insertions(+), 113 deletions(-) diff --git a/src/components/AiProviderModelListEditor.tsx b/src/components/AiProviderModelListEditor.tsx index 6bf935f4d..993ef88b8 100644 --- a/src/components/AiProviderModelListEditor.tsx +++ b/src/components/AiProviderModelListEditor.tsx @@ -43,7 +43,18 @@ import { import { cn } from "@app/lib/cn"; import { isModelKeyPattern } from "@server/lib/aiModelKeyMatch"; import { HorizontalTabs } from "@app/components/HorizontalTabs"; -import { BudgetsEditor } from "@app/components/BudgetsEditor"; +import { + BudgetRowsFields, + getBudgetRowsErrors, + rowsFromBudgets, + saveBudgetRows, + type BudgetRow +} from "@app/components/BudgetsEditor"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { aiBudgetQueries } from "@app/lib/queries"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; import { zodResolver } from "@hookform/resolvers/zod"; import { Asterisk, @@ -792,6 +803,9 @@ function EditModelCredenza({ onSave: (item: AiProviderModelListItem) => void; }) { const t = useTranslations(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const queryClient = useQueryClient(); const editSchema = useMemo( () => @@ -814,12 +828,78 @@ function EditModelCredenza({ defaultValues: { modelKey: item.modelKey } }); + const [pendingBudgetRows, setPendingBudgetRows] = useState( + [] + ); + const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); + const [savingBudgets, setSavingBudgets] = useState(false); + + const budgetScope = + item.modelId !== undefined + ? { type: "model" as const, id: item.modelId } + : null; + + const budgetsQuery = useQuery({ + ...aiBudgetQueries.scoped({ + scope: budgetScope ?? { type: "model", id: -1 } + }), + enabled: open && budgetScope !== null + }); + useEffect(() => { if (!open) return; form.reset({ modelKey: item.modelKey }); + setAttemptedBudgetsSave(false); }, [form, item.clientId, item.modelKey, open]); - function handleSubmit(values: EditFormValues) { + useEffect(() => { + if (!open || !budgetsQuery.data) return; + setPendingBudgetRows(rowsFromBudgets(budgetsQuery.data)); + }, [open, budgetsQuery.data]); + + async function handleSubmit(values: EditFormValues) { + const { conflictingKeys, invalidAmountKeys } = + getBudgetRowsErrors(pendingBudgetRows); + if (conflictingKeys.size > 0 || invalidAmountKeys.size > 0) { + setAttemptedBudgetsSave(true); + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: conflictingKeys.size + ? t("aiBudgetConflictError") + : t("aiBudgetInvalidAmountError") + }); + return; + } + + if (budgetScope) { + setSavingBudgets(true); + try { + const existingBudgets = await queryClient.fetchQuery( + aiBudgetQueries.scoped({ scope: budgetScope }) + ); + await saveBudgetRows({ + api, + orgId, + scope: budgetScope, + existingBudgets, + rows: pendingBudgetRows + }); + await queryClient.invalidateQueries( + aiBudgetQueries.scoped({ scope: budgetScope }) + ); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: formatAxiosError(e, t("aiBudgetErrorSave")) + }); + setSavingBudgets(false); + return; + } + setSavingBudgets(false); + } + onSave({ ...item, modelKey: values.modelKey.trim() @@ -881,21 +961,27 @@ function EditModelCredenza({ />
- {item.modelId !== undefined ? ( - + {budgetScope ? ( + <> +

+ {t( + "aiProviderModelsBudgetDescription" + )} +

+ + ) : (

{t( @@ -914,7 +1000,12 @@ function EditModelCredenza({ {t("cancel")} - diff --git a/src/components/BudgetsEditor.tsx b/src/components/BudgetsEditor.tsx index 7a3e15344..e9dbbaefb 100644 --- a/src/components/BudgetsEditor.tsx +++ b/src/components/BudgetsEditor.tsx @@ -40,6 +40,7 @@ import { import { aiBudgetQueries } from "@app/lib/queries"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import type { AiBudget } from "@server/db"; +import type { AxiosInstance } from "axios"; import { Plus, Trash2 } from "lucide-react"; import { useTranslations } from "next-intl"; import { useEffect, useMemo, useState } from "react"; @@ -52,7 +53,7 @@ export type BudgetRow = { period: AiBudgetPeriod; }; -function rowsFromBudgets(budgets: AiBudget[]): BudgetRow[] { +export function rowsFromBudgets(budgets: AiBudget[]): BudgetRow[] { return budgets.map((budget) => ({ key: String(budget.budgetId), budgetId: budget.budgetId, @@ -324,6 +325,66 @@ export function BudgetRowsFields({ ); } +export async function saveBudgetRows({ + api, + orgId, + scope, + existingBudgets, + rows +}: { + api: AxiosInstance; + orgId: string; + scope: AiBudgetScope; + existingBudgets: AiBudget[]; + rows: Pick[]; +}): Promise { + const existingById = new Map( + existingBudgets.map((budget) => [budget.budgetId, budget]) + ); + const currentBudgetIds = new Set( + rows + .filter((row) => row.budgetId !== undefined) + .map((row) => row.budgetId as number) + ); + const bodyField = getAiBudgetScopeBodyField(scope); + + const toDelete = existingBudgets.filter( + (budget) => !currentBudgetIds.has(budget.budgetId) + ); + const toCreate = rows.filter((row) => row.budgetId === undefined); + const toUpdate = rows.filter((row) => { + if (row.budgetId === undefined) return false; + const existingBudget = existingById.get(row.budgetId); + if (!existingBudget) return false; + return ( + existingBudget.amount !== Number(row.amount) || + existingBudget.unit !== row.unit || + existingBudget.period !== row.period + ); + }); + + await Promise.all([ + ...toDelete.map((budget) => + api.delete(`/ai-budget/${budget.budgetId}`) + ), + ...toCreate.map((row) => + api.put(`/org/${orgId}/ai-budget`, { + [bodyField]: scope.id, + amount: Number(row.amount), + unit: row.unit, + period: row.period + }) + ), + ...toUpdate.map((row) => + api.post(`/ai-budget/${row.budgetId}`, { + amount: Number(row.amount), + unit: row.unit, + period: row.period + }) + ) + ]); +} + export function BudgetsEditor({ scope, orgId, @@ -376,53 +437,14 @@ export function BudgetsEditor({ setSaveLoading(true); try { - const existing = budgetsQuery.data ?? []; - const existingById = new Map( - existing.map((budget) => [budget.budgetId, budget]) - ); - const currentBudgetIds = new Set( + await saveBudgetRows({ + api, + orgId, + scope, + existingBudgets: budgetsQuery.data ?? [], rows - .filter((row) => row.budgetId !== undefined) - .map((row) => row.budgetId as number) - ); - const bodyField = getAiBudgetScopeBodyField(scope); - - const toDelete = existing.filter( - (budget) => !currentBudgetIds.has(budget.budgetId) - ); - const toCreate = rows.filter((row) => row.budgetId === undefined); - const toUpdate = rows.filter((row) => { - if (row.budgetId === undefined) return false; - const existingBudget = existingById.get(row.budgetId); - if (!existingBudget) return false; - return ( - existingBudget.amount !== Number(row.amount) || - existingBudget.unit !== row.unit || - existingBudget.period !== row.period - ); }); - await Promise.all([ - ...toDelete.map((budget) => - api.delete(`/ai-budget/${budget.budgetId}`) - ), - ...toCreate.map((row) => - api.put(`/org/${orgId}/ai-budget`, { - [bodyField]: scope.id, - amount: Number(row.amount), - unit: row.unit, - period: row.period - }) - ), - ...toUpdate.map((row) => - api.post(`/ai-budget/${row.budgetId}`, { - amount: Number(row.amount), - unit: row.unit, - period: row.period - }) - ) - ]); - await queryClient.invalidateQueries( aiBudgetQueries.scoped({ scope }) ); diff --git a/src/components/EditRoleForm.tsx b/src/components/EditRoleForm.tsx index bebf50288..d02e30c0e 100644 --- a/src/components/EditRoleForm.tsx +++ b/src/components/EditRoleForm.tsx @@ -15,6 +15,8 @@ import { useEnvContext } from "@app/hooks/useEnvContext"; import { usePaidStatus } from "@app/hooks/usePaidStatus"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { aiBudgetQueries } from "@app/lib/queries"; +import { useQueryClient } from "@tanstack/react-query"; import type { Role } from "@server/db"; import type { UpdateRoleBody, UpdateRoleResponse } from "@server/routers/role"; import { AxiosResponse } from "axios"; @@ -26,6 +28,7 @@ import { RoleForm, type RoleFormValues } from "./RoleForm"; +import { saveBudgetRows } from "./BudgetsEditor"; import { tierMatrix } from "@server/lib/billing/tierMatrix"; type EditRoleFormProps = { @@ -44,6 +47,7 @@ export default function EditRoleForm({ const t = useTranslations(); const { isPaidUser } = usePaidStatus(); const api = createApiClient(useEnvContext()); + const queryClient = useQueryClient(); const [loading, startTransition] = useTransition(); async function onSubmit(values: RoleFormValues) { @@ -83,6 +87,34 @@ export default function EditRoleForm({ }); if (res && res.status === 200) { + if (values.budgets) { + try { + const scope = { type: "role" as const, id: role.roleId }; + const existingBudgets = await queryClient.fetchQuery( + aiBudgetQueries.scoped({ scope }) + ); + await saveBudgetRows({ + api, + orgId: role.orgId, + scope, + existingBudgets, + rows: values.budgets + }); + await queryClient.invalidateQueries( + aiBudgetQueries.scoped({ scope }) + ); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: formatAxiosError( + e, + t("aiBudgetErrorSave") + ) + }); + } + } + toast({ variant: "default", title: t("accessRoleUpdated"), diff --git a/src/components/RoleForm.tsx b/src/components/RoleForm.tsx index e55a7363b..4e8f457cb 100644 --- a/src/components/RoleForm.tsx +++ b/src/components/RoleForm.tsx @@ -30,17 +30,19 @@ import { import { useTranslations } from "next-intl"; import { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; +import { useQuery } from "@tanstack/react-query"; import { z } from "zod"; import { zodResolver } from "@hookform/resolvers/zod"; import { HorizontalTabs } from "@app/components/HorizontalTabs"; import { PaidFeaturesAlert } from "./PaidFeaturesAlert"; import { CheckboxWithLabel } from "./ui/checkbox"; import { - BudgetsEditor, BudgetRowsFields, getBudgetRowsErrors, + rowsFromBudgets, type BudgetRow } from "@app/components/BudgetsEditor"; +import { aiBudgetQueries } from "@app/lib/queries"; import type { AiBudgetPeriod, AiBudgetUnit } from "@app/lib/aiBudgetScope"; import { tierMatrix } from "@server/lib/billing/tierMatrix"; import type { Role } from "@server/db"; @@ -90,6 +92,7 @@ function hasOnlyAbsoluteSudoCommands(value: string | undefined): boolean { } export type PendingRoleBudget = { + budgetId?: number; amount: string; unit: AiBudgetUnit; period: AiBudgetPeriod; @@ -221,6 +224,19 @@ export function RoleForm({ ); const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); + const budgetsQuery = useQuery({ + ...aiBudgetQueries.scoped({ + scope: { type: "role", id: role?.roleId ?? -1 } + }), + enabled: variant === "edit" && !!role + }); + + useEffect(() => { + if (variant !== "edit" || !budgetsQuery.data) return; + setPendingBudgetRows(rowsFromBudgets(budgetsQuery.data)); + setAttemptedBudgetsSave(false); + }, [variant, budgetsQuery.data]); + useEffect(() => { if (sshDisabled) { form.setValue("allowSsh", false); @@ -271,32 +287,31 @@ export function RoleForm({ } function handleFormSubmit(values: z.infer) { - if (variant === "create") { - const { conflictingKeys, invalidAmountKeys } = - getBudgetRowsErrors(pendingBudgetRows); - if (conflictingKeys.size > 0 || invalidAmountKeys.size > 0) { - setAttemptedBudgetsSave(true); - toast({ - variant: "destructive", - title: t("aiBudgetErrorSave"), - description: conflictingKeys.size - ? t("aiBudgetConflictError") - : t("aiBudgetInvalidAmountError") - }); - return; - } + const { conflictingKeys, invalidAmountKeys } = + getBudgetRowsErrors(pendingBudgetRows); + if (conflictingKeys.size > 0 || invalidAmountKeys.size > 0) { + setAttemptedBudgetsSave(true); + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: conflictingKeys.size + ? t("aiBudgetConflictError") + : t("aiBudgetInvalidAmountError") + }); + return; + } - return onSubmit({ - ...values, - budgets: pendingBudgetRows.map(({ amount, unit, period }) => ({ + return onSubmit({ + ...values, + budgets: pendingBudgetRows.map( + ({ budgetId, amount, unit, period }) => ({ + budgetId, amount, unit, period - })) - }); - } - - return onSubmit(values); + }) + ) + }); } function getTextImportDropHandlers(field: RoleTextImportField) { @@ -689,33 +704,18 @@ export function RoleForm({ {/* Inference Budget tab */}

- {variant === "edit" && role ? ( - - ) : ( - <> -

- {t( - "accessRoleInferenceBudgetDescription" - )} -

- - - )} +

+ {t("accessRoleInferenceBudgetDescription")} +

+
)} From 8b8e7913dc3ee0c43dfb18de339801fe4c1ab3c8 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 17:17:24 -0400 Subject: [PATCH 105/296] budget enforcement logic --- server/db/pg/schema/schema.ts | 96 ++++++++ server/db/sqlite/schema/schema.ts | 98 ++++++++ server/lib/aiBudgetEnforcement.ts | 341 +++++++++++++++++++++++++++ server/routers/aiGateway/pipeline.ts | 78 +++++- 4 files changed, 607 insertions(+), 6 deletions(-) create mode 100644 server/lib/aiBudgetEnforcement.ts diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 7ded27eb7..c9a45618c 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1758,6 +1758,100 @@ export const aiBudgets = pgTable( ] ); +export const aiUsageRecords = pgTable( + "aiUsageRecords", + { + id: serial("id").primaryKey(), + orgId: varchar("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + resourceId: integer("resourceId").references( + () => resources.resourceId, + { onDelete: "cascade" } + ), + siteResourceId: integer("siteResourceId").references( + () => siteResources.siteResourceId, + { onDelete: "cascade" } + ), + userId: varchar("userId").references(() => users.userId, { + onDelete: "set null" + }), + requestedModel: varchar("requestedModel").notNull(), + promptTokens: integer("promptTokens").notNull().default(0), + cacheReadTokens: integer("cacheReadTokens").notNull().default(0), + cacheWriteTokens: integer("cacheWriteTokens").notNull().default(0), + completionTokens: integer("completionTokens").notNull().default(0), + reasoningTokens: integer("reasoningTokens").notNull().default(0), + totalTokens: integer("totalTokens").notNull().default(0), + costUsd: real("costUsd"), + estimated: boolean("estimated").notNull().default(false), + createdAt: bigint("createdAt", { mode: "number" }).notNull() + }, + (t) => [ + index("idx_ai_usage_records_org_provider_created").on( + t.orgId, + t.providerId, + t.createdAt + ), + index("idx_ai_usage_records_org_resource_created").on( + t.orgId, + t.resourceId, + t.createdAt + ), + index("idx_ai_usage_records_org_site_resource_created").on( + t.orgId, + t.siteResourceId, + t.createdAt + ), + index("idx_ai_usage_records_org_user_created").on( + t.orgId, + t.userId, + t.createdAt + ) + ] +); + +export const aiBudgetBreachEvents = pgTable( + "aiBudgetBreachEvents", + { + id: serial("id").primaryKey(), + orgId: varchar("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + budgetId: integer("budgetId") + .notNull() + .references(() => aiBudgets.budgetId, { onDelete: "cascade" }), + enforcement: varchar("enforcement").$type<"hard" | "soft">().notNull(), + unit: varchar("unit").$type<"usd" | "tokens">().notNull(), + period: varchar("period") + .$type< + | "monthly" + | "yearly" + | "lifetime" + | "daily" + | "hourly" + | "weekly" + >() + .notNull(), + amount: real("amount").notNull(), + usageAmount: real("usageAmount").notNull(), + blocked: boolean("blocked").notNull(), + requestUserId: varchar("requestUserId").references(() => users.userId, { + onDelete: "set null" + }), + createdAt: bigint("createdAt", { mode: "number" }).notNull() + }, + (t) => [ + index("idx_ai_budget_breach_events_budget_created").on( + t.budgetId, + t.createdAt + ) + ] +); + export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1845,6 +1939,8 @@ export type ResourcePolicyRule = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; export type AiBudget = InferSelectModel; +export type AiUsageRecord = InferSelectModel; +export type AiBudgetBreachEvent = InferSelectModel; export type ResourceAiProvider = InferSelectModel; export type SiteResourceAiProvider = InferSelectModel< typeof siteResourceAiProviders diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 34fe67a0f..8017cd3b7 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1744,6 +1744,102 @@ export const aiBudgets = sqliteTable( ] ); +export const aiUsageRecords = sqliteTable( + "aiUsageRecords", + { + id: integer("id").primaryKey({ autoIncrement: true }), + orgId: text("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + resourceId: integer("resourceId").references( + () => resources.resourceId, + { onDelete: "cascade" } + ), + siteResourceId: integer("siteResourceId").references( + () => siteResources.siteResourceId, + { onDelete: "cascade" } + ), + userId: text("userId").references(() => users.userId, { + onDelete: "set null" + }), + requestedModel: text("requestedModel").notNull(), + promptTokens: integer("promptTokens").notNull().default(0), + cacheReadTokens: integer("cacheReadTokens").notNull().default(0), + cacheWriteTokens: integer("cacheWriteTokens").notNull().default(0), + completionTokens: integer("completionTokens").notNull().default(0), + reasoningTokens: integer("reasoningTokens").notNull().default(0), + totalTokens: integer("totalTokens").notNull().default(0), + costUsd: real("costUsd"), + estimated: integer("estimated", { mode: "boolean" }) + .notNull() + .default(false), + createdAt: integer("createdAt").notNull() + }, + (t) => [ + index("idx_ai_usage_records_org_provider_created").on( + t.orgId, + t.providerId, + t.createdAt + ), + index("idx_ai_usage_records_org_resource_created").on( + t.orgId, + t.resourceId, + t.createdAt + ), + index("idx_ai_usage_records_org_site_resource_created").on( + t.orgId, + t.siteResourceId, + t.createdAt + ), + index("idx_ai_usage_records_org_user_created").on( + t.orgId, + t.userId, + t.createdAt + ) + ] +); + +export const aiBudgetBreachEvents = sqliteTable( + "aiBudgetBreachEvents", + { + id: integer("id").primaryKey({ autoIncrement: true }), + orgId: text("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + budgetId: integer("budgetId") + .notNull() + .references(() => aiBudgets.budgetId, { onDelete: "cascade" }), + enforcement: text("enforcement").$type<"hard" | "soft">().notNull(), + unit: text("unit").$type<"usd" | "tokens">().notNull(), + period: text("period") + .$type< + | "monthly" + | "yearly" + | "lifetime" + | "daily" + | "hourly" + | "weekly" + >() + .notNull(), + amount: real("amount").notNull(), + usageAmount: real("usageAmount").notNull(), + blocked: integer("blocked", { mode: "boolean" }).notNull(), + requestUserId: text("requestUserId").references(() => users.userId, { + onDelete: "set null" + }), + createdAt: integer("createdAt").notNull() + }, + (t) => [ + index("idx_ai_budget_breach_events_budget_created").on( + t.budgetId, + t.createdAt + ) + ] +); + export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1829,6 +1925,8 @@ export type UserPolicy = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; export type AiBudget = InferSelectModel; +export type AiUsageRecord = InferSelectModel; +export type AiBudgetBreachEvent = InferSelectModel; export type ResourceAiProvider = InferSelectModel; export type SiteResourceAiProvider = InferSelectModel< typeof siteResourceAiProviders diff --git a/server/lib/aiBudgetEnforcement.ts b/server/lib/aiBudgetEnforcement.ts new file mode 100644 index 000000000..8f70340fa --- /dev/null +++ b/server/lib/aiBudgetEnforcement.ts @@ -0,0 +1,341 @@ +import { and, eq, gte, inArray, isNull, or, sql, SQL } from "drizzle-orm"; +import { + AiBudget, + aiBudgetBreachEvents, + aiBudgets, + aiModels, + aiUsageRecords, + db, + userOrgRoles +} from "@server/db"; +import { modelKeyMatches } from "@server/lib/aiModelKeyMatch"; +import type { AiUsage } from "@server/lib/aiUsageExtraction"; +import logger from "@server/logger"; + +type BudgetPeriod = AiBudget["period"]; + +const PERIOD_DURATIONS_MS: Record, number> = { + hourly: 60 * 60 * 1000, + daily: 24 * 60 * 60 * 1000, + weekly: 7 * 24 * 60 * 60 * 1000, + monthly: 30 * 24 * 60 * 60 * 1000, + yearly: 365 * 24 * 60 * 60 * 1000 +}; + +// Budget periods are trailing windows from "now", not calendar-aligned +// (e.g. "daily" = last 24h). "lifetime" has no lower bound. +function windowStart(period: BudgetPeriod, now: number): number { + if (period === "lifetime") { + return 0; + } + return now - PERIOD_DURATIONS_MS[period]; +} + +export type BudgetScopeContext = { + orgId: string; + providerId: number; + requestedModel: string; + resourceId: number | null; + siteResourceId: number | null; + roleIds: number[]; + requestUserId: string | null; +}; + +/** + * Every budget that could apply to this request: the provider itself, any + * model on that provider whose (possibly wildcarded) modelKey matches the + * requested model, the target resource/site-resource, and any role the + * requesting user holds in the org. + */ +export async function resolveApplicableBudgets( + ctx: BudgetScopeContext +): Promise { + const providerModels = await db + .select({ modelId: aiModels.modelId, modelKey: aiModels.modelKey }) + .from(aiModels) + .where( + and( + eq(aiModels.providerId, ctx.providerId), + eq(aiModels.enabled, true) + ) + ); + + const matchingModelIds = providerModels + .filter((m) => modelKeyMatches(m.modelKey, ctx.requestedModel)) + .map((m) => m.modelId); + + const scopeConditions: SQL[] = [ + and( + eq(aiBudgets.providerId, ctx.providerId), + isNull(aiBudgets.modelId) + )! + ]; + if (matchingModelIds.length > 0) { + scopeConditions.push(inArray(aiBudgets.modelId, matchingModelIds)); + } + if (ctx.resourceId != null) { + scopeConditions.push(eq(aiBudgets.resourceId, ctx.resourceId)); + } + if (ctx.siteResourceId != null) { + scopeConditions.push(eq(aiBudgets.siteResourceId, ctx.siteResourceId)); + } + if (ctx.roleIds.length > 0) { + scopeConditions.push(inArray(aiBudgets.roleId, ctx.roleIds)); + } + + return db + .select() + .from(aiBudgets) + .where( + and( + eq(aiBudgets.orgId, ctx.orgId), + eq(aiBudgets.enabled, true), + or(...scopeConditions) + ) + ); +} + +async function sumUsageAmount( + where: SQL, + unit: AiBudget["unit"] +): Promise { + const column = + unit === "usd" ? aiUsageRecords.costUsd : aiUsageRecords.totalTokens; + const [row] = await db + .select({ total: sql`coalesce(sum(${column}), 0)` }) + .from(aiUsageRecords) + .where(where); + return Number(row?.total ?? 0); +} + +/** + * Sums recorded usage for a single budget's scope + rolling window. Model + * budgets can't be pushed down to SQL because the model's key may itself be + * a glob, so those rows are fetched for the provider+window and matched in + * JS the same way access-control matching does. + */ +export async function sumUsageForBudget( + budget: AiBudget, + ctx: BudgetScopeContext, + now: number +): Promise { + const start = windowStart(budget.period, now); + + if (budget.modelId != null) { + const [model] = await db + .select({ + providerId: aiModels.providerId, + modelKey: aiModels.modelKey + }) + .from(aiModels) + .where(eq(aiModels.modelId, budget.modelId)) + .limit(1); + if (!model) { + return 0; + } + const rows = await db + .select({ + requestedModel: aiUsageRecords.requestedModel, + costUsd: aiUsageRecords.costUsd, + totalTokens: aiUsageRecords.totalTokens + }) + .from(aiUsageRecords) + .where( + and( + eq(aiUsageRecords.orgId, ctx.orgId), + eq(aiUsageRecords.providerId, model.providerId), + gte(aiUsageRecords.createdAt, start) + ) + ); + return rows + .filter((r) => modelKeyMatches(model.modelKey, r.requestedModel)) + .reduce( + (sum, r) => + sum + + (budget.unit === "usd" ? (r.costUsd ?? 0) : r.totalTokens), + 0 + ); + } + + if (budget.providerId != null) { + return sumUsageAmount( + and( + eq(aiUsageRecords.orgId, ctx.orgId), + eq(aiUsageRecords.providerId, budget.providerId), + gte(aiUsageRecords.createdAt, start) + )!, + budget.unit + ); + } + + if (budget.resourceId != null) { + return sumUsageAmount( + and( + eq(aiUsageRecords.orgId, ctx.orgId), + eq(aiUsageRecords.resourceId, budget.resourceId), + gte(aiUsageRecords.createdAt, start) + )!, + budget.unit + ); + } + + if (budget.siteResourceId != null) { + return sumUsageAmount( + and( + eq(aiUsageRecords.orgId, ctx.orgId), + eq(aiUsageRecords.siteResourceId, budget.siteResourceId), + gte(aiUsageRecords.createdAt, start) + )!, + budget.unit + ); + } + + if (budget.roleId != null) { + const members = await db + .select({ userId: userOrgRoles.userId }) + .from(userOrgRoles) + .where( + and( + eq(userOrgRoles.roleId, budget.roleId), + eq(userOrgRoles.orgId, ctx.orgId) + ) + ); + const userIds = members.map((m) => m.userId); + if (userIds.length === 0) { + return 0; + } + return sumUsageAmount( + and( + eq(aiUsageRecords.orgId, ctx.orgId), + inArray(aiUsageRecords.userId, userIds), + gte(aiUsageRecords.createdAt, start) + )!, + budget.unit + ); + } + + return 0; +} + +// Throttled to one durable event per budget per breach window, so a soft +// budget being exceeded doesn't write a row on every subsequent request +// while it stays over. +async function recordBreachEventIfNew( + budget: AiBudget, + ctx: BudgetScopeContext, + usageAmount: number, + now: number +): Promise { + try { + const start = windowStart(budget.period, now); + const [existing] = await db + .select({ id: aiBudgetBreachEvents.id }) + .from(aiBudgetBreachEvents) + .where( + and( + eq(aiBudgetBreachEvents.budgetId, budget.budgetId), + gte(aiBudgetBreachEvents.createdAt, start) + ) + ) + .limit(1); + if (existing) { + return; + } + + await db.insert(aiBudgetBreachEvents).values({ + orgId: ctx.orgId, + budgetId: budget.budgetId, + enforcement: budget.enforcement, + unit: budget.unit, + period: budget.period, + amount: budget.amount, + usageAmount, + blocked: budget.enforcement === "hard", + requestUserId: ctx.requestUserId, + createdAt: now + }); + } catch (error) { + logger.error("Failed to record AI budget breach event", { + error, + budgetId: budget.budgetId + }); + } +} + +export type BudgetCheckResult = { + blocked: boolean; + blockingBudget?: AiBudget; +}; + +export async function checkBudgets( + ctx: BudgetScopeContext +): Promise { + const budgets = await resolveApplicableBudgets(ctx); + if (budgets.length === 0) { + return { blocked: false }; + } + + const now = Date.now(); + let blockingBudget: AiBudget | undefined; + + for (const budget of budgets) { + const usage = await sumUsageForBudget(budget, ctx, now); + if (usage < budget.amount) { + continue; + } + + await recordBreachEventIfNew(budget, ctx, usage, now); + + if (budget.enforcement === "hard" && !blockingBudget) { + blockingBudget = budget; + } + } + + return blockingBudget + ? { blocked: true, blockingBudget } + : { blocked: false }; +} + +export type UsageRecordInput = { + orgId: string; + providerId: number; + resourceId: number | null; + siteResourceId: number | null; + userId: string | null; + requestedModel: string; + usage: AiUsage; + costUsd: number | null; + createdAt?: number; +}; + +export async function recordUsage(input: UsageRecordInput): Promise { + try { + const { usage } = input; + const totalTokens = + usage.promptTokens + + usage.cacheReadTokens + + usage.cacheWriteTokens + + usage.completionTokens + + usage.reasoningTokens; + + await db.insert(aiUsageRecords).values({ + orgId: input.orgId, + providerId: input.providerId, + resourceId: input.resourceId, + siteResourceId: input.siteResourceId, + userId: input.userId, + requestedModel: input.requestedModel, + promptTokens: usage.promptTokens, + cacheReadTokens: usage.cacheReadTokens, + cacheWriteTokens: usage.cacheWriteTokens, + completionTokens: usage.completionTokens, + reasoningTokens: usage.reasoningTokens, + totalTokens, + costUsd: input.costUsd, + estimated: usage.estimated, + createdAt: input.createdAt ?? Date.now() + }); + } catch (error) { + logger.error("Failed to record AI usage", { error }); + } +} diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index bcff1a501..802f5f453 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -51,6 +51,7 @@ import { } from "@server/lib/aiModelKeyMatch"; import { aiGatewayUpstreamFetch } from "@server/lib/aiGatewayUpstreamFetch"; import { getModelPricing, calculateAiCost } from "@server/lib/aiModelPricing"; +import { checkBudgets, recordUsage } from "@server/lib/aiBudgetEnforcement"; import { extractUsage, estimateUsage, @@ -141,6 +142,7 @@ export type RequestUser = { email: string | null; name: string | null; role: string | null; + roleIds: number[]; }; // Identity headers forwarded to the upstream inference endpoint when the @@ -193,7 +195,8 @@ async function buildRequestUser( username: user.username, email: user.email, name: user.name, - role: orgRoles.map((r) => r.roleName).join(", ") || null + role: orgRoles.map((r) => r.roleName).join(", ") || null, + roleIds: orgRoles.map((r) => r.roleId) }; localCache.set(cacheKey, requestUser, REQUEST_USER_TTL_SEC); @@ -526,6 +529,10 @@ function logAiUsageAndCost(args: { responseText: string; isStream: boolean; headers: Headers; + orgId: string | null; + resourceId: number | null; + siteResourceId: number | null; + requestUserId: string | null; }): void { const { capability, @@ -534,7 +541,11 @@ function logAiUsageAndCost(args: { requestBody, responseText, isStream, - headers + headers, + orgId, + resourceId, + siteResourceId, + requestUserId } = args; let usage: AiUsage | null = extractUsage( @@ -565,6 +576,19 @@ function logAiUsageAndCost(args: { pricingApproximate: pricing?.approximate ?? null, totalCostUsd: cost?.totalCost ?? null }); + + if (orgId) { + void recordUsage({ + orgId, + providerId: provider.providerId, + resourceId, + siteResourceId, + userId: requestUserId, + requestedModel: model ?? "unknown", + usage, + costUsd: cost?.totalCost ?? null + }); + } } export async function handleAiGatewayProxy( @@ -597,8 +621,13 @@ export async function handleAiGatewayProxy( }); } - const { attachments, resourceListsByProvider, resourceId, orgId } = - target; + const { + attachments, + resourceListsByProvider, + resourceId, + siteResourceId, + orgId + } = target; const capableAttachments = attachments.filter((a) => providerHasCapability(a.provider.capabilities, capability) @@ -637,6 +666,35 @@ export async function handleAiGatewayProxy( const { provider } = selection; + if (orgId) { + const budgetCheck = await checkBudgets({ + orgId, + providerId: provider.providerId, + requestedModel: requestedModel!, + resourceId, + siteResourceId, + roleIds: requestUser?.roleIds ?? [], + requestUserId: requestUser?.userId ?? null + }); + + if (budgetCheck.blocked) { + logger.warn("AI gateway request blocked by budget", { + budgetId: budgetCheck.blockingBudget?.budgetId, + orgId, + providerId: provider.providerId, + requestedModel, + resourceId, + siteResourceId, + userId: requestUser?.userId ?? null + }); + return res.status(HttpCode.TOO_MANY_REQUESTS).json({ + error: { + message: "AI usage budget exceeded for this request" + } + }); + } + } + if (provider.type === "custom" && provider.routingMode === "target") { return await proxyAiGatewayToSiteTarget( req, @@ -814,7 +872,11 @@ export async function handleAiGatewayProxy( requestBody: req.body, responseText: fullText, isStream: true, - headers: upstreamRes.headers + headers: upstreamRes.headers, + orgId, + resourceId, + siteResourceId, + requestUserId: requestUser?.userId ?? null }); } return; @@ -829,7 +891,11 @@ export async function handleAiGatewayProxy( requestBody: req.body, responseText: text, isStream: false, - headers: upstreamRes.headers + headers: upstreamRes.headers, + orgId, + resourceId, + siteResourceId, + requestUserId: requestUser?.userId ?? null }); return res.send(text); } catch (error) { From 1e3a9fb92112c4febd68891d6bc675eeefd0802c Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Mon, 10 Aug 2026 17:19:58 -0400 Subject: [PATCH 106/296] add virtual api key schema and crud endpoints --- cli/commands/rotateServerSecret.ts | 49 +++- server/auth/actions.ts | 7 +- server/db/pg/schema/schema.ts | 51 ++++ server/db/sqlite/schema/schema.ts | 55 ++++- server/index.ts | 4 +- server/lib/virtualApiKey.ts | 209 +++++++++++++++++ server/middlewares/index.ts | 1 + server/middlewares/integration/index.ts | 1 + .../integration/verifyVirtualApiKeyAccess.ts | 87 +++++++ .../middlewares/verifyVirtualApiKeyAccess.ts | 101 ++++++++ server/openApi.ts | 3 +- server/routers/external.ts | 42 +++- server/routers/integration.ts | 40 ++++ .../virtualApiKey/createVirtualApiKey.ts | 177 ++++++++++++++ .../virtualApiKey/deleteVirtualApiKey.ts | 87 +++++++ .../routers/virtualApiKey/getVirtualApiKey.ts | 96 ++++++++ server/routers/virtualApiKey/index.ts | 6 + .../virtualApiKey/listVirtualApiKeys.ts | 218 ++++++++++++++++++ server/routers/virtualApiKey/types.ts | 16 ++ .../virtualApiKey/updateVirtualApiKey.ts | 218 ++++++++++++++++++ server/routers/virtualApiKey/validation.ts | 23 ++ 21 files changed, 1485 insertions(+), 6 deletions(-) create mode 100644 server/lib/virtualApiKey.ts create mode 100644 server/middlewares/integration/verifyVirtualApiKeyAccess.ts create mode 100644 server/middlewares/verifyVirtualApiKeyAccess.ts create mode 100644 server/routers/virtualApiKey/createVirtualApiKey.ts create mode 100644 server/routers/virtualApiKey/deleteVirtualApiKey.ts create mode 100644 server/routers/virtualApiKey/getVirtualApiKey.ts create mode 100644 server/routers/virtualApiKey/index.ts create mode 100644 server/routers/virtualApiKey/listVirtualApiKeys.ts create mode 100644 server/routers/virtualApiKey/types.ts create mode 100644 server/routers/virtualApiKey/updateVirtualApiKey.ts create mode 100644 server/routers/virtualApiKey/validation.ts diff --git a/cli/commands/rotateServerSecret.ts b/cli/commands/rotateServerSecret.ts index 2edb7592d..1bacfa99c 100644 --- a/cli/commands/rotateServerSecret.ts +++ b/cli/commands/rotateServerSecret.ts @@ -1,5 +1,5 @@ import { CommandModule } from "yargs"; -import { db, idpOidcConfig, licenseKey, certificates, eventStreamingDestinations, alertWebhookActions, aiProviders } from "@server/db"; +import { db, idpOidcConfig, licenseKey, certificates, eventStreamingDestinations, alertWebhookActions, aiProviders, virtualApiKeys } from "@server/db"; import { encrypt, decrypt } from "@server/lib/crypto"; import { configFilePath1, configFilePath2 } from "@server/lib/consts"; import { eq } from "drizzle-orm"; @@ -133,6 +133,7 @@ export const rotateServerSecret: CommandModule< const streamingDestinations = await db.select().from(eventStreamingDestinations); const webhookActions = await db.select().from(alertWebhookActions); const providers = await db.select().from(aiProviders); + const virtualKeys = await db.select().from(virtualApiKeys); console.log(`Found ${idpConfigs.length} OIDC IdP configuration(s)`); console.log(`Found ${licenseKeys.length} license key(s)`); @@ -140,6 +141,7 @@ export const rotateServerSecret: CommandModule< console.log(`Found ${streamingDestinations.length} event streaming destination(s)`); console.log(`Found ${webhookActions.length} alert webhook action(s)`); console.log(`Found ${providers.length} AI provider(s)`); + console.log(`Found ${virtualKeys.length} virtual API key(s)`); // Prepare all decrypted and re-encrypted values console.log("\nDecrypting and re-encrypting values..."); @@ -179,12 +181,18 @@ export const rotateServerSecret: CommandModule< encryptedHeaders: string | null; }; + type VirtualApiKeyUpdate = { + virtualApiKeyId: string; + encryptedToken: string; + }; + const idpUpdates: IdpUpdate[] = []; const licenseKeyUpdates: LicenseKeyUpdate[] = []; const certUpdates: CertUpdate[] = []; const streamingDestinationUpdates: StreamingDestinationUpdate[] = []; const webhookActionUpdates: WebhookActionUpdate[] = []; const aiProviderUpdates: AiProviderUpdate[] = []; + const virtualApiKeyUpdates: VirtualApiKeyUpdate[] = []; // Process idpOidcConfig entries for (const idpConfig of idpConfigs) { @@ -346,6 +354,29 @@ export const rotateServerSecret: CommandModule< } } + // Process virtualApiKeys entries (token) + for (const key of virtualKeys) { + try { + if (!key.token) { + continue; + } + + virtualApiKeyUpdates.push({ + virtualApiKeyId: key.virtualApiKeyId, + encryptedToken: encrypt( + decrypt(key.token, oldSecret), + newSecret + ) + }); + } catch (error) { + console.error( + `Error processing virtual API key ${key.virtualApiKeyId}:`, + error + ); + throw error; + } + } + // Perform all database updates in a single transaction console.log("\nUpdating database in transaction..."); await db.transaction(async (trx) => { @@ -427,6 +458,21 @@ export const rotateServerSecret: CommandModule< }) .where(eq(aiProviders.providerId, update.providerId)); } + + // Update virtual API key entries + for (const update of virtualApiKeyUpdates) { + await trx + .update(virtualApiKeys) + .set({ + token: update.encryptedToken + }) + .where( + eq( + virtualApiKeys.virtualApiKeyId, + update.virtualApiKeyId + ) + ); + } }); console.log(`Rotated ${idpUpdates.length} OIDC IdP configuration(s)`); @@ -435,6 +481,7 @@ export const rotateServerSecret: CommandModule< console.log(`Rotated ${streamingDestinationUpdates.length} event streaming destination(s)`); console.log(`Rotated ${webhookActionUpdates.length} alert webhook action(s)`); console.log(`Rotated ${aiProviderUpdates.length} AI provider(s)`); + console.log(`Rotated ${virtualApiKeyUpdates.length} virtual API key(s)`); // Update config file with new secret console.log("\nUpdating config file..."); diff --git a/server/auth/actions.ts b/server/auth/actions.ts index f9d781169..385238e87 100644 --- a/server/auth/actions.ts +++ b/server/auth/actions.ts @@ -199,7 +199,12 @@ export enum ActionsEnum { deleteAiBudget = "deleteAiBudget", getAiBudget = "getAiBudget", listAiBudgets = "listAiBudgets", - updateAiBudget = "updateAiBudget" + updateAiBudget = "updateAiBudget", + createVirtualApiKey = "createVirtualApiKey", + deleteVirtualApiKey = "deleteVirtualApiKey", + getVirtualApiKey = "getVirtualApiKey", + listVirtualApiKeys = "listVirtualApiKeys", + updateVirtualApiKey = "updateVirtualApiKey" } export async function checkUserActionPermission( diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index c9a45618c..39dfb889e 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -12,6 +12,7 @@ import { serial, text, unique, + uniqueIndex, varchar } from "drizzle-orm/pg-core"; @@ -1242,6 +1243,52 @@ export const apiKeyOrg = pgTable("apiKeyOrg", { .notNull() }); +export const virtualApiKeys = pgTable( + "virtualApiKeys", + { + virtualApiKeyId: varchar("virtualApiKeyId").primaryKey(), + orgId: varchar("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + kind: varchar("kind").$type<"user" | "manual">().notNull(), + userId: varchar("userId").references(() => users.userId, { + onDelete: "cascade" + }), + name: varchar("name"), + description: varchar("description"), + token: varchar("token").notNull(), + lastChars: varchar("lastChars").notNull(), + allResources: boolean("allResources").notNull().default(false), + expiresAt: bigint("expiresAt", { mode: "number" }), + lastUsedAt: bigint("lastUsedAt", { mode: "number" }), + createdAt: bigint("createdAt", { mode: "number" }).notNull(), + createdByUserId: varchar("createdByUserId").references( + () => users.userId, + { onDelete: "set null" } + ) + }, + (t) => [ + uniqueIndex("virtual_api_key_user_identity_uniq") + .on(t.orgId, t.userId) + .where(sql`${t.kind} = 'user'`) + ] +); + +export const virtualApiKeyResources = pgTable( + "virtualApiKeyResources", + { + virtualApiKeyId: varchar("virtualApiKeyId") + .notNull() + .references(() => virtualApiKeys.virtualApiKeyId, { + onDelete: "cascade" + }), + resourceId: integer("resourceId") + .notNull() + .references(() => resources.resourceId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.virtualApiKeyId, t.resourceId] })] +); + export const idpOrg = pgTable("idpOrg", { idpId: integer("idpId") .notNull() @@ -1907,6 +1954,10 @@ export type Idp = InferSelectModel; export type ApiKey = InferSelectModel; export type ApiKeyAction = InferSelectModel; export type ApiKeyOrg = InferSelectModel; +export type VirtualApiKey = InferSelectModel; +export type VirtualApiKeyResource = InferSelectModel< + typeof virtualApiKeyResources +>; export type Client = InferSelectModel; export type ClientSite = InferSelectModel; export type Olm = InferSelectModel; diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 8017cd3b7..1c2236dfe 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -8,7 +8,8 @@ import { real, sqliteTable, text, - unique + unique, + uniqueIndex } from "drizzle-orm/sqlite-core"; export const domains = sqliteTable("domains", { @@ -1499,6 +1500,54 @@ export const apiKeyOrg = sqliteTable("apiKeyOrg", { .notNull() }); +export const virtualApiKeys = sqliteTable( + "virtualApiKeys", + { + virtualApiKeyId: text("virtualApiKeyId").primaryKey(), + orgId: text("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + kind: text("kind").$type<"user" | "manual">().notNull(), + userId: text("userId").references(() => users.userId, { + onDelete: "cascade" + }), + name: text("name"), + description: text("description"), + token: text("token").notNull(), + lastChars: text("lastChars").notNull(), + allResources: integer("allResources", { mode: "boolean" }) + .notNull() + .default(false), + expiresAt: integer("expiresAt"), + lastUsedAt: integer("lastUsedAt"), + createdAt: integer("createdAt").notNull(), + createdByUserId: text("createdByUserId").references( + () => users.userId, + { onDelete: "set null" } + ) + }, + (t) => [ + uniqueIndex("virtual_api_key_user_identity_uniq") + .on(t.orgId, t.userId) + .where(sql`${t.kind} = 'user'`) + ] +); + +export const virtualApiKeyResources = sqliteTable( + "virtualApiKeyResources", + { + virtualApiKeyId: text("virtualApiKeyId") + .notNull() + .references(() => virtualApiKeys.virtualApiKeyId, { + onDelete: "cascade" + }), + resourceId: integer("resourceId") + .notNull() + .references(() => resources.resourceId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.virtualApiKeyId, t.resourceId] })] +); + export const idpOrg = sqliteTable("idpOrg", { idpId: integer("idpId") .notNull() @@ -1891,6 +1940,10 @@ export type Idp = InferSelectModel; export type ApiKey = InferSelectModel; export type ApiKeyAction = InferSelectModel; export type ApiKeyOrg = InferSelectModel; +export type VirtualApiKey = InferSelectModel; +export type VirtualApiKeyResource = InferSelectModel< + typeof virtualApiKeyResources +>; export type SiteResource = InferSelectModel; export type Network = InferSelectModel; export type OrgDomains = InferSelectModel; diff --git a/server/index.ts b/server/index.ts index 3eb16092a..c7b0a5b6e 100644 --- a/server/index.ts +++ b/server/index.ts @@ -17,7 +17,8 @@ import { Session, SiteResource, User, - UserOrg + UserOrg, + VirtualApiKey } from "@server/db"; import config from "@server/lib/config"; import { setHostMeta } from "@server/lib/hostMeta"; @@ -94,6 +95,7 @@ declare global { aiProvider?: AiProvider; aiModel?: AiModel; aiBudget?: AiBudget; + virtualApiKey?: VirtualApiKey; orgPolicyAllowed?: boolean; } } diff --git a/server/lib/virtualApiKey.ts b/server/lib/virtualApiKey.ts new file mode 100644 index 000000000..15f9b52ff --- /dev/null +++ b/server/lib/virtualApiKey.ts @@ -0,0 +1,209 @@ +import { + generateId, + generateIdFromEntropySize +} from "@server/auth/sessions/app"; +import { + db, + resources, + virtualApiKeyResources, + virtualApiKeys, + type Transaction, + type VirtualApiKey +} from "@server/db"; +import config from "@server/lib/config"; +import { decrypt, encrypt } from "@server/lib/crypto"; +import { and, eq, inArray } from "drizzle-orm"; + +export type MintedVirtualApiKeySecret = { + virtualApiKeyId: string; + secret: string; + lastChars: string; +}; + +export type PublicVirtualApiKey = Omit & { + secret?: string; +}; + +export function mintVirtualApiKeySecret(): MintedVirtualApiKeySecret { + const secret = generateIdFromEntropySize(16); + return { + virtualApiKeyId: generateId(8), + secret, + lastChars: secret.slice(-4) + }; +} + +export function encryptVirtualApiKeyToken(secret: string): string { + return encrypt(secret, config.getRawConfig().server.secret!); +} + +export function decryptVirtualApiKeyToken(ciphertext: string): string { + return decrypt(ciphertext, config.getRawConfig().server.secret!); +} + +export function toPublicVirtualApiKey( + row: VirtualApiKey, + options?: { includeSecret?: boolean } +): PublicVirtualApiKey { + const { token, ...rest } = row; + if (!options?.includeSecret) { + return rest; + } + return { + ...rest, + secret: decryptVirtualApiKeyToken(token) + }; +} + +export async function assertManualKeyResourcesInOrg(params: { + allResources: boolean; + resourceIds: number[]; + orgId: string; +}): Promise<{ ok: true } | { ok: false; message: string }> { + const { allResources, resourceIds, orgId } = params; + + if (allResources || resourceIds.length === 0) { + return { ok: true }; + } + + const uniqueIds = [...new Set(resourceIds)]; + const rows = await db + .select({ resourceId: resources.resourceId }) + .from(resources) + .where( + and( + eq(resources.orgId, orgId), + inArray(resources.resourceId, uniqueIds) + ) + ); + + if (rows.length !== uniqueIds.length) { + return { + ok: false, + message: "One or more resources are invalid for this organization" + }; + } + + return { ok: true }; +} + +export async function replaceVirtualApiKeyResources( + trx: Transaction | typeof db, + virtualApiKeyId: string, + resourceIds: number[] +): Promise { + await trx + .delete(virtualApiKeyResources) + .where(eq(virtualApiKeyResources.virtualApiKeyId, virtualApiKeyId)); + + const uniqueIds = [...new Set(resourceIds)]; + if (uniqueIds.length === 0) { + return; + } + + await trx.insert(virtualApiKeyResources).values( + uniqueIds.map((resourceId) => ({ + virtualApiKeyId, + resourceId + })) + ); +} + +async function selectUserVirtualApiKey( + orgId: string, + userId: string +): Promise { + const [existing] = await db + .select() + .from(virtualApiKeys) + .where( + and( + eq(virtualApiKeys.orgId, orgId), + eq(virtualApiKeys.userId, userId), + eq(virtualApiKeys.kind, "user") + ) + ) + .limit(1); + + return existing ?? null; +} + +export async function getOrCreateUserVirtualApiKey(params: { + orgId: string; + userId: string; + createdByUserId?: string | null; +}): Promise<{ key: VirtualApiKey; secret: string }> { + const { orgId, userId, createdByUserId } = params; + + const existing = await selectUserVirtualApiKey(orgId, userId); + if (existing) { + return { + key: existing, + secret: decryptVirtualApiKeyToken(existing.token) + }; + } + + const minted = mintVirtualApiKeySecret(); + const now = Date.now(); + + try { + const [created] = await db + .insert(virtualApiKeys) + .values({ + virtualApiKeyId: minted.virtualApiKeyId, + orgId, + kind: "user", + userId, + name: null, + description: null, + token: encryptVirtualApiKeyToken(minted.secret), + lastChars: minted.lastChars, + allResources: false, + expiresAt: null, + lastUsedAt: null, + createdAt: now, + createdByUserId: createdByUserId ?? null + }) + .returning(); + + return { key: created, secret: minted.secret }; + } catch { + const raced = await selectUserVirtualApiKey(orgId, userId); + if (raced) { + return { + key: raced, + secret: decryptVirtualApiKeyToken(raced.token) + }; + } + throw new Error("Failed to create user virtual API key"); + } +} + +export async function rotateUserVirtualApiKey(params: { + orgId: string; + userId: string; + createdByUserId?: string | null; +}): Promise<{ key: VirtualApiKey; secret: string }> { + const { orgId, userId, createdByUserId } = params; + const existing = await selectUserVirtualApiKey(orgId, userId); + + if (!existing) { + return getOrCreateUserVirtualApiKey(params); + } + + const minted = mintVirtualApiKeySecret(); + const [updated] = await db + .update(virtualApiKeys) + .set({ + token: encryptVirtualApiKeyToken(minted.secret), + lastChars: minted.lastChars, + createdByUserId: + createdByUserId !== undefined + ? createdByUserId + : existing.createdByUserId + }) + .where(eq(virtualApiKeys.virtualApiKeyId, existing.virtualApiKeyId)) + .returning(); + + return { key: updated, secret: minted.secret }; +} diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts index 7242ea0cf..cd09add8a 100644 --- a/server/middlewares/index.ts +++ b/server/middlewares/index.ts @@ -30,6 +30,7 @@ export * from "./verifyDomainAccess"; export * from "./verifyAiProviderAccess"; export * from "./verifyAiModelAccess"; export * from "./verifyAiBudgetAccess"; +export * from "./verifyVirtualApiKeyAccess"; export * from "./verifyUserIsOrgOwner"; export * from "./verifyUserFromResourceSession"; export * from "./verifySiteResourceAccess"; diff --git a/server/middlewares/integration/index.ts b/server/middlewares/integration/index.ts index 63df4b2bb..99a76cdf8 100644 --- a/server/middlewares/integration/index.ts +++ b/server/middlewares/integration/index.ts @@ -20,3 +20,4 @@ export * from "./verifyApiKeyAiProviderAccess"; export * from "./verifyApiKeyAiModelAccess"; export * from "./verifyApiKeyResourcePolicyAccess"; export * from "./verifyApiKeySiteProvisioningKeyAccess"; +export * from "./verifyVirtualApiKeyAccess"; diff --git a/server/middlewares/integration/verifyVirtualApiKeyAccess.ts b/server/middlewares/integration/verifyVirtualApiKeyAccess.ts new file mode 100644 index 000000000..a7dd4706a --- /dev/null +++ b/server/middlewares/integration/verifyVirtualApiKeyAccess.ts @@ -0,0 +1,87 @@ +import { Request, Response, NextFunction } from "express"; +import { apiKeyOrg, db, virtualApiKeys } from "@server/db"; +import { and, eq } from "drizzle-orm"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { getFirstString } from "@server/lib/requestParams"; + +export async function verifyApiKeyVirtualApiKeyAccess( + req: Request, + res: Response, + next: NextFunction +) { + try { + const apiKey = req.apiKey; + const virtualApiKeyId = getFirstString(req.params.virtualApiKeyId); + + if (!apiKey) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "Key not authenticated") + ); + } + + if (!virtualApiKeyId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Invalid virtual API key ID" + ) + ); + } + + const [key] = await db + .select() + .from(virtualApiKeys) + .where(eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId)) + .limit(1); + + if (!key || key.kind !== "manual") { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Virtual API key with ID ${virtualApiKeyId} not found` + ) + ); + } + + if (apiKey.isRoot) { + req.virtualApiKey = key; + return next(); + } + + const orgId = key.orgId; + + if (!req.apiKeyOrg || req.apiKeyOrg.orgId !== orgId) { + const apiKeyOrgRes = await db + .select() + .from(apiKeyOrg) + .where( + and( + eq(apiKeyOrg.apiKeyId, apiKey.apiKeyId), + eq(apiKeyOrg.orgId, orgId) + ) + ) + .limit(1); + req.apiKeyOrg = apiKeyOrgRes[0]; + } + + if (!req.apiKeyOrg) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "Key does not have access to this organization" + ) + ); + } + + req.virtualApiKey = key; + return next(); + } catch (error) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Error verifying virtual API key access" + ) + ); + } +} diff --git a/server/middlewares/verifyVirtualApiKeyAccess.ts b/server/middlewares/verifyVirtualApiKeyAccess.ts new file mode 100644 index 000000000..a2c27fd90 --- /dev/null +++ b/server/middlewares/verifyVirtualApiKeyAccess.ts @@ -0,0 +1,101 @@ +import { Request, Response, NextFunction } from "express"; +import { db, userOrgs, virtualApiKeys } from "@server/db"; +import { and, eq } from "drizzle-orm"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy"; +import { getUserOrgRoleIds } from "@server/lib/userOrgRoles"; +import { getFirstString } from "@server/lib/requestParams"; + +export async function verifyVirtualApiKeyAccess( + req: Request, + res: Response, + next: NextFunction +) { + try { + const userId = req.user!.userId; + const virtualApiKeyId = getFirstString(req.params.virtualApiKeyId); + + if (!userId) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "User not authenticated") + ); + } + + if (!virtualApiKeyId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Invalid virtual API key ID" + ) + ); + } + + const [key] = await db + .select() + .from(virtualApiKeys) + .where(eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId)) + .limit(1); + + if (!key || key.kind !== "manual") { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Virtual API key with ID ${virtualApiKeyId} not found` + ) + ); + } + + const orgId = key.orgId; + + if (!req.userOrg || req.userOrg.orgId !== orgId) { + const userOrgRole = await db + .select() + .from(userOrgs) + .where( + and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)) + ) + .limit(1); + req.userOrg = userOrgRole[0]; + } + + if (!req.userOrg) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + if (req.orgPolicyAllowed === undefined && req.userOrg.orgId) { + const policyCheck = await checkOrgAccessPolicy({ + orgId: req.userOrg.orgId, + userId, + session: req.session + }); + req.orgPolicyAllowed = policyCheck.allowed; + if (!policyCheck.allowed || policyCheck.error) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "" + (policyCheck.error || "Unknown error") + ) + ); + } + } + + req.userOrgId = orgId; + req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId); + req.virtualApiKey = key; + + return next(); + } catch (error) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Error verifying virtual API key access" + ) + ); + } +} diff --git a/server/openApi.ts b/server/openApi.ts index 920aaa471..b4b7c1a7a 100644 --- a/server/openApi.ts +++ b/server/openApi.ts @@ -31,7 +31,8 @@ export enum OpenAPITags { PrivateResourceLegacy = "Private Resource (Legacy)", AiProvider = "AI Provider", AiModel = "AI Model", - AiBudget = "AI Budget" + AiBudget = "AI Budget", + VirtualApiKey = "Virtual API Key" } // Order here controls the order tags are displayed in Swagger UI diff --git a/server/routers/external.ts b/server/routers/external.ts index 9e1a5f210..1eb32ec0f 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -48,7 +48,8 @@ import { verifyResourcePolicyAccess, verifyAiProviderAccess, verifyAiModelAccess, - verifyAiBudgetAccess + verifyAiBudgetAccess, + verifyVirtualApiKeyAccess } from "@server/middlewares"; import { ActionsEnum } from "@server/auth/actions"; import rateLimit, { ipKeyGenerator } from "express-rate-limit"; @@ -60,6 +61,7 @@ import { checkRoundTripMessage } from "./ws"; import * as labels from "@server/routers/labels"; import * as aiProvider from "@server/routers/aiProvider"; import * as aiBudget from "@server/routers/aiBudget"; +import * as virtualApiKey from "@server/routers/virtualApiKey"; // Root routes export const unauthenticated = Router(); @@ -1633,6 +1635,44 @@ authenticated.delete( aiBudget.deleteAiBudget ); +authenticated.put( + "/org/:orgId/virtual-api-key", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.createVirtualApiKey), + logActionAudit(ActionsEnum.createVirtualApiKey), + virtualApiKey.createVirtualApiKey +); + +authenticated.get( + "/org/:orgId/virtual-api-keys", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.listVirtualApiKeys), + virtualApiKey.listVirtualApiKeys +); + +authenticated.get( + "/virtual-api-key/:virtualApiKeyId", + verifyVirtualApiKeyAccess, + verifyUserHasAction(ActionsEnum.getVirtualApiKey), + virtualApiKey.getVirtualApiKey +); + +authenticated.post( + "/virtual-api-key/:virtualApiKeyId", + verifyVirtualApiKeyAccess, + verifyUserHasAction(ActionsEnum.updateVirtualApiKey), + logActionAudit(ActionsEnum.updateVirtualApiKey), + virtualApiKey.updateVirtualApiKey +); + +authenticated.delete( + "/virtual-api-key/:virtualApiKeyId", + verifyVirtualApiKeyAccess, + verifyUserHasAction(ActionsEnum.deleteVirtualApiKey), + logActionAudit(ActionsEnum.deleteVirtualApiKey), + virtualApiKey.deleteVirtualApiKey +); + authenticated.get( "/ai-provider/:providerId/ai-budgets", verifyAiProviderAccess, diff --git a/server/routers/integration.ts b/server/routers/integration.ts index e08c14920..24c9955da 100644 --- a/server/routers/integration.ts +++ b/server/routers/integration.ts @@ -14,6 +14,7 @@ import * as idp from "./idp"; import * as logs from "./auditLogs"; import * as siteResource from "./siteResource"; import * as aiProvider from "./aiProvider"; +import * as virtualApiKey from "./virtualApiKey"; import { verifyApiKey, verifyApiKeyOrgAccess, @@ -34,6 +35,7 @@ import { verifyApiKeyResourcePolicyAccess, verifyApiKeyAiProviderAccess, verifyApiKeyAiModelAccess, + verifyApiKeyVirtualApiKeyAccess, verifyUserHasAction } from "@server/middlewares"; import HttpCode from "@server/types/HttpCode"; @@ -1633,3 +1635,41 @@ authenticated.delete( logActionAudit(ActionsEnum.deleteAiModel), aiProvider.deleteAiModel ); + +authenticated.put( + "/org/:orgId/virtual-api-key", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.createVirtualApiKey), + logActionAudit(ActionsEnum.createVirtualApiKey), + virtualApiKey.createVirtualApiKey +); + +authenticated.get( + "/org/:orgId/virtual-api-keys", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.listVirtualApiKeys), + virtualApiKey.listVirtualApiKeys +); + +authenticated.get( + "/virtual-api-key/:virtualApiKeyId", + verifyApiKeyVirtualApiKeyAccess, + verifyApiKeyHasAction(ActionsEnum.getVirtualApiKey), + virtualApiKey.getVirtualApiKey +); + +authenticated.post( + "/virtual-api-key/:virtualApiKeyId", + verifyApiKeyVirtualApiKeyAccess, + verifyApiKeyHasAction(ActionsEnum.updateVirtualApiKey), + logActionAudit(ActionsEnum.updateVirtualApiKey), + virtualApiKey.updateVirtualApiKey +); + +authenticated.delete( + "/virtual-api-key/:virtualApiKeyId", + verifyApiKeyVirtualApiKeyAccess, + verifyApiKeyHasAction(ActionsEnum.deleteVirtualApiKey), + logActionAudit(ActionsEnum.deleteVirtualApiKey), + virtualApiKey.deleteVirtualApiKey +); diff --git a/server/routers/virtualApiKey/createVirtualApiKey.ts b/server/routers/virtualApiKey/createVirtualApiKey.ts new file mode 100644 index 000000000..5e49a5717 --- /dev/null +++ b/server/routers/virtualApiKey/createVirtualApiKey.ts @@ -0,0 +1,177 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, userOrgs, virtualApiKeys } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq } from "drizzle-orm"; +import { createDate, TimeSpan } from "oslo"; +import { + assertManualKeyResourcesInOrg, + encryptVirtualApiKeyToken, + mintVirtualApiKeySecret, + replaceVirtualApiKeyResources, + toPublicVirtualApiKey +} from "@server/lib/virtualApiKey"; +import type { CreateOrEditVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; +import { createVirtualApiKeyBodySchema } from "@server/routers/virtualApiKey/validation"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +registry.registerPath({ + method: "put", + path: "/org/{orgId}/virtual-api-key", + description: "Create a manual virtual API key for an organization.", + tags: [OpenAPITags.VirtualApiKey], + request: { + params: paramsSchema, + body: { + content: { + "application/json": { + schema: createVirtualApiKeyBodySchema + } + } + } + }, + responses: { + 201: { + description: "Successful response" + } + } +}); + +export async function createVirtualApiKey( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedBody = createVirtualApiKeyBodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { orgId } = parsedParams.data; + const { + name, + description, + userId, + allResources, + resourceIds, + validForSeconds + } = parsedBody.data; + + if (req.user && orgId && orgId !== req.userOrgId) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + if (userId) { + const [membership] = await db + .select() + .from(userOrgs) + .where( + and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)) + ) + .limit(1); + + if (!membership) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "User is not a member of this organization" + ) + ); + } + } + + const assignedResourceIds = allResources ? [] : (resourceIds ?? []); + const resourceCheck = await assertManualKeyResourcesInOrg({ + allResources, + resourceIds: assignedResourceIds, + orgId + }); + if (!resourceCheck.ok) { + return next( + createHttpError(HttpCode.BAD_REQUEST, resourceCheck.message) + ); + } + + const minted = mintVirtualApiKeySecret(); + const expiresAt = validForSeconds + ? createDate(new TimeSpan(validForSeconds, "s")).getTime() + : null; + const now = Date.now(); + + const created = await db.transaction(async (trx) => { + const [row] = await trx + .insert(virtualApiKeys) + .values({ + virtualApiKeyId: minted.virtualApiKeyId, + orgId, + kind: "manual", + userId: userId ?? null, + name, + description: description ?? null, + token: encryptVirtualApiKeyToken(minted.secret), + lastChars: minted.lastChars, + allResources, + expiresAt, + lastUsedAt: null, + createdAt: now, + createdByUserId: req.user?.userId ?? null + }) + .returning(); + + await replaceVirtualApiKeyResources( + trx, + row.virtualApiKeyId, + assignedResourceIds + ); + + return row; + }); + + return response(res, { + data: { + virtualApiKey: { + ...toPublicVirtualApiKey(created, { includeSecret: true }), + resourceIds: assignedResourceIds + } + }, + success: true, + error: false, + message: "Virtual API key created successfully", + status: HttpCode.CREATED + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/virtualApiKey/deleteVirtualApiKey.ts b/server/routers/virtualApiKey/deleteVirtualApiKey.ts new file mode 100644 index 000000000..c5223633a --- /dev/null +++ b/server/routers/virtualApiKey/deleteVirtualApiKey.ts @@ -0,0 +1,87 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, virtualApiKeys } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { eq } from "drizzle-orm"; + +const paramsSchema = z.strictObject({ + virtualApiKeyId: z.string().nonempty() +}); + +registry.registerPath({ + method: "delete", + path: "/virtual-api-key/{virtualApiKeyId}", + description: "Delete a manual virtual API key.", + tags: [OpenAPITags.VirtualApiKey], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function deleteVirtualApiKey( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { virtualApiKeyId } = parsedParams.data; + + const [existing] = + req.virtualApiKey && + req.virtualApiKey.virtualApiKeyId === virtualApiKeyId + ? [req.virtualApiKey] + : await db + .select() + .from(virtualApiKeys) + .where( + eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId) + ) + .limit(1); + + if (!existing || existing.kind !== "manual") { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Virtual API key with ID ${virtualApiKeyId} not found` + ) + ); + } + + await db + .delete(virtualApiKeys) + .where(eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId)); + + return response(res, { + data: null, + success: true, + error: false, + message: "Virtual API key deleted successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/virtualApiKey/getVirtualApiKey.ts b/server/routers/virtualApiKey/getVirtualApiKey.ts new file mode 100644 index 000000000..fddae3aef --- /dev/null +++ b/server/routers/virtualApiKey/getVirtualApiKey.ts @@ -0,0 +1,96 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, virtualApiKeyResources, virtualApiKeys } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { eq } from "drizzle-orm"; +import { toPublicVirtualApiKey } from "@server/lib/virtualApiKey"; +import type { GetVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; + +const paramsSchema = z.strictObject({ + virtualApiKeyId: z.string().nonempty() +}); + +registry.registerPath({ + method: "get", + path: "/virtual-api-key/{virtualApiKeyId}", + description: + "Get a manual virtual API key by ID, including the decrypted secret.", + tags: [OpenAPITags.VirtualApiKey], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function getVirtualApiKey( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { virtualApiKeyId } = parsedParams.data; + + const [key] = + req.virtualApiKey && + req.virtualApiKey.virtualApiKeyId === virtualApiKeyId + ? [req.virtualApiKey] + : await db + .select() + .from(virtualApiKeys) + .where( + eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId) + ) + .limit(1); + + if (!key || key.kind !== "manual") { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Virtual API key with ID ${virtualApiKeyId} not found` + ) + ); + } + + const resourceRows = await db + .select({ resourceId: virtualApiKeyResources.resourceId }) + .from(virtualApiKeyResources) + .where(eq(virtualApiKeyResources.virtualApiKeyId, virtualApiKeyId)); + + return response(res, { + data: { + virtualApiKey: { + ...toPublicVirtualApiKey(key, { includeSecret: true }), + resourceIds: resourceRows.map((row) => row.resourceId) + } + }, + success: true, + error: false, + message: "Virtual API key retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/virtualApiKey/index.ts b/server/routers/virtualApiKey/index.ts new file mode 100644 index 000000000..b80fba765 --- /dev/null +++ b/server/routers/virtualApiKey/index.ts @@ -0,0 +1,6 @@ +export * from "./createVirtualApiKey"; +export * from "./listVirtualApiKeys"; +export * from "./getVirtualApiKey"; +export * from "./updateVirtualApiKey"; +export * from "./deleteVirtualApiKey"; +export * from "./types"; diff --git a/server/routers/virtualApiKey/listVirtualApiKeys.ts b/server/routers/virtualApiKey/listVirtualApiKeys.ts new file mode 100644 index 000000000..be7250f1e --- /dev/null +++ b/server/routers/virtualApiKey/listVirtualApiKeys.ts @@ -0,0 +1,218 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { + db, + virtualApiKeyResources, + virtualApiKeys, + type VirtualApiKey +} from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, asc, eq, exists, inArray, like, or, sql } from "drizzle-orm"; +import { toPublicVirtualApiKey } from "@server/lib/virtualApiKey"; +import type { ListVirtualApiKeysResponse } from "@server/routers/virtualApiKey/types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +const listSchema = z.object({ + pageSize: z.coerce + .number() + .int() + .positive() + .optional() + .catch(20) + .default(20) + .openapi({ + type: "integer", + default: 20, + description: "Number of items per page" + }), + page: z.coerce + .number() + .int() + .min(0) + .optional() + .catch(1) + .default(1) + .openapi({ + type: "integer", + default: 1, + description: "Page number to retrieve" + }), + search: z.string().optional(), + userId: z.string().optional(), + resourceId: z.coerce.number().int().positive().optional() +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/virtual-api-keys", + description: "List manual virtual API keys for an organization.", + tags: [OpenAPITags.VirtualApiKey], + request: { + params: paramsSchema, + query: listSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listVirtualApiKeys( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = listSchema.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error).toString() + ) + ); + } + + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { orgId } = parsedParams.data; + + if (req.user && orgId && orgId !== req.userOrgId) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + const { pageSize, page, search, userId, resourceId } = parsedQuery.data; + const conditions = [ + eq(virtualApiKeys.orgId, orgId), + eq(virtualApiKeys.kind, "manual") + ]; + + if (userId) { + conditions.push(eq(virtualApiKeys.userId, userId)); + } + + if (search) { + const term = "%" + search.toLowerCase() + "%"; + conditions.push( + or( + like(sql`LOWER(${virtualApiKeys.name})`, term), + like(sql`LOWER(${virtualApiKeys.description})`, term), + like(sql`LOWER(${virtualApiKeys.lastChars})`, term) + )! + ); + } + + if (resourceId !== undefined) { + conditions.push( + or( + eq(virtualApiKeys.allResources, true), + exists( + db + .select() + .from(virtualApiKeyResources) + .where( + and( + eq( + virtualApiKeyResources.virtualApiKeyId, + virtualApiKeys.virtualApiKeyId + ), + eq( + virtualApiKeyResources.resourceId, + resourceId + ) + ) + ) + ) + )! + ); + } + + const whereClause = and(...conditions); + + const [totalCount, rows] = await Promise.all([ + db.$count( + db + .select() + .from(virtualApiKeys) + .where(whereClause) + .as("filtered_virtual_api_keys") + ), + db + .select() + .from(virtualApiKeys) + .where(whereClause) + .limit(pageSize) + .offset(pageSize * (page - 1)) + .orderBy( + asc(virtualApiKeys.name), + asc(virtualApiKeys.createdAt) + ) + ]); + + const keyIds = rows.map((row) => row.virtualApiKeyId); + const resourceRows = + keyIds.length === 0 + ? [] + : await db + .select() + .from(virtualApiKeyResources) + .where( + inArray( + virtualApiKeyResources.virtualApiKeyId, + keyIds + ) + ); + + const resourceIdsByKey = new Map(); + for (const row of resourceRows) { + const existing = resourceIdsByKey.get(row.virtualApiKeyId) ?? []; + existing.push(row.resourceId); + resourceIdsByKey.set(row.virtualApiKeyId, existing); + } + + return response(res, { + data: { + virtualApiKeys: rows.map((row: VirtualApiKey) => ({ + ...toPublicVirtualApiKey(row), + resourceIds: resourceIdsByKey.get(row.virtualApiKeyId) ?? [] + })), + pagination: { + total: totalCount, + pageSize, + page + } + }, + success: true, + error: false, + message: "Virtual API keys retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/virtualApiKey/types.ts b/server/routers/virtualApiKey/types.ts new file mode 100644 index 000000000..28823577f --- /dev/null +++ b/server/routers/virtualApiKey/types.ts @@ -0,0 +1,16 @@ +import type { PublicVirtualApiKey } from "@server/lib/virtualApiKey"; +import type { PaginatedResponse } from "@server/types/Pagination"; + +export type { PublicVirtualApiKey }; + +export type ListVirtualApiKeysResponse = PaginatedResponse<{ + virtualApiKeys: (PublicVirtualApiKey & { resourceIds: number[] })[]; +}>; + +export type GetVirtualApiKeyResponse = { + virtualApiKey: PublicVirtualApiKey & { resourceIds: number[] }; +}; + +export type CreateOrEditVirtualApiKeyResponse = { + virtualApiKey: PublicVirtualApiKey & { resourceIds: number[] }; +}; diff --git a/server/routers/virtualApiKey/updateVirtualApiKey.ts b/server/routers/virtualApiKey/updateVirtualApiKey.ts new file mode 100644 index 000000000..42554803f --- /dev/null +++ b/server/routers/virtualApiKey/updateVirtualApiKey.ts @@ -0,0 +1,218 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { + db, + userOrgs, + virtualApiKeyResources, + virtualApiKeys +} from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq } from "drizzle-orm"; +import { createDate, TimeSpan } from "oslo"; +import { + assertManualKeyResourcesInOrg, + replaceVirtualApiKeyResources, + toPublicVirtualApiKey +} from "@server/lib/virtualApiKey"; +import type { CreateOrEditVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; +import { updateVirtualApiKeyBodySchema } from "@server/routers/virtualApiKey/validation"; + +const paramsSchema = z.strictObject({ + virtualApiKeyId: z.string().nonempty() +}); + +registry.registerPath({ + method: "post", + path: "/virtual-api-key/{virtualApiKeyId}", + description: + "Update a manual virtual API key metadata and resource assignment.", + tags: [OpenAPITags.VirtualApiKey], + request: { + params: paramsSchema, + body: { + content: { + "application/json": { + schema: updateVirtualApiKeyBodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function updateVirtualApiKey( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedBody = updateVirtualApiKeyBodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { virtualApiKeyId } = parsedParams.data; + const body = parsedBody.data; + + const [existing] = + req.virtualApiKey && + req.virtualApiKey.virtualApiKeyId === virtualApiKeyId + ? [req.virtualApiKey] + : await db + .select() + .from(virtualApiKeys) + .where( + eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId) + ) + .limit(1); + + if (!existing || existing.kind !== "manual") { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Virtual API key with ID ${virtualApiKeyId} not found` + ) + ); + } + + if (body.userId) { + const [membership] = await db + .select() + .from(userOrgs) + .where( + and( + eq(userOrgs.userId, body.userId), + eq(userOrgs.orgId, existing.orgId) + ) + ) + .limit(1); + + if (!membership) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "User is not a member of this organization" + ) + ); + } + } + + const nextAllResources = + body.allResources !== undefined + ? body.allResources + : existing.allResources; + + let nextResourceIds: number[] | undefined; + if (nextAllResources) { + nextResourceIds = []; + } else if (body.resourceIds !== undefined) { + nextResourceIds = body.resourceIds; + } + + if (nextResourceIds !== undefined) { + const resourceCheck = await assertManualKeyResourcesInOrg({ + allResources: nextAllResources, + resourceIds: nextResourceIds, + orgId: existing.orgId + }); + if (!resourceCheck.ok) { + return next( + createHttpError(HttpCode.BAD_REQUEST, resourceCheck.message) + ); + } + } + + const updates: Partial = {}; + + if (body.name !== undefined) { + updates.name = body.name; + } + if (body.description !== undefined) { + updates.description = body.description; + } + if (body.userId !== undefined) { + updates.userId = body.userId; + } + if (body.allResources !== undefined) { + updates.allResources = body.allResources; + } + if (body.validForSeconds !== undefined) { + updates.expiresAt = + body.validForSeconds === null + ? null + : createDate( + new TimeSpan(body.validForSeconds, "s") + ).getTime(); + } + + const updated = await db.transaction(async (trx) => { + let row = existing; + + if (Object.keys(updates).length > 0) { + const [updatedRow] = await trx + .update(virtualApiKeys) + .set(updates) + .where(eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId)) + .returning(); + row = updatedRow; + } + + if (nextResourceIds !== undefined) { + await replaceVirtualApiKeyResources( + trx, + virtualApiKeyId, + nextResourceIds + ); + } + + return row; + }); + + const resourceRows = await db + .select({ resourceId: virtualApiKeyResources.resourceId }) + .from(virtualApiKeyResources) + .where(eq(virtualApiKeyResources.virtualApiKeyId, virtualApiKeyId)); + + return response(res, { + data: { + virtualApiKey: { + ...toPublicVirtualApiKey(updated), + resourceIds: resourceRows.map((row) => row.resourceId) + } + }, + success: true, + error: false, + message: "Virtual API key updated successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/virtualApiKey/validation.ts b/server/routers/virtualApiKey/validation.ts new file mode 100644 index 000000000..a1c9ae671 --- /dev/null +++ b/server/routers/virtualApiKey/validation.ts @@ -0,0 +1,23 @@ +import { z } from "zod"; + +export const virtualApiKeyResourceIdsSchema = z + .array(z.coerce.number().int().positive()) + .optional(); + +export const createVirtualApiKeyBodySchema = z.strictObject({ + name: z.string().nonempty(), + description: z.string().optional().nullable(), + userId: z.string().optional().nullable(), + allResources: z.boolean().optional().default(false), + resourceIds: virtualApiKeyResourceIdsSchema, + validForSeconds: z.int().positive().optional() +}); + +export const updateVirtualApiKeyBodySchema = z.strictObject({ + name: z.string().nonempty().optional(), + description: z.string().optional().nullable(), + userId: z.string().optional().nullable(), + allResources: z.boolean().optional(), + resourceIds: virtualApiKeyResourceIdsSchema, + validForSeconds: z.int().positive().optional().nullable() +}); From 4c1f7f624376a65207ff92d8f88c3b7c29748501 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 10 Aug 2026 17:38:49 -0400 Subject: [PATCH 107/296] add caching to budget check --- server/lib/aiBudgetEnforcement.ts | 132 ++++++++++++++++++++++++++- server/lib/cache.ts | 2 +- server/routers/aiGateway/pipeline.ts | 31 ++++++- 3 files changed, 155 insertions(+), 10 deletions(-) diff --git a/server/lib/aiBudgetEnforcement.ts b/server/lib/aiBudgetEnforcement.ts index 8f70340fa..2a516695d 100644 --- a/server/lib/aiBudgetEnforcement.ts +++ b/server/lib/aiBudgetEnforcement.ts @@ -10,6 +10,7 @@ import { } from "@server/db"; import { modelKeyMatches } from "@server/lib/aiModelKeyMatch"; import type { AiUsage } from "@server/lib/aiUsageExtraction"; +import { regionalCache as cache } from "#dynamic/lib/cache"; import logger from "@server/logger"; type BudgetPeriod = AiBudget["period"]; @@ -22,6 +23,39 @@ const PERIOD_DURATIONS_MS: Record, number> = { yearly: 365 * 24 * 60 * 60 * 1000 }; +// Budgets are cheap to be a little stale about (enforcement is already +// check-then-act, not transactional). Re-derive each budget's usage sum +// from aiUsageRecords at most this often; in between, completed requests +// just add their own contribution onto the cached sum instead of +// re-querying/re-aggregating from scratch. +const BUDGET_CACHE_REFRESH_MS = 8_000; +// Redis-level TTL is only a safety net for eviction if a budget stops +// seeing traffic - the actual staleness check is the computedAt timestamp +// stored in the cached value, compared against BUDGET_CACHE_REFRESH_MS. +const BUDGET_CACHE_SAFETY_TTL_SEC = 60; + +function applicableBudgetsCacheKey(ctx: BudgetScopeContext): string { + const roleKey = [...ctx.roleIds].sort((a, b) => a - b).join(","); + return [ + "aiBudget:applicable", + ctx.orgId, + ctx.providerId, + ctx.requestedModel, + ctx.resourceId ?? "", + ctx.siteResourceId ?? "", + roleKey + ].join(":"); +} + +function budgetUsageCacheKey(budgetId: number): string { + return `aiBudget:usage:${budgetId}`; +} + +type CachedBudgetUsage = { + sum: number; + computedAt: number; +}; + // Budget periods are trailing windows from "now", not calendar-aligned // (e.g. "daily" = last 24h). "lifetime" has no lower bound. function windowStart(period: BudgetPeriod, now: number): number { @@ -45,10 +79,26 @@ export type BudgetScopeContext = { * Every budget that could apply to this request: the provider itself, any * model on that provider whose (possibly wildcarded) modelKey matches the * requested model, the target resource/site-resource, and any role the - * requesting user holds in the org. + * requesting user holds in the org. Cached for BUDGET_CACHE_REFRESH_MS since + * budget/model config changes are rare and a request-scoped org/provider/ + * model/resource/role combination repeats constantly under real traffic. */ export async function resolveApplicableBudgets( ctx: BudgetScopeContext +): Promise { + const cacheKey = applicableBudgetsCacheKey(ctx); + const cached = await cache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + + const budgets = await fetchApplicableBudgets(ctx); + await cache.set(cacheKey, budgets, BUDGET_CACHE_REFRESH_MS / 1000); + return budgets; +} + +async function fetchApplicableBudgets( + ctx: BudgetScopeContext ): Promise { const providerModels = await db .select({ modelId: aiModels.modelId, modelKey: aiModels.modelKey }) @@ -217,6 +267,74 @@ export async function sumUsageForBudget( return 0; } +/** + * Cached wrapper around sumUsageForBudget. Reuses a per-budget cached sum + * for up to BUDGET_CACHE_REFRESH_MS, and otherwise falls through to the DB + * aggregation and reseeds the cache. Completed requests within that window + * top the cached sum up via applyUsageToBudgetCache below rather than + * forcing a re-aggregation on every request. + */ +async function getBudgetUsage( + budget: AiBudget, + ctx: BudgetScopeContext, + now: number +): Promise { + const cacheKey = budgetUsageCacheKey(budget.budgetId); + const cached = await cache.get(cacheKey); + if (cached && now - cached.computedAt < BUDGET_CACHE_REFRESH_MS) { + return cached.sum; + } + + const sum = await sumUsageForBudget(budget, ctx, now); + await cache.set( + cacheKey, + { sum, computedAt: now } satisfies CachedBudgetUsage, + BUDGET_CACHE_SAFETY_TTL_SEC + ); + return sum; +} + +/** + * Called once a request's actual usage is known, for every budget that was + * resolved as applicable to it (i.e. checkBudgets' returned `budgets`). + * Adds this request's contribution directly onto each budget's cached sum + * so the next request in the same refresh window doesn't need to re-query + * or re-aggregate. If there's no warm cache entry, or it's already due for + * a refresh, this is a no-op - the next reader re-derives from the DB, + * which by then already includes this request's row via recordUsage. + */ +export async function applyUsageToBudgetCache( + budgets: AiBudget[], + usage: { usd: number; tokens: number } +): Promise { + await Promise.all( + budgets.map(async (budget) => { + const delta = budget.unit === "usd" ? usage.usd : usage.tokens; + if (!delta) { + return; + } + + const cacheKey = budgetUsageCacheKey(budget.budgetId); + const cached = await cache.get(cacheKey); + if ( + !cached || + Date.now() - cached.computedAt >= BUDGET_CACHE_REFRESH_MS + ) { + return; + } + + await cache.set( + cacheKey, + { + sum: cached.sum + delta, + computedAt: cached.computedAt + } satisfies CachedBudgetUsage, + BUDGET_CACHE_SAFETY_TTL_SEC + ); + }) + ); +} + // Throttled to one durable event per budget per breach window, so a soft // budget being exceeded doesn't write a row on every subsequent request // while it stays over. @@ -265,6 +383,10 @@ async function recordBreachEventIfNew( export type BudgetCheckResult = { blocked: boolean; blockingBudget?: AiBudget; + // Every budget resolved as applicable to this request, regardless of + // whether it was breached - pass to applyUsageToBudgetCache once this + // request's actual usage is known. + budgets: AiBudget[]; }; export async function checkBudgets( @@ -272,14 +394,14 @@ export async function checkBudgets( ): Promise { const budgets = await resolveApplicableBudgets(ctx); if (budgets.length === 0) { - return { blocked: false }; + return { blocked: false, budgets: [] }; } const now = Date.now(); let blockingBudget: AiBudget | undefined; for (const budget of budgets) { - const usage = await sumUsageForBudget(budget, ctx, now); + const usage = await getBudgetUsage(budget, ctx, now); if (usage < budget.amount) { continue; } @@ -292,8 +414,8 @@ export async function checkBudgets( } return blockingBudget - ? { blocked: true, blockingBudget } - : { blocked: false }; + ? { blocked: true, blockingBudget, budgets } + : { blocked: false, budgets }; } export type UsageRecordInput = { diff --git a/server/lib/cache.ts b/server/lib/cache.ts index 498d0dd3f..0113099e1 100644 --- a/server/lib/cache.ts +++ b/server/lib/cache.ts @@ -168,5 +168,5 @@ class AdaptiveCache { // Export singleton instance export const cache = new AdaptiveCache(); -export const regionalCache = cache; // Alias for compatability with the private version +export const regionalCache = cache; // Alias for compatibility with the private version export default cache; diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 802f5f453..22274e049 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -1,6 +1,7 @@ import { Request, Response } from "express"; import { and, eq, inArray } from "drizzle-orm"; import { + AiBudget, AiProvider, aiModels, aiProviders, @@ -51,7 +52,11 @@ import { } from "@server/lib/aiModelKeyMatch"; import { aiGatewayUpstreamFetch } from "@server/lib/aiGatewayUpstreamFetch"; import { getModelPricing, calculateAiCost } from "@server/lib/aiModelPricing"; -import { checkBudgets, recordUsage } from "@server/lib/aiBudgetEnforcement"; +import { + applyUsageToBudgetCache, + checkBudgets, + recordUsage +} from "@server/lib/aiBudgetEnforcement"; import { extractUsage, estimateUsage, @@ -533,6 +538,7 @@ function logAiUsageAndCost(args: { resourceId: number | null; siteResourceId: number | null; requestUserId: string | null; + budgets: AiBudget[]; }): void { const { capability, @@ -545,7 +551,8 @@ function logAiUsageAndCost(args: { orgId, resourceId, siteResourceId, - requestUserId + requestUserId, + budgets } = args; let usage: AiUsage | null = extractUsage( @@ -588,6 +595,18 @@ function logAiUsageAndCost(args: { usage, costUsd: cost?.totalCost ?? null }); + + if (budgets.length > 0) { + void applyUsageToBudgetCache(budgets, { + usd: cost?.totalCost ?? 0, + tokens: + usage.promptTokens + + usage.cacheReadTokens + + usage.cacheWriteTokens + + usage.completionTokens + + usage.reasoningTokens + }); + } } } @@ -666,6 +685,7 @@ export async function handleAiGatewayProxy( const { provider } = selection; + let appliedBudgets: AiBudget[] = []; if (orgId) { const budgetCheck = await checkBudgets({ orgId, @@ -676,6 +696,7 @@ export async function handleAiGatewayProxy( roleIds: requestUser?.roleIds ?? [], requestUserId: requestUser?.userId ?? null }); + appliedBudgets = budgetCheck.budgets; if (budgetCheck.blocked) { logger.warn("AI gateway request blocked by budget", { @@ -876,7 +897,8 @@ export async function handleAiGatewayProxy( orgId, resourceId, siteResourceId, - requestUserId: requestUser?.userId ?? null + requestUserId: requestUser?.userId ?? null, + budgets: appliedBudgets }); } return; @@ -895,7 +917,8 @@ export async function handleAiGatewayProxy( orgId, resourceId, siteResourceId, - requestUserId: requestUser?.userId ?? null + requestUserId: requestUser?.userId ?? null, + budgets: appliedBudgets }); return res.send(text); } catch (error) { From 6d45486bb5fe986e8b8d62b9c8e51f51bea67f1d Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Mon, 10 Aug 2026 21:44:22 -0400 Subject: [PATCH 108/296] add manual virtual api key create ui --- messages/en-US.json | 54 ++ server/lib/virtualApiKey.ts | 14 +- server/routers/resource/listResources.ts | 13 +- server/routers/virtualApiKey/validation.ts | 25 +- .../settings/virtual-api-keys/page.tsx | 153 ++++++ src/app/navigation.tsx | 10 + src/components/CreateVirtualApiKeyForm.tsx | 419 ++++++++++++++ src/components/EditVirtualApiKeyForm.tsx | 396 ++++++++++++++ src/components/PermissionsSelectBox.tsx | 8 + src/components/ViewVirtualApiKeySecret.tsx | 134 +++++ src/components/VirtualApiKeysDataTable.tsx | 41 ++ src/components/VirtualApiKeysTable.tsx | 516 ++++++++++++++++++ src/components/multi-resource-selector.tsx | 142 +++++ src/components/resource-selector.tsx | 19 +- src/components/user-selector.tsx | 38 +- src/lib/queries.ts | 10 +- 16 files changed, 1974 insertions(+), 18 deletions(-) create mode 100644 src/app/[orgId]/settings/virtual-api-keys/page.tsx create mode 100644 src/components/CreateVirtualApiKeyForm.tsx create mode 100644 src/components/EditVirtualApiKeyForm.tsx create mode 100644 src/components/ViewVirtualApiKeySecret.tsx create mode 100644 src/components/VirtualApiKeysDataTable.tsx create mode 100644 src/components/VirtualApiKeysTable.tsx create mode 100644 src/components/multi-resource-selector.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 4ff3f9da2..b614bfd38 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1436,6 +1436,11 @@ "actionGetAiModel": "Get AI Model", "actionListAiModels": "List AI Models", "actionUpdateAiModel": "Update AI Model", + "actionCreateVirtualApiKey": "Create Virtual API Key", + "actionDeleteVirtualApiKey": "Delete Virtual API Key", + "actionGetVirtualApiKey": "Get Virtual API Key", + "actionListVirtualApiKeys": "List Virtual API Keys", + "actionUpdateVirtualApiKey": "Update Virtual API Key", "actionApplyBlueprint": "Apply Blueprint", "actionListBlueprints": "List Blueprints", "actionGetBlueprint": "Get Blueprint", @@ -1637,6 +1642,53 @@ "sidebarAiGateway": "AI Gateway", "sidebarAiProviders": "Providers", "commandAiProviders": "AI Providers", + "sidebarVirtualApiKeys": "Virtual API Keys", + "commandVirtualApiKeys": "Virtual API Keys", + "virtualApiKeysTitle": "Manage Virtual API Keys", + "virtualApiKeysDescription": "Create and manage manual API keys for AI Gateway access to public inference resources", + "virtualApiKeys": "Virtual API Keys", + "virtualApiKeysSearch": "Search keys...", + "virtualApiKeysCreate": "Create Virtual API Key", + "virtualApiKeysCreateDescription": "Mint a manual key that can call public inference resources in this organization", + "virtualApiKeysCreateButton": "Create Key", + "virtualApiKeysEmpty": "No virtual API keys yet", + "virtualApiKeysName": "Name", + "virtualApiKeysDescriptionOptional": "Description (optional)", + "virtualApiKeysAssociateUserOptional": "Associate User (optional)", + "virtualApiKeysAssociateUserDescription": "Attribution only. Does not grant access by itself.", + "virtualApiKeysAllResources": "All public inference resources", + "virtualApiKeysAllResourcesDescription": "Allow this key to access every public inference resource in the organization", + "virtualApiKeysSelectResources": "Public Inference Resources", + "virtualApiKeysSelectResourcesPlaceholder": "Select resources", + "virtualApiKeysSelectResourcesDescription": "Choose which public inference resources this key can access", + "virtualApiKeysNoResources": "No resources", + "virtualApiKeysSecret": "Key", + "virtualApiKeysSeeOnce": "Copy this key now. You can also view it again later from the table.", + "virtualApiKeysSecretHint": "Use this value as a Bearer token: vk-[id].[secret]", + "virtualApiKeysViewSecret": "View Secret", + "virtualApiKeysViewSecretTitle": "Virtual API Key Secret", + "virtualApiKeysViewSecretDescription": "This secret grants access to the public inference resources assigned to this key", + "virtualApiKeysEdit": "Edit Virtual API Key", + "virtualApiKeysEditDescription": "Update the associated user and public inference resource access for this key", + "virtualApiKeysSaveButton": "Save Changes", + "virtualApiKeysSelectResourcesRequired": "Select at least one public inference resource, or enable all public inference resources", + "virtualApiKeysUpdated": "Virtual API key updated", + "virtualApiKeysUpdatedDescription": "The virtual API key has been updated", + "virtualApiKeysErrorUpdate": "Error updating virtual API key", + "virtualApiKeysErrorUpdateDescription": "Failed to update virtual API key", + "virtualApiKeysErrorCreate": "Error creating virtual API key", + "virtualApiKeysErrorCreateDescription": "Failed to create virtual API key", + "virtualApiKeysErrorDelete": "Error deleting virtual API key", + "virtualApiKeysErrorDeleteMessage": "Failed to delete virtual API key", + "virtualApiKeysDeleted": "Virtual API key deleted", + "virtualApiKeysDeletedDescription": "The virtual API key has been deleted", + "virtualApiKeysDelete": "Delete Virtual API Key", + "virtualApiKeysDeleteConfirm": "Delete Key", + "virtualApiKeysQuestionRemove": "Are you sure you want to delete this virtual API key?", + "virtualApiKeysMessageRemove": "Clients using this key will lose access immediately.", + "virtualApiKeysErrorFetchSecret": "Error loading secret", + "virtualApiKeysErrorFetchSecretDescription": "Failed to load the virtual API key secret", + "virtualApiKeysFilterUnassigned": "Unassigned", "aiProvidersTitle": "AI Providers", "aiProvidersDescription": "Connect model providers for AI workloads in this organization", "aiProvidersAdd": "Add Provider", @@ -2430,6 +2482,8 @@ "subnetPlaceholder": "Subnet", "addressDescription": "The internal address of the client. Must fall within the organization's subnet.", "selectSites": "Select sites", + "selectResources": "Select resources", + "multiResourcesSelectorResourcesCount": "{count, plural, one {# resource} other {# resources}}", "selectLabels": "Select labels", "sitesDescription": "The client will have connectivity to the selected sites", "clientInstallOlm": "Install Machine Client", diff --git a/server/lib/virtualApiKey.ts b/server/lib/virtualApiKey.ts index 15f9b52ff..66ebb4b5a 100644 --- a/server/lib/virtualApiKey.ts +++ b/server/lib/virtualApiKey.ts @@ -62,10 +62,18 @@ export async function assertManualKeyResourcesInOrg(params: { }): Promise<{ ok: true } | { ok: false; message: string }> { const { allResources, resourceIds, orgId } = params; - if (allResources || resourceIds.length === 0) { + if (allResources) { return { ok: true }; } + if (resourceIds.length === 0) { + return { + ok: false, + message: + "Select at least one public inference resource, or enable all public inference resources" + }; + } + const uniqueIds = [...new Set(resourceIds)]; const rows = await db .select({ resourceId: resources.resourceId }) @@ -73,6 +81,7 @@ export async function assertManualKeyResourcesInOrg(params: { .where( and( eq(resources.orgId, orgId), + eq(resources.mode, "inference"), inArray(resources.resourceId, uniqueIds) ) ); @@ -80,7 +89,8 @@ export async function assertManualKeyResourcesInOrg(params: { if (rows.length !== uniqueIds.length) { return { ok: false, - message: "One or more resources are invalid for this organization" + message: + "One or more resources are invalid public inference resources for this organization" }; } diff --git a/server/routers/resource/listResources.ts b/server/routers/resource/listResources.ts index e5408f630..124c480c2 100644 --- a/server/routers/resource/listResources.ts +++ b/server/routers/resource/listResources.ts @@ -124,12 +124,21 @@ const listResourcesSchema = z.strictObject({ "Filter resources based on health status of their targets. `healthy` means all targets are healthy. `degraded` means at least one target is unhealthy, but not all are unhealthy. `offline` means all targets are unhealthy. `unknown` means all targets have unknown health status." }), protocol: z - .enum(["http", "https", "tcp", "udp", "ssh", "rdp", "vnc"]) + .enum(["http", "https", "tcp", "udp", "ssh", "rdp", "vnc", "inference"]) .optional() .catch(undefined) .openapi({ type: "string", - enum: ["http", "https", "tcp", "udp", "ssh", "rdp", "vnc"], + enum: [ + "http", + "https", + "tcp", + "udp", + "ssh", + "rdp", + "vnc", + "inference" + ], description: "Filter resources by protocol. `http` and `https` match HTTP resources without and with SSL respectively." }), diff --git a/server/routers/virtualApiKey/validation.ts b/server/routers/virtualApiKey/validation.ts index a1c9ae671..42964b3ba 100644 --- a/server/routers/virtualApiKey/validation.ts +++ b/server/routers/virtualApiKey/validation.ts @@ -4,14 +4,23 @@ export const virtualApiKeyResourceIdsSchema = z .array(z.coerce.number().int().positive()) .optional(); -export const createVirtualApiKeyBodySchema = z.strictObject({ - name: z.string().nonempty(), - description: z.string().optional().nullable(), - userId: z.string().optional().nullable(), - allResources: z.boolean().optional().default(false), - resourceIds: virtualApiKeyResourceIdsSchema, - validForSeconds: z.int().positive().optional() -}); +export const createVirtualApiKeyBodySchema = z + .strictObject({ + name: z.string().nonempty(), + description: z.string().optional().nullable(), + userId: z.string().optional().nullable(), + allResources: z.boolean().optional().default(false), + resourceIds: virtualApiKeyResourceIdsSchema, + validForSeconds: z.int().positive().optional() + }) + .refine( + (data) => data.allResources || (data.resourceIds?.length ?? 0) > 0, + { + message: + "Select at least one public inference resource, or enable all public inference resources", + path: ["resourceIds"] + } + ); export const updateVirtualApiKeyBodySchema = z.strictObject({ name: z.string().nonempty().optional(), diff --git a/src/app/[orgId]/settings/virtual-api-keys/page.tsx b/src/app/[orgId]/settings/virtual-api-keys/page.tsx new file mode 100644 index 000000000..ba8f4ea01 --- /dev/null +++ b/src/app/[orgId]/settings/virtual-api-keys/page.tsx @@ -0,0 +1,153 @@ +import { internal } from "@app/lib/api"; +import { authCookieHeader } from "@app/lib/api/cookies"; +import { AxiosResponse } from "axios"; +import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; +import { redirect } from "next/navigation"; +import { cache } from "react"; +import { GetOrgResponse } from "@server/routers/org"; +import OrgProvider from "@app/providers/OrgProvider"; +import VirtualApiKeysTable, { + type VirtualApiKeyRow +} from "@app/components/VirtualApiKeysTable"; +import { getTranslations } from "next-intl/server"; +import type { Metadata } from "next"; +import type { ListVirtualApiKeysResponse } from "@server/routers/virtualApiKey/types"; +import type { ListUsersResponse } from "@server/routers/user"; +import type { ListResourcesResponse } from "@server/routers/resource"; + +export const metadata: Metadata = { + title: "Virtual API Keys" +}; + +type VirtualApiKeysPageProps = { + params: Promise<{ orgId: string }>; +}; + +export const dynamic = "force-dynamic"; + +export default async function VirtualApiKeysPage( + props: VirtualApiKeysPageProps +) { + const params = await props.params; + const cookieHeader = await authCookieHeader(); + const t = await getTranslations(); + + let keys: ListVirtualApiKeysResponse["virtualApiKeys"] = []; + let users: { + userId: string; + email: string | null; + name: string | null; + username: string | null; + }[] = []; + let resources: { + resourceId: number; + name: string; + niceId: string; + }[] = []; + + try { + const [keysRes, usersRes, resourcesRes] = await Promise.all([ + internal.get>( + `/org/${params.orgId}/virtual-api-keys?page=1&pageSize=1000`, + cookieHeader + ), + internal.get>( + `/org/${params.orgId}/users?page=1&pageSize=1000`, + cookieHeader + ), + internal.get>( + `/org/${params.orgId}/resources?page=1&pageSize=1000`, + cookieHeader + ) + ]); + + keys = keysRes.data.data.virtualApiKeys ?? []; + users = (usersRes.data.data.users ?? []).map((u) => ({ + userId: u.id, + email: u.email ?? null, + name: u.name ?? null, + username: u.username ?? null + })); + resources = (resourcesRes.data.data.resources ?? []).map((r) => ({ + resourceId: r.resourceId, + name: r.name, + niceId: r.niceId + })); + } catch { + // leave empty; page still renders + } + + let org = null; + try { + const getOrg = cache(async () => + internal.get>( + `/org/${params.orgId}`, + cookieHeader + ) + ); + const res = await getOrg(); + org = res.data.data; + } catch { + redirect(`/${params.orgId}/settings/resources`); + } + + if (!org) { + redirect(`/${params.orgId}/settings/resources`); + } + + const userById = new Map(users.map((u) => [u.userId, u])); + const resourceById = new Map(resources.map((r) => [r.resourceId, r])); + + const rows: VirtualApiKeyRow[] = keys.map((key) => { + const user = key.userId ? userById.get(key.userId) : undefined; + const keyResources = key.resourceIds + .map((id) => resourceById.get(id)) + .filter(Boolean) as { + resourceId: number; + name: string; + niceId: string; + }[]; + + const resourceNames = key.allResources + ? t("virtualApiKeysAllResources") + : keyResources.map((r) => r.name).join(", ") || + t("virtualApiKeysNoResources"); + + return { + virtualApiKeyId: key.virtualApiKeyId, + orgId: key.orgId, + kind: key.kind, + userId: key.userId, + name: key.name, + description: key.description, + lastChars: key.lastChars, + allResources: key.allResources, + expiresAt: key.expiresAt, + lastUsedAt: key.lastUsedAt, + createdAt: key.createdAt, + createdByUserId: key.createdByUserId, + resourceIds: key.resourceIds, + userName: user?.name ?? null, + username: user?.username ?? null, + userEmail: user?.email ?? null, + resourceNames, + resources: keyResources + }; + }); + + return ( + <> + + + + + + + ); +} diff --git a/src/app/navigation.tsx b/src/app/navigation.tsx index e6971bc94..20af6d05d 100644 --- a/src/app/navigation.tsx +++ b/src/app/navigation.tsx @@ -195,6 +195,11 @@ export const orgNavSections = ( title: "sidebarAiProviders", href: "/{orgId}/settings/ai-providers", icon: + }, + { + title: "sidebarVirtualApiKeys", + href: "/{orgId}/settings/virtual-api-keys", + icon: } ] }, @@ -490,6 +495,11 @@ export const commandBarNavSections = ( title: "commandAiProviders", href: "/{orgId}/settings/ai-providers", icon: + }, + { + title: "commandVirtualApiKeys", + href: "/{orgId}/settings/virtual-api-keys", + icon: } ] }, diff --git a/src/components/CreateVirtualApiKeyForm.tsx b/src/components/CreateVirtualApiKeyForm.tsx new file mode 100644 index 000000000..32a3877aa --- /dev/null +++ b/src/components/CreateVirtualApiKeyForm.tsx @@ -0,0 +1,419 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { Input } from "@app/components/ui/input"; +import { toast } from "@app/hooks/useToast"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { AxiosResponse } from "axios"; +import { useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; +import CopyTextBox from "@app/components/CopyTextBox"; +import { + Credenza, + CredenzaBody, + CredenzaClose, + CredenzaContent, + CredenzaDescription, + CredenzaFooter, + CredenzaHeader, + CredenzaTitle +} from "@app/components/Credenza"; +import { useOrgContext } from "@app/hooks/useOrgContext"; +import { formatAxiosError, createApiClient } from "@app/lib/api"; +import { cn } from "@app/lib/cn"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { + Popover, + PopoverContent, + PopoverTrigger +} from "@app/components/ui/popover"; +import { CaretSortIcon } from "@radix-ui/react-icons"; +import { Checkbox } from "@app/components/ui/checkbox"; +import { useTranslations } from "next-intl"; +import { UserSelector, type SelectedUser } from "@app/components/user-selector"; +import type { CreateOrEditVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; +import { + MultiResourcesSelector, + formatMultiResourcesSelectorLabel +} from "@app/components/multi-resource-selector"; +import type { SelectedResource } from "@app/components/resource-selector"; + +export type CreatedVirtualApiKey = { + virtualApiKeyId: string; + orgId: string; + kind: "manual" | "user"; + userId: string | null; + name: string | null; + description: string | null; + lastChars: string; + allResources: boolean; + expiresAt: number | null; + lastUsedAt: number | null; + createdAt: number; + createdByUserId: string | null; + resourceIds: number[]; + userName?: string | null; + username?: string | null; + userEmail?: string | null; + resourceNames: string; + resources: { resourceId: number; name: string; niceId: string }[]; +}; + +type FormProps = { + open: boolean; + setOpen: (open: boolean) => void; + onCreated?: (result: CreatedVirtualApiKey) => void; +}; + +export default function CreateVirtualApiKeyForm({ + open, + setOpen, + onCreated +}: FormProps) { + const { org } = useOrgContext(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const t = useTranslations(); + + const [credential, setCredential] = useState(null); + const [loading, setLoading] = useState(false); + const [allResources, setAllResources] = useState(false); + const [selectedUser, setSelectedUser] = useState(null); + const [selectedResources, setSelectedResources] = useState< + SelectedResource[] + >([]); + + const formSchema = z.object({ + name: z.string().min(1), + description: z.string().optional() + }); + + const form = useForm>({ + resolver: zodResolver(formSchema), + defaultValues: { + name: "", + description: "" + } + }); + + function resetLocalState() { + setCredential(null); + setLoading(false); + setAllResources(false); + setSelectedUser(null); + setSelectedResources([]); + form.reset(); + } + + async function onSubmit(values: z.infer) { + setLoading(true); + + const res = await api + .put>( + `/org/${org.org.orgId}/virtual-api-key`, + { + name: values.name, + description: values.description || null, + userId: selectedUser?.id ?? null, + allResources, + resourceIds: allResources + ? [] + : selectedResources.map((r) => r.resourceId) + } + ) + .catch((e) => { + console.error(e); + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorCreate"), + description: formatAxiosError( + e, + t("virtualApiKeysErrorCreateDescription") + ) + }); + }); + + if (res?.data.data.virtualApiKey) { + const key = res.data.data.virtualApiKey; + if (key.secret) { + setCredential(`vk-${key.virtualApiKeyId}.${key.secret}`); + } + + const resourceLookup = new Map( + selectedResources.map((r) => [ + r.resourceId, + { name: r.name, niceId: r.niceId } + ]) + ); + const resourceNames = key.allResources + ? t("virtualApiKeysAllResources") + : key.resourceIds + .map((id) => resourceLookup.get(id)?.name) + .filter(Boolean) + .join(", ") || t("virtualApiKeysNoResources"); + + onCreated?.({ + virtualApiKeyId: key.virtualApiKeyId, + orgId: key.orgId, + kind: key.kind, + userId: key.userId, + name: key.name, + description: key.description, + lastChars: key.lastChars, + allResources: key.allResources, + expiresAt: key.expiresAt, + lastUsedAt: key.lastUsedAt, + createdAt: key.createdAt, + createdByUserId: key.createdByUserId, + resourceIds: key.resourceIds, + userName: selectedUser?.text ?? null, + username: null, + userEmail: null, + resourceNames, + resources: key.resourceIds.map((id) => ({ + resourceId: id, + name: resourceLookup.get(id)?.name ?? String(id), + niceId: resourceLookup.get(id)?.niceId ?? "" + })) + }); + } + + setLoading(false); + } + + return ( + { + setOpen(val); + if (!val) { + resetLocalState(); + } + }} + > + + + {t("virtualApiKeysCreate")} + + {t("virtualApiKeysCreateDescription")} + + + +
+ {!credential && ( +
+ + ( + + + {t("virtualApiKeysName")} + + + + + + + )} + /> + + ( + + + {t( + "virtualApiKeysDescriptionOptional" + )} + + + + + + + )} + /> + +
+ + {t( + "virtualApiKeysAssociateUserOptional" + )} + + + + + + + + + +

+ {t( + "virtualApiKeysAssociateUserDescription" + )} +

+
+ +
+
+ { + setAllResources( + val as boolean + ); + if (val) { + setSelectedResources( + [] + ); + } + }} + className="mt-0.5" + /> +
+ +

+ {t( + "virtualApiKeysAllResourcesDescription" + )} +

+
+
+ + {!allResources && ( +
+ + {t( + "virtualApiKeysSelectResources" + )} + + + + + + + + 0 + } + onClear={() => + setSelectedResources( + [] + ) + } + /> + + + + {t( + "virtualApiKeysSelectResourcesDescription" + )} + +
+ )} +
+ + + )} + {credential && ( +
+

{t("virtualApiKeysSeeOnce")}

+ +
+ )} +
+ + + + + + + + + + ); +} diff --git a/src/components/EditVirtualApiKeyForm.tsx b/src/components/EditVirtualApiKeyForm.tsx new file mode 100644 index 000000000..22a479df3 --- /dev/null +++ b/src/components/EditVirtualApiKeyForm.tsx @@ -0,0 +1,396 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { toast } from "@app/hooks/useToast"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { AxiosResponse } from "axios"; +import { useEffect, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; +import { + Credenza, + CredenzaBody, + CredenzaClose, + CredenzaContent, + CredenzaDescription, + CredenzaFooter, + CredenzaHeader, + CredenzaTitle +} from "@app/components/Credenza"; +import { useOrgContext } from "@app/hooks/useOrgContext"; +import { formatAxiosError, createApiClient } from "@app/lib/api"; +import { cn } from "@app/lib/cn"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { + Popover, + PopoverContent, + PopoverTrigger +} from "@app/components/ui/popover"; +import { CaretSortIcon } from "@radix-ui/react-icons"; +import { Checkbox } from "@app/components/ui/checkbox"; +import { useTranslations } from "next-intl"; +import { UserSelector, type SelectedUser } from "@app/components/user-selector"; +import type { CreateOrEditVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; +import { + MultiResourcesSelector, + formatMultiResourcesSelectorLabel +} from "@app/components/multi-resource-selector"; +import type { SelectedResource } from "@app/components/resource-selector"; +import { getUserDisplayName } from "@app/lib/getUserDisplayName"; +import type { CreatedVirtualApiKey } from "@app/components/CreateVirtualApiKeyForm"; + +type FormProps = { + open: boolean; + setOpen: (open: boolean) => void; + virtualApiKey: CreatedVirtualApiKey | null; + onUpdated?: (result: CreatedVirtualApiKey) => void; +}; + +function resourcesFromRow(key: CreatedVirtualApiKey): SelectedResource[] { + return key.resources.map((r) => ({ + resourceId: r.resourceId, + name: r.name, + niceId: r.niceId, + fullDomain: null, + ssl: false, + wildcard: false + })); +} + +function userFromRow(key: CreatedVirtualApiKey): SelectedUser | null { + if (!key.userId) { + return null; + } + return { + id: key.userId, + text: getUserDisplayName({ + email: key.userEmail, + name: key.userName, + username: key.username + }) + }; +} + +export default function EditVirtualApiKeyForm({ + open, + setOpen, + virtualApiKey, + onUpdated +}: FormProps) { + const { org } = useOrgContext(); + const { env } = useEnvContext(); + const api = createApiClient({ env }); + const t = useTranslations(); + + const [loading, setLoading] = useState(false); + const [selectedUser, setSelectedUser] = useState(null); + const [selectedResources, setSelectedResources] = useState< + SelectedResource[] + >([]); + + const formSchema = z + .object({ + allResources: z.boolean() + }) + .superRefine((data, ctx) => { + if (!data.allResources && selectedResources.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: t("virtualApiKeysSelectResourcesRequired"), + path: ["allResources"] + }); + } + }); + + const form = useForm>({ + resolver: zodResolver(formSchema), + defaultValues: { + allResources: false + } + }); + + const allResources = form.watch("allResources"); + + useEffect(() => { + if (!open || !virtualApiKey) { + return; + } + setLoading(false); + setSelectedUser(userFromRow(virtualApiKey)); + setSelectedResources( + virtualApiKey.allResources ? [] : resourcesFromRow(virtualApiKey) + ); + form.reset({ + allResources: virtualApiKey.allResources + }); + }, [open, virtualApiKey, form]); + + async function onSubmit(values: z.infer) { + if (!virtualApiKey) { + return; + } + + setLoading(true); + + const res = await api + .post>( + `/virtual-api-key/${virtualApiKey.virtualApiKeyId}`, + { + userId: selectedUser?.id ?? null, + allResources: values.allResources, + resourceIds: values.allResources + ? [] + : selectedResources.map((r) => r.resourceId) + } + ) + .catch((e) => { + console.error(e); + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorUpdate"), + description: formatAxiosError( + e, + t("virtualApiKeysErrorUpdateDescription") + ) + }); + }); + + if (res?.data.data.virtualApiKey) { + const key = res.data.data.virtualApiKey; + const resourceLookup = new Map( + selectedResources.map((r) => [ + r.resourceId, + { name: r.name, niceId: r.niceId } + ]) + ); + const resourceNames = key.allResources + ? t("virtualApiKeysAllResources") + : key.resourceIds + .map((id) => resourceLookup.get(id)?.name) + .filter(Boolean) + .join(", ") || t("virtualApiKeysNoResources"); + + onUpdated?.({ + ...virtualApiKey, + userId: key.userId, + allResources: key.allResources, + resourceIds: key.resourceIds, + userName: selectedUser?.text ?? null, + username: null, + userEmail: null, + resourceNames, + resources: key.resourceIds.map((id) => ({ + resourceId: id, + name: resourceLookup.get(id)?.name ?? String(id), + niceId: resourceLookup.get(id)?.niceId ?? "" + })) + }); + + toast({ + title: t("virtualApiKeysUpdated"), + description: t("virtualApiKeysUpdatedDescription") + }); + setOpen(false); + } + + setLoading(false); + } + + return ( + { + setOpen(val); + }} + > + + + {t("virtualApiKeysEdit")} + + {t("virtualApiKeysEditDescription")} + + + +
+
+ +
+ + {t( + "virtualApiKeysAssociateUserOptional" + )} + + + + + + + + + +

+ {t( + "virtualApiKeysAssociateUserDescription" + )} +

+
+ +
+ ( + +
+ + { + field.onChange( + val as boolean + ); + if (val) { + setSelectedResources( + [] + ); + } + }} + className="mt-0.5" + /> + +
+ +

+ {t( + "virtualApiKeysAllResourcesDescription" + )} +

+
+
+ + + )} + /> + + {!allResources && ( +
+ + {t( + "virtualApiKeysSelectResources" + )} + + + + + + + + 0 + } + onClear={() => + setSelectedResources( + [] + ) + } + /> + + + + {t( + "virtualApiKeysSelectResourcesRequired" + )} + +
+ )} +
+
+ +
+ + + + + + + + + + ); +} diff --git a/src/components/PermissionsSelectBox.tsx b/src/components/PermissionsSelectBox.tsx index fe33337d4..ef78d4c71 100644 --- a/src/components/PermissionsSelectBox.tsx +++ b/src/components/PermissionsSelectBox.tsx @@ -166,6 +166,14 @@ function getActionsCategories(root: boolean) { [t("actionGetAiModel")]: "getAiModel", [t("actionListAiModels")]: "listAiModels", [t("actionUpdateAiModel")]: "updateAiModel" + }, + + "Virtual API Key": { + [t("actionCreateVirtualApiKey")]: "createVirtualApiKey", + [t("actionDeleteVirtualApiKey")]: "deleteVirtualApiKey", + [t("actionGetVirtualApiKey")]: "getVirtualApiKey", + [t("actionListVirtualApiKeys")]: "listVirtualApiKeys", + [t("actionUpdateVirtualApiKey")]: "updateVirtualApiKey" } }; diff --git a/src/components/ViewVirtualApiKeySecret.tsx b/src/components/ViewVirtualApiKeySecret.tsx new file mode 100644 index 000000000..800fb3484 --- /dev/null +++ b/src/components/ViewVirtualApiKeySecret.tsx @@ -0,0 +1,134 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useTranslations } from "next-intl"; +import { AxiosResponse } from "axios"; +import { + Credenza, + CredenzaBody, + CredenzaClose, + CredenzaContent, + CredenzaDescription, + CredenzaFooter, + CredenzaHeader, + CredenzaTitle +} from "@app/components/Credenza"; +import { Button } from "@app/components/ui/button"; +import CopyTextBox from "@app/components/CopyTextBox"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import type { GetVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; + +type ViewVirtualApiKeySecretProps = { + open: boolean; + setOpen: (open: boolean) => void; + virtualApiKeyId: string | null; + name?: string | null; +}; + +export default function ViewVirtualApiKeySecret({ + open, + setOpen, + virtualApiKeyId, + name +}: ViewVirtualApiKeySecretProps) { + const t = useTranslations(); + const api = createApiClient(useEnvContext()); + const [loading, setLoading] = useState(false); + const [credential, setCredential] = useState(null); + + useEffect(() => { + if (!open || !virtualApiKeyId) { + return; + } + + let cancelled = false; + setLoading(true); + setCredential(null); + + api.get>( + `/virtual-api-key/${virtualApiKeyId}` + ) + .then((res) => { + if (cancelled) { + return; + } + const key = res.data.data.virtualApiKey; + if (key.secret) { + setCredential(`vk-${key.virtualApiKeyId}.${key.secret}`); + } else { + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorFetchSecret"), + description: t( + "virtualApiKeysErrorFetchSecretDescription" + ) + }); + } + }) + .catch((e) => { + if (cancelled) { + return; + } + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorFetchSecret"), + description: formatAxiosError( + e, + t("virtualApiKeysErrorFetchSecretDescription") + ) + }); + }) + .finally(() => { + if (!cancelled) { + setLoading(false); + } + }); + + return () => { + cancelled = true; + }; + }, [open, virtualApiKeyId]); + + return ( + { + setOpen(val); + if (!val) { + setCredential(null); + setLoading(false); + } + }} + > + + + + {t("virtualApiKeysViewSecretTitle")} + + + {name ? name : t("virtualApiKeysViewSecretDescription")} + + + +
+ {loading && ( +

+ {t("loading")} +

+ )} + {!loading && credential && ( + + )} +
+ + + + + + + + + ); +} diff --git a/src/components/VirtualApiKeysDataTable.tsx b/src/components/VirtualApiKeysDataTable.tsx new file mode 100644 index 000000000..d691102eb --- /dev/null +++ b/src/components/VirtualApiKeysDataTable.tsx @@ -0,0 +1,41 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { DataTable } from "@app/components/ui/data-table"; +import { useTranslations } from "next-intl"; + +type DataTableProps = { + columns: ColumnDef[]; + data: TData[]; + createVirtualApiKey?: () => void; + onRefresh?: () => void; + isRefreshing?: boolean; +}; + +export function VirtualApiKeysDataTable({ + columns, + data, + createVirtualApiKey, + onRefresh, + isRefreshing +}: DataTableProps) { + const t = useTranslations(); + + return ( + + ); +} diff --git a/src/components/VirtualApiKeysTable.tsx b/src/components/VirtualApiKeysTable.tsx new file mode 100644 index 000000000..34b670328 --- /dev/null +++ b/src/components/VirtualApiKeysTable.tsx @@ -0,0 +1,516 @@ +"use client"; + +import { ExtendedColumnDef } from "@app/components/ui/data-table"; +import { VirtualApiKeysDataTable } from "@app/components/VirtualApiKeysDataTable"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger +} from "@app/components/ui/dropdown-menu"; +import { Button } from "@app/components/ui/button"; +import { Badge } from "@app/components/ui/badge"; +import { + Popover, + PopoverContent, + PopoverTrigger +} from "@app/components/ui/popover"; +import { + ArrowRight, + ArrowUpDown, + ArrowUpRight, + Funnel, + MoreHorizontal +} from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useEffect, useMemo, useState } from "react"; +import ConfirmDeleteDialog from "@app/components/ConfirmDeleteDialog"; +import { formatAxiosError, createApiClient } from "@app/lib/api"; +import { toast } from "@app/hooks/useToast"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import moment from "moment"; +import CreateVirtualApiKeyForm, { + type CreatedVirtualApiKey +} from "@app/components/CreateVirtualApiKeyForm"; +import EditVirtualApiKeyForm from "@app/components/EditVirtualApiKeyForm"; +import ViewVirtualApiKeySecret from "@app/components/ViewVirtualApiKeySecret"; +import { useTranslations } from "next-intl"; +import { getUserDisplayName } from "@app/lib/getUserDisplayName"; +import { UserSelector, type SelectedUser } from "@app/components/user-selector"; +import { + ResourceSelector, + type SelectedResource +} from "@app/components/resource-selector"; +import { cn } from "@app/lib/cn"; +import { dataTableFilterPopoverContentClassName } from "@app/lib/dataTableFilterPopover"; + +export type VirtualApiKeyRow = CreatedVirtualApiKey; + +type VirtualApiKeysTableProps = { + virtualApiKeys: VirtualApiKeyRow[]; + orgId: string; +}; + +export default function VirtualApiKeysTable({ + virtualApiKeys, + orgId +}: VirtualApiKeysTableProps) { + const router = useRouter(); + const t = useTranslations(); + const api = createApiClient(useEnvContext()); + + const [isCreateModalOpen, setIsCreateModalOpen] = useState(false); + const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const [isViewSecretOpen, setIsViewSecretOpen] = useState(false); + const [isEditModalOpen, setIsEditModalOpen] = useState(false); + const [selectedKey, setSelectedKey] = useState( + null + ); + const [rows, setRows] = useState(virtualApiKeys); + const [isRefreshing, setIsRefreshing] = useState(false); + + const [userFilterOpen, setUserFilterOpen] = useState(false); + const [resourceFilterOpen, setResourceFilterOpen] = useState(false); + const [selectedUser, setSelectedUser] = useState(null); + const [selectedResource, setSelectedResource] = + useState(null); + const [unassignedOnly, setUnassignedOnly] = useState(false); + + useEffect(() => { + setRows(virtualApiKeys); + }, [virtualApiKeys]); + + const filteredRows = useMemo(() => { + return rows.filter((row) => { + if (unassignedOnly && row.userId) { + return false; + } + if (selectedUser && row.userId !== selectedUser.id) { + return false; + } + if (selectedResource) { + if ( + !row.allResources && + !row.resourceIds.includes(selectedResource.resourceId) + ) { + return false; + } + } + return true; + }); + }, [rows, selectedUser, selectedResource, unassignedOnly]); + + const refreshData = async () => { + setIsRefreshing(true); + try { + await new Promise((resolve) => setTimeout(resolve, 200)); + router.refresh(); + } catch { + toast({ + title: t("error"), + description: t("refreshError"), + variant: "destructive" + }); + } finally { + setIsRefreshing(false); + } + }; + + async function deleteKey(id: string) { + await api.delete(`/virtual-api-key/${id}`).catch((e) => { + toast({ + title: t("virtualApiKeysErrorDelete"), + description: formatAxiosError( + e, + t("virtualApiKeysErrorDeleteMessage") + ) + }); + throw e; + }); + + setRows((prev) => prev.filter((r) => r.virtualApiKeyId !== id)); + + toast({ + title: t("virtualApiKeysDeleted"), + description: t("virtualApiKeysDeletedDescription") + }); + } + + const clearUserFilter = () => { + setSelectedUser(null); + setUnassignedOnly(false); + setUserFilterOpen(false); + }; + + const clearResourceFilter = () => { + setSelectedResource(null); + setResourceFilterOpen(false); + }; + + const columns: ExtendedColumnDef[] = [ + { + accessorKey: "name", + enableHiding: false, + friendlyName: t("virtualApiKeysName"), + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => row.original.name || "-" + }, + { + id: "resources", + accessorFn: (row) => row.resourceNames, + friendlyName: t("resource"), + header: () => ( + + + + + + { + setSelectedResource(resource); + setResourceFilterOpen(false); + }} + /> + + + ), + cell: ({ row }) => { + const r = row.original; + if (r.allResources) { + return t("virtualApiKeysAllResources"); + } + if (r.resources.length === 0) { + return {t("virtualApiKeysNoResources")}; + } + if (r.resources.length === 1) { + const resource = r.resources[0]; + if (!resource.niceId) { + return resource.name; + } + return ( + + + + ); + } + return r.resourceNames; + } + }, + { + accessorKey: "userId", + friendlyName: t("user"), + header: () => ( + + + + + + { + setSelectedUser(null); + setUnassignedOnly(true); + setUserFilterOpen(false); + } + }} + onSelectUser={(user) => { + setSelectedUser(user); + setUnassignedOnly(false); + setUserFilterOpen(false); + }} + /> + + + ), + cell: ({ row }) => { + const r = row.original; + if (!r.userId) { + return -; + } + return ( + + + + ); + } + }, + { + accessorKey: "lastChars", + friendlyName: t("virtualApiKeysSecret"), + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => + `vk-${row.original.virtualApiKeyId}.••••${row.original.lastChars}` + }, + { + accessorKey: "createdAt", + friendlyName: t("created"), + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => moment(row.original.createdAt).format("lll") + }, + { + accessorKey: "expiresAt", + friendlyName: t("expires"), + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => { + const expiresAt = row.original.expiresAt; + if (expiresAt) { + return moment(expiresAt).format("lll"); + } + return t("never"); + } + }, + { + id: "actions", + enableHiding: false, + header: () => , + cell: ({ row }) => { + const keyRow = row.original; + return ( +
+ + + + + + { + setSelectedKey(keyRow); + setIsViewSecretOpen(true); + }} + > + {t("virtualApiKeysViewSecret")} + + { + setSelectedKey(keyRow); + setIsDeleteModalOpen(true); + }} + > + + {t("delete")} + + + + + +
+ ); + } + } + ]; + + return ( + <> + {selectedKey && ( + { + setIsDeleteModalOpen(val); + if (!val) setSelectedKey(null); + }} + dialog={ +
+

{t("virtualApiKeysQuestionRemove")}

+

{t("virtualApiKeysMessageRemove")}

+
+ } + buttonText={t("virtualApiKeysDeleteConfirm")} + onConfirm={async () => + deleteKey(selectedKey.virtualApiKeyId) + } + string={selectedKey.name || selectedKey.virtualApiKeyId} + title={t("virtualApiKeysDelete")} + /> + )} + + { + setIsViewSecretOpen(val); + if (!val) setSelectedKey(null); + }} + virtualApiKeyId={selectedKey?.virtualApiKeyId ?? null} + name={selectedKey?.name} + /> + + { + setRows([val, ...rows]); + }} + /> + + { + setIsEditModalOpen(val); + if (!val) setSelectedKey(null); + }} + virtualApiKey={selectedKey} + onUpdated={(val) => { + setRows((prev) => + prev.map((row) => + row.virtualApiKeyId === val.virtualApiKeyId + ? val + : row + ) + ); + }} + /> + + { + setIsCreateModalOpen(true); + }} + onRefresh={refreshData} + isRefreshing={isRefreshing} + /> + + ); +} diff --git a/src/components/multi-resource-selector.tsx b/src/components/multi-resource-selector.tsx new file mode 100644 index 000000000..dc2685fef --- /dev/null +++ b/src/components/multi-resource-selector.tsx @@ -0,0 +1,142 @@ +import { orgQueries } from "@app/lib/queries"; +import { useQuery } from "@tanstack/react-query"; +import { useMemo, useState } from "react"; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList +} from "./ui/command"; +import { Checkbox } from "./ui/checkbox"; +import { useTranslations } from "next-intl"; +import { useDebounce } from "use-debounce"; +import { type SelectedResource } from "./resource-selector"; + +export type MultiResourcesSelectorProps = { + orgId: string; + selectedResources: SelectedResource[]; + onSelectionChange: (resources: SelectedResource[]) => void; + excludeWildcard?: boolean; + onClear?: () => void; + showClear?: boolean; + protocol?: string; +}; + +export function formatMultiResourcesSelectorLabel( + selectedResources: SelectedResource[], + t: (key: string, values?: { count: number }) => string, + emptyLabelKey = "selectResources" +): string { + if (selectedResources.length === 0) { + return t(emptyLabelKey); + } + if (selectedResources.length === 1) { + return selectedResources[0]!.name; + } + return t("multiResourcesSelectorResourcesCount", { + count: selectedResources.length + }); +} + +export function MultiResourcesSelector({ + orgId, + selectedResources, + onSelectionChange, + excludeWildcard = false, + onClear, + showClear = false, + protocol +}: MultiResourcesSelectorProps) { + const t = useTranslations(); + const [resourceSearchQuery, setResourceSearchQuery] = useState(""); + const [debouncedQuery] = useDebounce(resourceSearchQuery, 150); + + const { data: resources = [] } = useQuery( + orgQueries.proxyResources({ + orgId, + query: debouncedQuery, + perPage: 10, + protocol + }) + ); + + const resourcesShown = useMemo(() => { + const base: SelectedResource[] = excludeWildcard + ? resources.filter((r) => !r.wildcard) + : [...resources]; + if ( + debouncedQuery.trim().length === 0 && + selectedResources.length > 0 + ) { + const selectedNotInBase = selectedResources.filter( + (sel) => + !base.some((r) => r.resourceId === sel.resourceId) && + !(excludeWildcard && sel.wildcard) + ); + return [...selectedNotInBase, ...base]; + } + return base; + }, [debouncedQuery, resources, selectedResources, excludeWildcard]); + + const selectedIds = useMemo( + () => new Set(selectedResources.map((r) => r.resourceId)), + [selectedResources] + ); + + const toggleResource = (resource: SelectedResource) => { + if (selectedIds.has(resource.resourceId)) { + onSelectionChange( + selectedResources.filter( + (r) => r.resourceId !== resource.resourceId + ) + ); + } else { + onSelectionChange([...selectedResources, resource]); + } + }; + + return ( + + setResourceSearchQuery(v)} + /> + + {t("resourcesNotFound")} + + {showClear && onClear && ( + + {t("accessFilterClear")} + + )} + {resourcesShown.map((resource) => ( + { + toggleResource(resource); + }} + > + {}} + aria-hidden + tabIndex={-1} + /> + + {resource.name} + + + ))} + + + + ); +} diff --git a/src/components/resource-selector.tsx b/src/components/resource-selector.tsx index 625bb2d64..5dad2bf0e 100644 --- a/src/components/resource-selector.tsx +++ b/src/components/resource-selector.tsx @@ -25,13 +25,19 @@ export type ResourceSelectorProps = { selectedResource?: SelectedResource | null; onSelectResource: (resource: SelectedResource) => void; excludeWildcard?: boolean; + showClear?: boolean; + onClear?: () => void; + protocol?: string; }; export function ResourceSelector({ orgId, selectedResource, onSelectResource, - excludeWildcard = false + excludeWildcard = false, + showClear = false, + onClear, + protocol }: ResourceSelectorProps) { const t = useTranslations(); const [resourceSearchQuery, setResourceSearchQuery] = useState(""); @@ -42,7 +48,8 @@ export function ResourceSelector({ orgQueries.proxyResources({ orgId: orgId, query: debouncedSearchQuery, - perPage: 10 + perPage: 10, + protocol }) ); @@ -75,6 +82,14 @@ export function ResourceSelector({ {t("resourcesNotFound")} + {showClear && onClear && ( + + {t("accessFilterClear")} + + )} {resourcesShown.map((r) => ( void; allowClear?: boolean; + showClear?: boolean; + onClear?: () => void; + unassignedOption?: { + label: string; + selected: boolean; + onSelect: () => void; + }; }; export function UserSelector({ orgId, selectedUser, onSelectUser, - allowClear = true + allowClear = true, + showClear = false, + onClear, + unassignedOption }: UserSelectorProps) { const t = useTranslations(); const [userSearchQuery, setUserSearchQuery] = useState(""); const [debouncedValue] = useDebounce(userSearchQuery, 150); const { data: users = [] } = useQuery( - orgQueries.users({ orgId, perPage: 10, query: debouncedValue }) + orgQueries.users({ orgId, perPage: 10, term: debouncedValue }) ); const usersShown = useMemo(() => { @@ -64,6 +74,14 @@ export function UserSelector({ {t("usersNotFound")} + {showClear && onClear && ( + + {t("accessFilterClear")} + + )} {allowClear && ( )} + {unassignedOption && ( + + + {unassignedOption.label} + + )} {usersShown.map((user) => ( queryOptions({ queryKey: [ "ORG", orgId, "PROXY_RESOURCES", - { query, perPage } + { query, perPage, protocol } ] as const, queryFn: async ({ signal, meta }) => { const sp = new URLSearchParams({ @@ -404,6 +406,10 @@ export const orgQueries = { sp.set("query", query); } + if (protocol) { + sp.set("protocol", protocol); + } + const res = await meta!.api.get< AxiosResponse >(`/org/${orgId}/resources?${sp.toString()}`, { signal }); From 732fd4eba16e267f0fcf266221991d47ffb7f8da Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 11 Aug 2026 10:46:22 -0400 Subject: [PATCH 109/296] allow retrieving api key secret --- messages/en-US.json | 4 +- src/components/CreateVirtualApiKeyForm.tsx | 2 +- src/components/EditVirtualApiKeyForm.tsx | 63 +++++++++++++++-- src/components/VirtualApiKeysTable.tsx | 64 ++++++++++++++++- src/components/ui/data-table.tsx | 82 ++++++++++++---------- 5 files changed, 165 insertions(+), 50 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 5dbbc09f7..abb6bf405 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1658,7 +1658,7 @@ "virtualApiKeysName": "Name", "virtualApiKeysDescriptionOptional": "Description (optional)", "virtualApiKeysAssociateUserOptional": "Associate User (optional)", - "virtualApiKeysAssociateUserDescription": "Attribution only. Does not grant access by itself.", + "virtualApiKeysAssociateUserDescription": "Associate this key with a user to track usage.", "virtualApiKeysAllResources": "All public inference resources", "virtualApiKeysAllResourcesDescription": "Allow this key to access every public inference resource in the organization", "virtualApiKeysSelectResources": "Public Inference Resources", @@ -1666,7 +1666,7 @@ "virtualApiKeysSelectResourcesDescription": "Choose which public inference resources this key can access", "virtualApiKeysNoResources": "No resources", "virtualApiKeysSecret": "Key", - "virtualApiKeysSeeOnce": "Copy this key now. You can also view it again later from the table.", + "virtualApiKeysCopyKey": "Copy this key. You can view it again later from the table or when editing.", "virtualApiKeysSecretHint": "Use this value as a Bearer token: vk-[id].[secret]", "virtualApiKeysViewSecret": "View Secret", "virtualApiKeysViewSecretTitle": "Virtual API Key Secret", diff --git a/src/components/CreateVirtualApiKeyForm.tsx b/src/components/CreateVirtualApiKeyForm.tsx index 32a3877aa..5a3e88ce8 100644 --- a/src/components/CreateVirtualApiKeyForm.tsx +++ b/src/components/CreateVirtualApiKeyForm.tsx @@ -391,7 +391,7 @@ export default function CreateVirtualApiKeyForm({ )} {credential && (
-

{t("virtualApiKeysSeeOnce")}

+

{t("virtualApiKeysCopyKey")}

([]); + const [credential, setCredential] = useState(null); + const [credentialLoading, setCredentialLoading] = useState(false); const formSchema = z .object({ @@ -132,6 +136,55 @@ export default function EditVirtualApiKeyForm({ form.reset({ allResources: virtualApiKey.allResources }); + + let cancelled = false; + setCredentialLoading(true); + setCredential(null); + + api.get>( + `/virtual-api-key/${virtualApiKey.virtualApiKeyId}` + ) + .then((res) => { + if (cancelled) { + return; + } + const secret = res.data.data.virtualApiKey.secret; + if (secret) { + setCredential( + `vk-${virtualApiKey.virtualApiKeyId}.${secret}` + ); + } else { + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorFetchSecret"), + description: t( + "virtualApiKeysErrorFetchSecretDescription" + ) + }); + } + }) + .catch((e) => { + if (cancelled) { + return; + } + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorFetchSecret"), + description: formatAxiosError( + e, + t("virtualApiKeysErrorFetchSecretDescription") + ) + }); + }) + .finally(() => { + if (!cancelled) { + setCredentialLoading(false); + } + }); + + return () => { + cancelled = true; + }; }, [open, virtualApiKey, form]); async function onSubmit(values: z.infer) { @@ -228,11 +281,11 @@ export default function EditVirtualApiKeyForm({ id="edit-virtual-api-key-form" >
- + + - - - {addActions.map((action, i) => ( - action.onSelect()}> - {action.label} - - ))} - - - ) : onAdd && addButtonText ? ( - - ) : null; + const addAction = + addActions && addActions.length > 0 && addButtonText ? ( + + + + + + {addActions.map((action, i) => ( + action.onSelect()} + > + {action.label} + + ))} + + + ) : onAdd && addButtonText ? ( + + ) : null; return (
@@ -674,7 +685,9 @@ export function DataTable({
- ) - : undefined + showAddActionInEmptyState ? ( +
+ {addAction} +
+ ) : undefined } /> )} From 2187f23588b57c3b206e62125f2074d4f7bce829 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 11 Aug 2026 11:53:39 -0400 Subject: [PATCH 110/296] add page to retrieve user virtual api keys --- messages/en-US.json | 19 +- server/routers/external.ts | 12 + .../virtualApiKey/getMyVirtualApiKey.ts | 113 +++++++ server/routers/virtualApiKey/index.ts | 2 + .../virtualApiKey/listMyVirtualApiKeys.ts | 219 ++++++++++++++ server/routers/virtualApiKey/types.ts | 19 +- src/app/[orgId]/keys/page.tsx | 113 +++++++ .../resource/[resourceGuid]/keys/page.tsx | 121 ++++++++ src/components/UserVirtualApiKeys.tsx | 277 ++++++++++++++++++ 9 files changed, 891 insertions(+), 4 deletions(-) create mode 100644 server/routers/virtualApiKey/getMyVirtualApiKey.ts create mode 100644 server/routers/virtualApiKey/listMyVirtualApiKeys.ts create mode 100644 src/app/[orgId]/keys/page.tsx create mode 100644 src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx create mode 100644 src/components/UserVirtualApiKeys.tsx diff --git a/messages/en-US.json b/messages/en-US.json index abb6bf405..7943138d9 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1667,7 +1667,6 @@ "virtualApiKeysNoResources": "No resources", "virtualApiKeysSecret": "Key", "virtualApiKeysCopyKey": "Copy this key. You can view it again later from the table or when editing.", - "virtualApiKeysSecretHint": "Use this value as a Bearer token: vk-[id].[secret]", "virtualApiKeysViewSecret": "View Secret", "virtualApiKeysViewSecretTitle": "Virtual API Key Secret", "virtualApiKeysViewSecretDescription": "This secret grants access to the public inference resources assigned to this key", @@ -1692,6 +1691,24 @@ "virtualApiKeysErrorFetchSecret": "Error loading secret", "virtualApiKeysErrorFetchSecretDescription": "Failed to load the virtual API key secret", "virtualApiKeysFilterUnassigned": "Unassigned", + "myVirtualApiKeysTitle": "Your API Keys", + "myVirtualApiKeysDescription": "View your identity key and any virtual API keys attributed to you in this organization", + "myVirtualApiKeysResourceTitle": "Your API Keys for This Resource", + "myVirtualApiKeysResourceDescription": "View your identity key and virtual API keys attributed to you that can access this resource", + "myVirtualApiKeysIdentityTitle": "Identity Key", + "myVirtualApiKeysIdentityHeadline": "Your Personal API Key", + "myVirtualApiKeysIdentityDescription": "Your personal key for this organization. It is unique to your account and used to identify you when calling AI Gateway resources.", + "myVirtualApiKeysIdentityResourceHeadline": "Your Personal API Key for This Resource", + "myVirtualApiKeysIdentityResourceDescription": "Your personal key for this organization. Use it to call this AI Gateway resource.", + "myVirtualApiKeysManualTitle": "Attributed Keys", + "myVirtualApiKeysManualDescription": "Manual virtual API keys an admin associated with your account", + "myVirtualApiKeysManualResourceDescription": "Manual virtual API keys associated with your account that can access this resource", + "myVirtualApiKeysManualEmpty": "No attributed keys yet", + "myVirtualApiKeysKindUser": "Identity", + "myVirtualApiKeysKindManual": "Manual", + "myVirtualApiKeysUnnamed": "Unnamed key", + "myVirtualApiKeysRevealSecret": "Reveal Secret", + "myVirtualApiKeysViewSecretDescription": "This secret authenticates you to AI Gateway resources", "aiProvidersTitle": "AI Providers", "aiProvidersDescription": "Connect model providers for AI workloads in this organization", "aiProvidersAdd": "Add Provider", diff --git a/server/routers/external.ts b/server/routers/external.ts index 1eb32ec0f..141660f0f 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1650,6 +1650,18 @@ authenticated.get( virtualApiKey.listVirtualApiKeys ); +authenticated.get( + "/org/:orgId/my-virtual-api-keys", + verifyOrgAccess, + virtualApiKey.listMyVirtualApiKeys +); + +authenticated.get( + "/org/:orgId/my-virtual-api-keys/:virtualApiKeyId", + verifyOrgAccess, + virtualApiKey.getMyVirtualApiKey +); + authenticated.get( "/virtual-api-key/:virtualApiKeyId", verifyVirtualApiKeyAccess, diff --git a/server/routers/virtualApiKey/getMyVirtualApiKey.ts b/server/routers/virtualApiKey/getMyVirtualApiKey.ts new file mode 100644 index 000000000..2c8887f55 --- /dev/null +++ b/server/routers/virtualApiKey/getMyVirtualApiKey.ts @@ -0,0 +1,113 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, virtualApiKeyResources, virtualApiKeys } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq } from "drizzle-orm"; +import { toPublicVirtualApiKey } from "@server/lib/virtualApiKey"; +import type { GetMyVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty(), + virtualApiKeyId: z.string().nonempty() +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/my-virtual-api-keys/{virtualApiKeyId}", + description: + "Get a virtual API key owned by the signed-in user, including the decrypted secret.", + tags: [OpenAPITags.VirtualApiKey], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function getMyVirtualApiKey( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { orgId, virtualApiKeyId } = parsedParams.data; + const userId = req.user?.userId; + + if (!userId) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "User not authenticated") + ); + } + + if (orgId !== req.userOrgId) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + const [key] = await db + .select() + .from(virtualApiKeys) + .where( + and( + eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId), + eq(virtualApiKeys.orgId, orgId), + eq(virtualApiKeys.userId, userId) + ) + ) + .limit(1); + + if (!key) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Virtual API key with ID ${virtualApiKeyId} not found` + ) + ); + } + + const resourceRows = await db + .select({ resourceId: virtualApiKeyResources.resourceId }) + .from(virtualApiKeyResources) + .where(eq(virtualApiKeyResources.virtualApiKeyId, virtualApiKeyId)); + + return response(res, { + data: { + virtualApiKey: { + ...toPublicVirtualApiKey(key, { includeSecret: true }), + resourceIds: resourceRows.map((row) => row.resourceId) + } + }, + success: true, + error: false, + message: "Virtual API key retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/virtualApiKey/index.ts b/server/routers/virtualApiKey/index.ts index b80fba765..2cb06a5ca 100644 --- a/server/routers/virtualApiKey/index.ts +++ b/server/routers/virtualApiKey/index.ts @@ -1,6 +1,8 @@ export * from "./createVirtualApiKey"; export * from "./listVirtualApiKeys"; +export * from "./listMyVirtualApiKeys"; export * from "./getVirtualApiKey"; +export * from "./getMyVirtualApiKey"; export * from "./updateVirtualApiKey"; export * from "./deleteVirtualApiKey"; export * from "./types"; diff --git a/server/routers/virtualApiKey/listMyVirtualApiKeys.ts b/server/routers/virtualApiKey/listMyVirtualApiKeys.ts new file mode 100644 index 000000000..244377ef1 --- /dev/null +++ b/server/routers/virtualApiKey/listMyVirtualApiKeys.ts @@ -0,0 +1,219 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { + db, + resources, + virtualApiKeyResources, + virtualApiKeys, + type VirtualApiKey +} from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, asc, eq, exists, inArray, or } from "drizzle-orm"; +import { + getOrCreateUserVirtualApiKey, + toPublicVirtualApiKey +} from "@server/lib/virtualApiKey"; +import type { ListMyVirtualApiKeysResponse } from "@server/routers/virtualApiKey/types"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +const querySchema = z.object({ + resourceGuid: z.string().nonempty().optional() +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/my-virtual-api-keys", + description: + "List the signed-in user's identity virtual API key and manual keys attributed to them.", + tags: [OpenAPITags.VirtualApiKey], + request: { + params: paramsSchema, + query: querySchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +async function resourceIdsForKeys( + keyIds: string[] +): Promise> { + const resourceIdsByKey = new Map(); + if (keyIds.length === 0) { + return resourceIdsByKey; + } + + const resourceRows = await db + .select() + .from(virtualApiKeyResources) + .where(inArray(virtualApiKeyResources.virtualApiKeyId, keyIds)); + + for (const row of resourceRows) { + const existing = resourceIdsByKey.get(row.virtualApiKeyId) ?? []; + existing.push(row.resourceId); + resourceIdsByKey.set(row.virtualApiKeyId, existing); + } + + return resourceIdsByKey; +} + +function toKeyWithResources( + row: VirtualApiKey, + resourceIdsByKey: Map +) { + return { + ...toPublicVirtualApiKey(row), + resourceIds: resourceIdsByKey.get(row.virtualApiKeyId) ?? [] + }; +} + +export async function listMyVirtualApiKeys( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedQuery = querySchema.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error).toString() + ) + ); + } + + const { orgId } = parsedParams.data; + const { resourceGuid } = parsedQuery.data; + const userId = req.user?.userId; + + if (!userId) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "User not authenticated") + ); + } + + if (orgId !== req.userOrgId) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + let resourceId: number | undefined; + if (resourceGuid) { + const [resource] = await db + .select({ + resourceId: resources.resourceId + }) + .from(resources) + .where( + and( + eq(resources.resourceGuid, resourceGuid), + eq(resources.orgId, orgId) + ) + ) + .limit(1); + + if (!resource) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Resource with GUID ${resourceGuid} not found` + ) + ); + } + + resourceId = resource.resourceId; + } + + const { key: userKeyRow } = await getOrCreateUserVirtualApiKey({ + orgId, + userId, + createdByUserId: userId + }); + + const manualConditions = [ + eq(virtualApiKeys.orgId, orgId), + eq(virtualApiKeys.userId, userId), + eq(virtualApiKeys.kind, "manual") + ]; + + if (resourceId !== undefined) { + manualConditions.push( + or( + eq(virtualApiKeys.allResources, true), + exists( + db + .select() + .from(virtualApiKeyResources) + .where( + and( + eq( + virtualApiKeyResources.virtualApiKeyId, + virtualApiKeys.virtualApiKeyId + ), + eq( + virtualApiKeyResources.resourceId, + resourceId + ) + ) + ) + ) + )! + ); + } + + const manualRows = await db + .select() + .from(virtualApiKeys) + .where(and(...manualConditions)) + .orderBy(asc(virtualApiKeys.name), asc(virtualApiKeys.createdAt)); + + const allKeyIds = [ + userKeyRow.virtualApiKeyId, + ...manualRows.map((row) => row.virtualApiKeyId) + ]; + const resourceIdsByKey = await resourceIdsForKeys(allKeyIds); + + return response(res, { + data: { + userKey: toKeyWithResources(userKeyRow, resourceIdsByKey), + manualKeys: manualRows.map((row) => + toKeyWithResources(row, resourceIdsByKey) + ) + }, + success: true, + error: false, + message: "Virtual API keys retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/virtualApiKey/types.ts b/server/routers/virtualApiKey/types.ts index 28823577f..73e475632 100644 --- a/server/routers/virtualApiKey/types.ts +++ b/server/routers/virtualApiKey/types.ts @@ -3,14 +3,27 @@ import type { PaginatedResponse } from "@server/types/Pagination"; export type { PublicVirtualApiKey }; +export type VirtualApiKeyWithResources = PublicVirtualApiKey & { + resourceIds: number[]; +}; + export type ListVirtualApiKeysResponse = PaginatedResponse<{ - virtualApiKeys: (PublicVirtualApiKey & { resourceIds: number[] })[]; + virtualApiKeys: VirtualApiKeyWithResources[]; }>; export type GetVirtualApiKeyResponse = { - virtualApiKey: PublicVirtualApiKey & { resourceIds: number[] }; + virtualApiKey: VirtualApiKeyWithResources; }; export type CreateOrEditVirtualApiKeyResponse = { - virtualApiKey: PublicVirtualApiKey & { resourceIds: number[] }; + virtualApiKey: VirtualApiKeyWithResources; +}; + +export type ListMyVirtualApiKeysResponse = { + userKey: VirtualApiKeyWithResources; + manualKeys: VirtualApiKeyWithResources[]; +}; + +export type GetMyVirtualApiKeyResponse = { + virtualApiKey: VirtualApiKeyWithResources; }; diff --git a/src/app/[orgId]/keys/page.tsx b/src/app/[orgId]/keys/page.tsx new file mode 100644 index 000000000..007ed2869 --- /dev/null +++ b/src/app/[orgId]/keys/page.tsx @@ -0,0 +1,113 @@ +import { Layout } from "@app/components/Layout"; +import UserVirtualApiKeys from "@app/components/UserVirtualApiKeys"; +import { commandBarNavSections } from "@app/app/navigation"; +import { internal } from "@app/lib/api"; +import { authCookieHeader } from "@app/lib/api/cookies"; +import { verifySession } from "@app/lib/auth/verifySession"; +import { pullEnv } from "@app/lib/pullEnv"; +import UserProvider from "@app/providers/UserProvider"; +import { ListUserOrgsResponse } from "@server/routers/org"; +import { GetOrgOverviewResponse } from "@server/routers/org/getOrgOverview"; +import type { ListMyVirtualApiKeysResponse } from "@server/routers/virtualApiKey/types"; +import { AxiosResponse } from "axios"; +import type { Metadata } from "next"; +import { getTranslations } from "next-intl/server"; +import { redirect } from "next/navigation"; +import { cache } from "react"; + +export async function generateMetadata(): Promise { + const t = await getTranslations(); + return { + title: t("myVirtualApiKeysTitle") + }; +} + +type KeysPageProps = { + params: Promise<{ orgId: string }>; +}; + +export const dynamic = "force-dynamic"; + +export default async function KeysPage(props: KeysPageProps) { + const params = await props.params; + const orgId = params.orgId; + + if (!orgId) { + redirect(`/`); + } + + const getUser = cache(verifySession); + const user = await getUser(); + + if (!user) { + redirect("/"); + } + + const cookieHeader = await authCookieHeader(); + + let overview: GetOrgOverviewResponse | undefined; + try { + const res = await internal.get>( + `/org/${orgId}/overview`, + cookieHeader + ); + overview = res.data.data; + } catch { + // leave undefined + } + + let orgs: ListUserOrgsResponse["orgs"] = []; + try { + const getOrgs = cache(async () => + internal.get>( + `/user/${user.userId}/orgs`, + cookieHeader + ) + ); + const res = await getOrgs(); + if (res && res.data.data.orgs) { + orgs = res.data.data.orgs; + } + } catch { + // leave empty + } + + if (!orgs.some((org) => org.orgId === orgId)) { + redirect("/"); + } + + let keysData: ListMyVirtualApiKeysResponse | null = null; + try { + const res = await internal.get< + AxiosResponse + >(`/org/${orgId}/my-virtual-api-keys`, cookieHeader); + keysData = res.data.data; + } catch { + redirect(`/${orgId}`); + } + + if (!keysData) { + redirect(`/${orgId}`); + } + + const env = pullEnv(); + const primaryOrg = orgs.find((o) => o.orgId === orgId)?.isPrimaryOrg; + const isAdminOrOwner = Boolean(overview?.isAdmin || overview?.isOwner); + + return ( + + + + + + ); +} diff --git a/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx b/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx new file mode 100644 index 000000000..e85bff9fb --- /dev/null +++ b/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx @@ -0,0 +1,121 @@ +import { Layout } from "@app/components/Layout"; +import UserVirtualApiKeys from "@app/components/UserVirtualApiKeys"; +import { commandBarNavSections } from "@app/app/navigation"; +import { internal } from "@app/lib/api"; +import { authCookieHeader } from "@app/lib/api/cookies"; +import { verifySession } from "@app/lib/auth/verifySession"; +import { pullEnv } from "@app/lib/pullEnv"; +import UserProvider from "@app/providers/UserProvider"; +import { ListUserOrgsResponse } from "@server/routers/org"; +import { GetOrgOverviewResponse } from "@server/routers/org/getOrgOverview"; +import type { ListMyVirtualApiKeysResponse } from "@server/routers/virtualApiKey/types"; +import { AxiosResponse } from "axios"; +import type { Metadata } from "next"; +import { getTranslations } from "next-intl/server"; +import { redirect } from "next/navigation"; +import { cache } from "react"; + +export async function generateMetadata(): Promise { + const t = await getTranslations(); + return { + title: t("myVirtualApiKeysResourceTitle") + }; +} + +type ResourceKeysPageProps = { + params: Promise<{ orgId: string; resourceGuid: string }>; +}; + +export const dynamic = "force-dynamic"; + +export default async function ResourceKeysPage(props: ResourceKeysPageProps) { + const params = await props.params; + const orgId = params.orgId; + const resourceGuid = params.resourceGuid; + + if (!orgId || !resourceGuid) { + redirect(`/`); + } + + const getUser = cache(verifySession); + const user = await getUser(); + + if (!user) { + redirect("/"); + } + + const cookieHeader = await authCookieHeader(); + + let overview: GetOrgOverviewResponse | undefined; + try { + const res = await internal.get>( + `/org/${orgId}/overview`, + cookieHeader + ); + overview = res.data.data; + } catch { + // leave undefined + } + + let orgs: ListUserOrgsResponse["orgs"] = []; + try { + const getOrgs = cache(async () => + internal.get>( + `/user/${user.userId}/orgs`, + cookieHeader + ) + ); + const res = await getOrgs(); + if (res && res.data.data.orgs) { + orgs = res.data.data.orgs; + } + } catch { + // leave empty + } + + if (!orgs.some((org) => org.orgId === orgId)) { + redirect("/"); + } + + let keysData: ListMyVirtualApiKeysResponse | null = null; + try { + const res = await internal.get< + AxiosResponse + >( + `/org/${orgId}/my-virtual-api-keys?resourceGuid=${encodeURIComponent(resourceGuid)}`, + cookieHeader + ); + keysData = res.data.data; + } catch { + redirect(`/${orgId}/keys`); + } + + if (!keysData) { + redirect(`/${orgId}/keys`); + } + + const env = pullEnv(); + const primaryOrg = orgs.find((o) => o.orgId === orgId)?.isPrimaryOrg; + const isAdminOrOwner = Boolean(overview?.isAdmin || overview?.isOwner); + + return ( + + + + + + ); +} diff --git a/src/components/UserVirtualApiKeys.tsx b/src/components/UserVirtualApiKeys.tsx new file mode 100644 index 000000000..5513b59e4 --- /dev/null +++ b/src/components/UserVirtualApiKeys.tsx @@ -0,0 +1,277 @@ +"use client"; + +import { useState } from "react"; +import { useTranslations } from "next-intl"; +import { AxiosResponse } from "axios"; +import moment from "moment"; +import { Badge } from "@app/components/ui/badge"; +import { Button } from "@app/components/ui/button"; +import CopyTextBox from "@app/components/CopyTextBox"; +import CopyToClipboard from "@app/components/CopyToClipboard"; +import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; +import { + SettingsContainer, + SettingsFormCell, + SettingsFormGrid, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionHeader, + SettingsSectionTitle as SectionTitle +} from "@app/components/Settings"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import type { + GetMyVirtualApiKeyResponse, + ListMyVirtualApiKeysResponse, + VirtualApiKeyWithResources +} from "@server/routers/virtualApiKey/types"; + +type UserVirtualApiKeysProps = { + orgId: string; + resourceGuid?: string; + initialData: ListMyVirtualApiKeysResponse; +}; + +function keyPreview(virtualApiKeyId: string, lastChars: string): string { + return `vk-${virtualApiKeyId}••••${lastChars}`; +} + +function useRevealSecret(orgId: string, virtualApiKeyId: string) { + const t = useTranslations(); + const api = createApiClient(useEnvContext()); + const [credential, setCredential] = useState(null); + const [loading, setLoading] = useState(false); + + const revealSecret = () => { + if (credential || loading) { + return; + } + + setLoading(true); + api.get>( + `/org/${orgId}/my-virtual-api-keys/${virtualApiKeyId}` + ) + .then((res) => { + const secret = res.data.data.virtualApiKey.secret; + if (secret) { + setCredential(`vk-${virtualApiKeyId}.${secret}`); + } else { + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorFetchSecret"), + description: t( + "virtualApiKeysErrorFetchSecretDescription" + ) + }); + } + }) + .catch((e) => { + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorFetchSecret"), + description: formatAxiosError( + e, + t("virtualApiKeysErrorFetchSecretDescription") + ) + }); + }) + .finally(() => { + setLoading(false); + }); + }; + + return { credential, loading, revealSecret }; +} + +function OwnedKeySecret({ + orgId, + virtualApiKeyId, + lastChars +}: { + orgId: string; + virtualApiKeyId: string; + lastChars: string; +}) { + const t = useTranslations(); + const preview = keyPreview(virtualApiKeyId, lastChars); + const { credential, loading, revealSecret } = useRevealSecret( + orgId, + virtualApiKeyId + ); + const displayValue = credential ?? preview; + + return ( +
+
+ +
+ {!credential ? ( + + ) : null} +
+ ); +} + +function IdentityKeyCenterpiece({ + orgId, + virtualApiKeyId, + lastChars, + resourceGuid +}: { + orgId: string; + virtualApiKeyId: string; + lastChars: string; + resourceGuid?: string; +}) { + const t = useTranslations(); + const preview = keyPreview(virtualApiKeyId, lastChars); + const { credential, loading, revealSecret } = useRevealSecret( + orgId, + virtualApiKeyId + ); + const displayValue = credential ?? preview; + const headline = resourceGuid + ? t("myVirtualApiKeysIdentityResourceHeadline") + : t("myVirtualApiKeysIdentityHeadline"); + const description = resourceGuid + ? t("myVirtualApiKeysIdentityResourceDescription") + : t("myVirtualApiKeysIdentityDescription"); + + return ( +
+

+ {headline} +

+

+ {description} +

+
+
+ +
+ {!credential ? ( +
+ +
+ ) : null} +
+
+ ); +} + +function ManualKeyRow({ + orgId, + keyRow +}: { + orgId: string; + keyRow: VirtualApiKeyWithResources; +}) { + const t = useTranslations(); + + return ( +
+
+
+

+ {keyRow.name || t("myVirtualApiKeysUnnamed")} +

+
+ {keyRow.description ? ( +

+ {keyRow.description} +

+ ) : null} +
+ +
+

+ {t("created")} {moment(keyRow.createdAt).format("lll")} +

+
+
+ ); +} + +export default function UserVirtualApiKeys({ + orgId, + resourceGuid, + initialData +}: UserVirtualApiKeysProps) { + const t = useTranslations(); + + const title = resourceGuid + ? t("myVirtualApiKeysResourceTitle") + : t("myVirtualApiKeysTitle"); + const description = resourceGuid + ? t("myVirtualApiKeysResourceDescription") + : t("myVirtualApiKeysDescription"); + + return ( + <> + + + + {initialData.manualKeys.length > 0 ? ( + + + + {t("myVirtualApiKeysManualTitle")} + + + {resourceGuid + ? t( + "myVirtualApiKeysManualResourceDescription" + ) + : t("myVirtualApiKeysManualDescription")} + + + + + {initialData.manualKeys.map((keyRow) => ( + + + + ))} + + + + ) : null} + + + ); +} From b33e776072018b3bfa2c57ba1c7862e058793c53 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 11 Aug 2026 12:09:11 -0400 Subject: [PATCH 111/296] add virtual keys link in header on launcher --- messages/en-US.json | 2 + src/app/[orgId]/keys/page.tsx | 1 + .../resource/[resourceGuid]/keys/page.tsx | 1 + src/app/navigation.tsx | 5 ++ src/components/Layout.tsx | 14 ++++- src/components/LayoutHeader.tsx | 44 +++++++++++++- src/components/LayoutMobileMenu.tsx | 59 +++++++++++++++++-- .../useCommandPaletteActions.tsx | 6 ++ 8 files changed, 123 insertions(+), 9 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 7943138d9..b2594efcb 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1646,6 +1646,8 @@ "sidebarAiProviders": "Providers", "commandAiProviders": "AI Providers", "sidebarVirtualApiKeys": "Virtual API Keys", + "sidebarMyApiKeys": "Your API Keys", + "sidebarAccount": "Launcher", "commandVirtualApiKeys": "Virtual API Keys", "virtualApiKeysTitle": "Manage Virtual API Keys", "virtualApiKeysDescription": "Create and manage manual API keys for AI Gateway access to public inference resources", diff --git a/src/app/[orgId]/keys/page.tsx b/src/app/[orgId]/keys/page.tsx index 007ed2869..e8341dd29 100644 --- a/src/app/[orgId]/keys/page.tsx +++ b/src/app/[orgId]/keys/page.tsx @@ -104,6 +104,7 @@ export default async function KeysPage(props: KeysPageProps) { isPrimaryOrg: primaryOrg })} showSidebar={false} + launcherMode showViewAsAdmin={isAdminOrOwner} > diff --git a/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx b/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx index e85bff9fb..29e11eb92 100644 --- a/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx +++ b/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx @@ -108,6 +108,7 @@ export default async function ResourceKeysPage(props: ResourceKeysPageProps) { isPrimaryOrg: primaryOrg })} showSidebar={false} + launcherMode showViewAsAdmin={isAdminOrOwner} > + }, + { + title: "sidebarMyApiKeys", + href: "/{orgId}/keys", + icon: } ]; diff --git a/src/components/Layout.tsx b/src/components/Layout.tsx index 085abbfac..7adc2b5a0 100644 --- a/src/components/Layout.tsx +++ b/src/components/Layout.tsx @@ -1,10 +1,12 @@ import React from "react"; import { cn } from "@app/lib/cn"; import { ListUserOrgsResponse } from "@server/routers/org"; -import type { - CommandBarNavSection, - SidebarNavSection +import { + orgLangingNavItems, + type CommandBarNavSection, + type SidebarNavSection } from "@app/app/navigation"; +import type { SidebarNavItem } from "@app/components/SidebarNav"; import { LayoutSidebar } from "@app/components/LayoutSidebar"; import { LayoutHeader } from "@app/components/LayoutHeader"; import { LayoutMobileMenu } from "@app/components/LayoutMobileMenu"; @@ -46,6 +48,10 @@ export async function Layout({ sidebarStateCookie === "collapsed" || (sidebarStateCookie !== "expanded" && defaultSidebarCollapsed); + const launcherNavItems: SidebarNavItem[] = launcherMode + ? orgLangingNavItems + : []; + return ( )} diff --git a/src/components/LayoutHeader.tsx b/src/components/LayoutHeader.tsx index 2440d2494..ac265ff08 100644 --- a/src/components/LayoutHeader.tsx +++ b/src/components/LayoutHeader.tsx @@ -2,6 +2,7 @@ import React, { useEffect, useState } from "react"; import Link from "next/link"; +import { usePathname } from "next/navigation"; import ProfileIcon from "@app/components/ProfileIcon"; import ThemeSwitcher from "@app/components/ThemeSwitcher"; import { useTheme } from "next-themes"; @@ -13,6 +14,8 @@ import { LauncherOrgSelector } from "@app/components/resource-launcher/LauncherO import { Button } from "@app/components/ui/button"; import { useTranslations } from "next-intl"; import { CommandPaletteTrigger } from "@app/components/command-palette/CommandPaletteTrigger"; +import type { SidebarNavItem } from "@app/components/SidebarNav"; +import { cn } from "@app/lib/cn"; type LayoutHeaderProps = { showTopBar: boolean; @@ -20,6 +23,7 @@ type LayoutHeaderProps = { orgId?: string; orgs?: ListUserOrgsResponse["orgs"]; showViewAsAdmin?: boolean; + launcherNavItems?: SidebarNavItem[]; }; export function LayoutHeader({ @@ -27,13 +31,15 @@ export function LayoutHeader({ launcherMode = false, orgId, orgs, - showViewAsAdmin = false + showViewAsAdmin = false, + launcherNavItems = [] }: LayoutHeaderProps) { const { theme } = useTheme(); const [path, setPath] = useState(""); const { env } = useEnvContext(); const { isUnlocked } = useLicenseStatusContext(); const t = useTranslations(); + const pathname = usePathname(); const logoWidth = isUnlocked() ? env.branding.logo?.navbar?.width || 98 @@ -85,6 +91,42 @@ export function LayoutHeader({ orgId={orgId} orgs={orgs} /> + {orgId + ? launcherNavItems + .filter((item) => item.href) + .map((item) => { + const href = + item.href!.replace( + "{orgId}", + orgId + ); + const isActive = + href === `/${orgId}` + ? pathname === href + : pathname === href || + pathname?.startsWith( + `${href}/` + ); + + return ( + + ); + }) + : null} {showViewAsAdmin && orgId ? (
- {showViewAsAdmin && orgId ? ( -
+
+ {orgId + ? launcherNavItems + .filter( + (item) => + item.href + ) + .map((item) => { + const href = + item.href!.replace( + "{orgId}", + orgId + ); + return ( +
+ + setIsMobileMenuOpen( + false + ) + } + > + {item.icon ? ( + + { + item.icon + } + + ) : null} + + {t( + item.title + )} + + +
+ ); + }) + : null} + {showViewAsAdmin && orgId ? (
-
- ) : null} + ) : null} +
) : ( <> diff --git a/src/components/command-palette/useCommandPaletteActions.tsx b/src/components/command-palette/useCommandPaletteActions.tsx index da44e5d2a..87e102307 100644 --- a/src/components/command-palette/useCommandPaletteActions.tsx +++ b/src/components/command-palette/useCommandPaletteActions.tsx @@ -75,6 +75,12 @@ export function useCommandPaletteActions( }); } } else if (orgId) { + actions.push({ + id: "my-api-keys", + label: t("sidebarMyApiKeys"), + icon: , + href: `/${orgId}/keys` + }); actions.push({ id: "create-site", label: t("commandPaletteCreateSite"), From b60390d805eb91f26d3129931dfcd438ccdb1217 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 11:10:30 -0400 Subject: [PATCH 112/296] batch up usage records for insert --- server/cleanup.ts | 2 + server/lib/aiBudgetEnforcement.ts | 106 +++++++++++++++++++++++++++++- server/private/cleanup.ts | 2 + 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/server/cleanup.ts b/server/cleanup.ts index 10e9f4cc3..3e7a335b4 100644 --- a/server/cleanup.ts +++ b/server/cleanup.ts @@ -3,12 +3,14 @@ import { flushConnectionLogToDb } from "#dynamic/routers/newt"; import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth"; import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator"; import { cleanup as wsCleanup } from "#dynamic/routers/ws"; +import { shutdownUsageRecorder } from "@server/lib/aiBudgetEnforcement"; async function cleanup() { await stopPingAccumulator(); await flushBandwidthToDb(); await flushConnectionLogToDb(); await flushSiteBandwidthToDb(); + await shutdownUsageRecorder(); await wsCleanup(); process.exit(0); diff --git a/server/lib/aiBudgetEnforcement.ts b/server/lib/aiBudgetEnforcement.ts index 2a516695d..4177af9b7 100644 --- a/server/lib/aiBudgetEnforcement.ts +++ b/server/lib/aiBudgetEnforcement.ts @@ -1,4 +1,4 @@ -import { and, eq, gte, inArray, isNull, or, sql, SQL } from "drizzle-orm"; +import { and, eq, gte, inArray, isNull, or, sql, SQL, type InferInsertModel } from "drizzle-orm"; import { AiBudget, aiBudgetBreachEvents, @@ -430,6 +430,91 @@ export type UsageRecordInput = { createdAt?: number; }; +type AiUsageRecordInsert = InferInsertModel; + +// In-memory buffer for batching AI usage record inserts, mirroring the +// approach in server/routers/badger/logRequestAudit.ts. Usage rows are read +// back on every budget-cache miss (see getBudgetUsage above), which happens +// at least every BUDGET_CACHE_REFRESH_MS, so this buffer is flushed much +// more aggressively than the request audit log to keep the table from +// lagging behind what budget enforcement needs. Unlike the audit log, there +// is no retention/cleanup job for this table - usage history is kept +// indefinitely for billing and historical reporting. +const usageRecordBuffer: AiUsageRecordInsert[] = []; + +const USAGE_BATCH_SIZE = 20; // Write to DB every 20 records +const USAGE_BATCH_INTERVAL_MS = 1000; // Or every 1 second, whichever comes first +const USAGE_MAX_BUFFER_SIZE = 5000; // Prevent unbounded memory growth +let usageFlushTimer: NodeJS.Timeout | null = null; +let isUsageFlushInProgress = false; + +async function flushUsageRecords() { + if (usageRecordBuffer.length === 0 || isUsageFlushInProgress) { + return; + } + + isUsageFlushInProgress = true; + + const recordsToWrite = usageRecordBuffer.splice(0, usageRecordBuffer.length); + + try { + // Use a transaction to ensure all inserts succeed or fail together + await db.transaction(async (tx) => { + // Batch insert in groups to avoid overwhelming the database + const DB_BATCH_SIZE = 25; + for (let i = 0; i < recordsToWrite.length; i += DB_BATCH_SIZE) { + const batch = recordsToWrite.slice(i, i + DB_BATCH_SIZE); + await tx.insert(aiUsageRecords).values(batch); + } + }); + logger.debug(`Flushed ${recordsToWrite.length} AI usage records to database`); + } catch (error) { + logger.error("Error flushing AI usage records:", error); + // On transaction error, put records back at the front of the buffer + // to retry, but only if the buffer isn't too large + if (usageRecordBuffer.length < USAGE_MAX_BUFFER_SIZE - recordsToWrite.length) { + usageRecordBuffer.unshift(...recordsToWrite); + logger.info(`Re-queued ${recordsToWrite.length} AI usage records for retry`); + } else { + logger.error(`Buffer full, dropped ${recordsToWrite.length} AI usage records`); + } + } finally { + isUsageFlushInProgress = false; + // If buffer filled up while we were flushing, flush again + if (usageRecordBuffer.length >= USAGE_BATCH_SIZE) { + flushUsageRecords().catch((err) => + logger.error("Error in follow-up AI usage flush:", err) + ); + } + } +} + +function scheduleUsageFlush() { + if (usageFlushTimer === null) { + usageFlushTimer = setTimeout(() => { + usageFlushTimer = null; + flushUsageRecords().catch((err) => + logger.error("Error in scheduled AI usage flush:", err) + ); + }, USAGE_BATCH_INTERVAL_MS); + } +} + +/** + * Gracefully flush all pending AI usage records (call this on shutdown). + */ +export async function shutdownUsageRecorder() { + if (usageFlushTimer) { + clearTimeout(usageFlushTimer); + usageFlushTimer = null; + } + // Force flush even if one is in progress by waiting and retrying + while (isUsageFlushInProgress) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + await flushUsageRecords(); +} + export async function recordUsage(input: UsageRecordInput): Promise { try { const { usage } = input; @@ -440,7 +525,15 @@ export async function recordUsage(input: UsageRecordInput): Promise { usage.completionTokens + usage.reasoningTokens; - await db.insert(aiUsageRecords).values({ + // Prevent unbounded buffer growth - drop oldest entries if buffer is too large + if (usageRecordBuffer.length >= USAGE_MAX_BUFFER_SIZE) { + const dropped = usageRecordBuffer.splice(0, USAGE_BATCH_SIZE); + logger.warn( + `AI usage record buffer exceeded max size (${USAGE_MAX_BUFFER_SIZE}), dropped ${dropped.length} oldest entries` + ); + } + + usageRecordBuffer.push({ orgId: input.orgId, providerId: input.providerId, resourceId: input.resourceId, @@ -457,6 +550,15 @@ export async function recordUsage(input: UsageRecordInput): Promise { estimated: usage.estimated, createdAt: input.createdAt ?? Date.now() }); + + // Flush immediately if buffer is full, otherwise schedule a flush + if (usageRecordBuffer.length >= USAGE_BATCH_SIZE) { + flushUsageRecords().catch((err) => + logger.error("Error flushing AI usage records:", err) + ); + } else { + scheduleUsageFlush(); + } } catch (error) { logger.error("Failed to record AI usage", { error }); } diff --git a/server/private/cleanup.ts b/server/private/cleanup.ts index 6c934cb0b..a1ad24976 100644 --- a/server/private/cleanup.ts +++ b/server/private/cleanup.ts @@ -18,12 +18,14 @@ import { flushBandwidthToDb } from "@server/routers/newt/handleReceiveBandwidthM import { flushConnectionLogToDb } from "#private/routers/newt"; import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth"; import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator"; +import { shutdownUsageRecorder } from "@server/lib/aiBudgetEnforcement"; async function cleanup() { await stopPingAccumulator(); await flushBandwidthToDb(); await flushConnectionLogToDb(); await flushSiteBandwidthToDb(); + await shutdownUsageRecorder(); await rateLimitService.cleanup(); await wsCleanup(); await logStreamingManager.shutdown(); From 68a84181d9c0801009db7cf367720451561849e2 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 12:11:21 -0400 Subject: [PATCH 113/296] remove excess logging --- server/lib/cache.ts | 10 +++++----- server/private/lib/cache.ts | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/server/lib/cache.ts b/server/lib/cache.ts index 0113099e1..4e2b7c782 100644 --- a/server/lib/cache.ts +++ b/server/lib/cache.ts @@ -48,11 +48,11 @@ class AdaptiveCache { async get(key: string): Promise { // Use local cache as fallback or primary const value = localCache.get(key); - if (value !== undefined) { - logger.debug(`Cache hit in local cache: ${key}`); - } else { - logger.debug(`Cache miss in local cache: ${key}`); - } + // if (value !== undefined) { + // logger.debug(`Cache hit in local cache: ${key}`); + // } else { + // logger.debug(`Cache miss in local cache: ${key}`); + // } return value; } diff --git a/server/private/lib/cache.ts b/server/private/lib/cache.ts index 9c94109c2..fe7b771c9 100644 --- a/server/private/lib/cache.ts +++ b/server/private/lib/cache.ts @@ -111,11 +111,11 @@ class AdaptiveCache { // Use local cache as fallback or primary const value = localCache.get(key); - if (value !== undefined) { - logger.debug(`Cache hit in local cache: ${key}`); - } else { - logger.debug(`Cache miss in local cache: ${key}`); - } + // if (value !== undefined) { + // logger.debug(`Cache hit in local cache: ${key}`); + // } else { + // logger.debug(`Cache miss in local cache: ${key}`); + // } return value; } @@ -367,11 +367,11 @@ class RegionalAdaptiveCache { } const value = regionalLocalCache.get(key); - if (value !== undefined) { - logger.debug(`[regional] Cache hit in local cache: ${key}`); - } else { - logger.debug(`[regional] Cache miss in local cache: ${key}`); - } + // if (value !== undefined) { + // logger.debug(`[regional] Cache hit in local cache: ${key}`); + // } else { + // logger.debug(`[regional] Cache miss in local cache: ${key}`); + // } return value; } From 48c4b44f7289be80827225da92da0bb0f518222d Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 12:19:17 -0400 Subject: [PATCH 114/296] log chat sessions to database --- messages/en-US.json | 2 + server/cleanup.ts | 2 + server/db/pg/schema/schema.ts | 70 ++++++ server/db/sqlite/schema/schema.ts | 74 ++++++ server/lib/cleanupLogs.ts | 21 +- server/private/cleanup.ts | 2 + .../routers/billing/featureLifecycle.ts | 11 + server/routers/aiGateway/logAiSession.ts | 238 ++++++++++++++++++ server/routers/aiGateway/pipeline.ts | 142 +++++------ .../aiGateway/streamAiGatewayResponse.ts | 86 +++++++ server/routers/aiGateway/targetRouting.ts | 88 ++++--- server/routers/org/updateOrg.ts | 20 ++ .../settings/general/security/page.tsx | 109 +++++++- 13 files changed, 749 insertions(+), 116 deletions(-) create mode 100644 server/routers/aiGateway/logAiSession.ts create mode 100644 server/routers/aiGateway/streamAiGatewayResponse.ts diff --git a/messages/en-US.json b/messages/en-US.json index b2594efcb..c7db842a7 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -3397,6 +3397,8 @@ "logRetentionActionDescription": "How long to retain action logs", "logRetentionConnectionLabel": "Network Log Retention", "logRetentionConnectionDescription": "How long to retain connection logs", + "logRetentionAISessionsLabel": "AI Gateway Session Log Retention", + "logRetentionAISessionsDescription": "How long to retain AI gateway prompt/response session logs", "logRetentionDisabled": "Disabled", "logRetention3Days": "3 days", "logRetention7Days": "7 days", diff --git a/server/cleanup.ts b/server/cleanup.ts index 3e7a335b4..3ff4ccf19 100644 --- a/server/cleanup.ts +++ b/server/cleanup.ts @@ -4,6 +4,7 @@ import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth" import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator"; import { cleanup as wsCleanup } from "#dynamic/routers/ws"; import { shutdownUsageRecorder } from "@server/lib/aiBudgetEnforcement"; +import { shutdownAiSessionLogger } from "@server/routers/aiGateway/logAiSession"; async function cleanup() { await stopPingAccumulator(); @@ -11,6 +12,7 @@ async function cleanup() { await flushConnectionLogToDb(); await flushSiteBandwidthToDb(); await shutdownUsageRecorder(); + await shutdownAiSessionLogger(); await wsCleanup(); process.exit(0); diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 39dfb889e..7895b0baf 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -65,6 +65,11 @@ export const orgs = pgTable("orgs", { ) // where 0 = dont keep logs and -1 = keep forever and 9001 = end of the following year .notNull() .default(0), + settingsLogRetentionDaysAISessions: integer( + "settingsLogRetentionDaysAISessions" + ) // where 0 = dont keep logs and -1 = keep forever and 9001 = end of the following year + .notNull() + .default(0), sshCaPrivateKey: text("sshCaPrivateKey"), // Encrypted SSH CA private key (PEM format) sshCaPublicKey: text("sshCaPublicKey"), // SSH CA public key (OpenSSH format) isBillingOrg: boolean("isBillingOrg"), @@ -1899,6 +1904,70 @@ export const aiBudgetBreachEvents = pgTable( ] ); +// Logs the aggregated prompt + response for a single AI gateway request, for +// session replay. One row per request (not per streaming chunk). `sessionId` +// is a fresh random id per row for now - no cross-request correlation yet, +// but the column exists so a future pass can link multiple rows into a real +// multi-turn session. +export const aiSessionLog = pgTable( + "aiSessionLog", + { + id: serial("id").primaryKey(), + sessionId: varchar("sessionId").notNull(), + orgId: varchar("orgId").references(() => orgs.orgId, { + onDelete: "cascade" + }), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + capability: varchar("capability").notNull(), + resourceId: integer("resourceId").references( + () => resources.resourceId, + { onDelete: "cascade" } + ), + siteResourceId: integer("siteResourceId").references( + () => siteResources.siteResourceId, + { onDelete: "cascade" } + ), + userId: varchar("userId").references(() => users.userId, { + onDelete: "set null" + }), + requestedModel: varchar("requestedModel"), + isStream: boolean("isStream").notNull().default(false), + requestBody: text("requestBody"), + responseBody: text("responseBody"), + // True if requestBody/responseBody were cut short at + // AI_SESSION_LOG_MAX_BODY_CHARS before storage. + truncated: boolean("truncated").notNull().default(false), + statusCode: integer("statusCode"), + createdAt: bigint("createdAt", { mode: "number" }).notNull() // epoch ms + }, + (t) => [ + index("idx_ai_session_log_org_created").on(t.orgId, t.createdAt), + index("idx_ai_session_log_org_provider_created").on( + t.orgId, + t.providerId, + t.createdAt + ), + index("idx_ai_session_log_org_resource_created").on( + t.orgId, + t.resourceId, + t.createdAt + ), + index("idx_ai_session_log_org_site_resource_created").on( + t.orgId, + t.siteResourceId, + t.createdAt + ), + index("idx_ai_session_log_org_user_created").on( + t.orgId, + t.userId, + t.createdAt + ), + index("idx_ai_session_log_session").on(t.sessionId) + ] +); + export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1992,6 +2061,7 @@ export type AiModel = InferSelectModel; export type AiBudget = InferSelectModel; export type AiUsageRecord = InferSelectModel; export type AiBudgetBreachEvent = InferSelectModel; +export type AiSessionLog = InferSelectModel; export type ResourceAiProvider = InferSelectModel; export type SiteResourceAiProvider = InferSelectModel< typeof siteResourceAiProviders diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 1c2236dfe..3fb15c0d6 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -64,6 +64,11 @@ export const orgs = sqliteTable("orgs", { ) // where 0 = dont keep logs and -1 = keep forever and 9001 = end of the following year .notNull() .default(0), + settingsLogRetentionDaysAISessions: integer( + "settingsLogRetentionDaysAISessions" + ) // where 0 = dont keep logs and -1 = keep forever and 9001 = end of the following year + .notNull() + .default(0), sshCaPrivateKey: text("sshCaPrivateKey"), // Encrypted SSH CA private key (PEM format) sshCaPublicKey: text("sshCaPublicKey"), // SSH CA public key (OpenSSH format) isBillingOrg: integer("isBillingOrg", { mode: "boolean" }), @@ -1889,6 +1894,74 @@ export const aiBudgetBreachEvents = sqliteTable( ] ); +// Logs the aggregated prompt + response for a single AI gateway request, for +// session replay. One row per request (not per streaming chunk). `sessionId` +// is a fresh random id per row for now - no cross-request correlation yet, +// but the column exists so a future pass can link multiple rows into a real +// multi-turn session. +export const aiSessionLog = sqliteTable( + "aiSessionLog", + { + id: integer("id").primaryKey({ autoIncrement: true }), + sessionId: text("sessionId").notNull(), + orgId: text("orgId").references(() => orgs.orgId, { + onDelete: "cascade" + }), + providerId: integer("providerId") + .notNull() + .references(() => aiProviders.providerId, { onDelete: "cascade" }), + capability: text("capability").notNull(), + resourceId: integer("resourceId").references( + () => resources.resourceId, + { onDelete: "cascade" } + ), + siteResourceId: integer("siteResourceId").references( + () => siteResources.siteResourceId, + { onDelete: "cascade" } + ), + userId: text("userId").references(() => users.userId, { + onDelete: "set null" + }), + requestedModel: text("requestedModel"), + isStream: integer("isStream", { mode: "boolean" }) + .notNull() + .default(false), + requestBody: text("requestBody"), + responseBody: text("responseBody"), + // True if requestBody/responseBody were cut short at + // AI_SESSION_LOG_MAX_BODY_CHARS before storage. + truncated: integer("truncated", { mode: "boolean" }) + .notNull() + .default(false), + statusCode: integer("statusCode"), + createdAt: integer("createdAt").notNull() // epoch ms + }, + (t) => [ + index("idx_ai_session_log_org_created").on(t.orgId, t.createdAt), + index("idx_ai_session_log_org_provider_created").on( + t.orgId, + t.providerId, + t.createdAt + ), + index("idx_ai_session_log_org_resource_created").on( + t.orgId, + t.resourceId, + t.createdAt + ), + index("idx_ai_session_log_org_site_resource_created").on( + t.orgId, + t.siteResourceId, + t.createdAt + ), + index("idx_ai_session_log_org_user_created").on( + t.orgId, + t.userId, + t.createdAt + ), + index("idx_ai_session_log_session").on(t.sessionId) + ] +); + export type Org = InferSelectModel; export type User = InferSelectModel; export type Site = InferSelectModel; @@ -1980,6 +2053,7 @@ export type AiModel = InferSelectModel; export type AiBudget = InferSelectModel; export type AiUsageRecord = InferSelectModel; export type AiBudgetBreachEvent = InferSelectModel; +export type AiSessionLog = InferSelectModel; export type ResourceAiProvider = InferSelectModel; export type SiteResourceAiProvider = InferSelectModel< typeof siteResourceAiProviders diff --git a/server/lib/cleanupLogs.ts b/server/lib/cleanupLogs.ts index f5b6d8b2f..ae2e0cdb7 100644 --- a/server/lib/cleanupLogs.ts +++ b/server/lib/cleanupLogs.ts @@ -3,12 +3,14 @@ import { cleanUpOldLogs as cleanUpOldAccessLogs } from "#dynamic/lib/logAccessAu import { cleanUpOldLogs as cleanUpOldActionLogs } from "#dynamic/middlewares/logActionAudit"; import { cleanUpOldLogs as cleanUpOldRequestLogs } from "@server/routers/badger/logRequestAudit"; import { cleanUpOldLogs as cleanUpOldConnectionLogs } from "#dynamic/routers/newt"; +import { cleanUpOldLogs as cleanUpOldAiSessionLogs } from "@server/routers/aiGateway/logAiSession"; import { gt, or } from "drizzle-orm"; import { cleanUpOldFingerprintSnapshots } from "@server/routers/olm/fingerprintingUtils"; import { build } from "@server/build"; export function initLogCleanupInterval() { - if (build == "saas") { // skip log cleanup for saas builds + if (build == "saas") { + // skip log cleanup for saas builds return null; } return setInterval( @@ -23,7 +25,9 @@ export function initLogCleanupInterval() { settingsLogRetentionDaysRequest: orgs.settingsLogRetentionDaysRequest, settingsLogRetentionDaysConnection: - orgs.settingsLogRetentionDaysConnection + orgs.settingsLogRetentionDaysConnection, + settingsLogRetentionDaysAISessions: + orgs.settingsLogRetentionDaysAISessions }) .from(orgs) .where( @@ -31,7 +35,8 @@ export function initLogCleanupInterval() { gt(orgs.settingsLogRetentionDaysAction, 0), gt(orgs.settingsLogRetentionDaysAccess, 0), gt(orgs.settingsLogRetentionDaysRequest, 0), - gt(orgs.settingsLogRetentionDaysConnection, 0) + gt(orgs.settingsLogRetentionDaysConnection, 0), + gt(orgs.settingsLogRetentionDaysAISessions, 0) ) ); @@ -42,7 +47,8 @@ export function initLogCleanupInterval() { settingsLogRetentionDaysAction, settingsLogRetentionDaysAccess, settingsLogRetentionDaysRequest, - settingsLogRetentionDaysConnection + settingsLogRetentionDaysConnection, + settingsLogRetentionDaysAISessions } = org; if (settingsLogRetentionDaysAction > 0) { @@ -72,6 +78,13 @@ export function initLogCleanupInterval() { settingsLogRetentionDaysConnection ); } + + if (settingsLogRetentionDaysAISessions > 0) { + await cleanUpOldAiSessionLogs( + orgId, + settingsLogRetentionDaysAISessions + ); + } } await cleanUpOldFingerprintSnapshots(365); diff --git a/server/private/cleanup.ts b/server/private/cleanup.ts index a1ad24976..c949e4e0c 100644 --- a/server/private/cleanup.ts +++ b/server/private/cleanup.ts @@ -19,6 +19,7 @@ import { flushConnectionLogToDb } from "#private/routers/newt"; import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth"; import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator"; import { shutdownUsageRecorder } from "@server/lib/aiBudgetEnforcement"; +import { shutdownAiSessionLogger } from "@server/routers/aiGateway/logAiSession"; async function cleanup() { await stopPingAccumulator(); @@ -26,6 +27,7 @@ async function cleanup() { await flushConnectionLogToDb(); await flushSiteBandwidthToDb(); await shutdownUsageRecorder(); + await shutdownAiSessionLogger(); await rateLimitService.cleanup(); await wsCleanup(); await logStreamingManager.shutdown(); diff --git a/server/private/routers/billing/featureLifecycle.ts b/server/private/routers/billing/featureLifecycle.ts index 75d11c756..84a7b4f5a 100644 --- a/server/private/routers/billing/featureLifecycle.ts +++ b/server/private/routers/billing/featureLifecycle.ts @@ -134,6 +134,17 @@ async function capRetentionDays( ); } + if ( + org.settingsLogRetentionDaysAISessions !== null && + org.settingsLogRetentionDaysAISessions > maxRetentionDays + ) { + updates.settingsLogRetentionDaysAISessions = maxRetentionDays; + needsUpdate = true; + logger.info( + `Capping AI session log retention from ${org.settingsLogRetentionDaysAISessions} to ${maxRetentionDays} days for org ${orgId}` + ); + } + // Apply updates if needed if (needsUpdate) { await db.update(orgs).set(updates).where(eq(orgs.orgId, orgId)); diff --git a/server/routers/aiGateway/logAiSession.ts b/server/routers/aiGateway/logAiSession.ts new file mode 100644 index 000000000..4c7384497 --- /dev/null +++ b/server/routers/aiGateway/logAiSession.ts @@ -0,0 +1,238 @@ +import { randomUUID } from "crypto"; +import { logsDb, db, orgs, aiSessionLog, type AiProvider } from "@server/db"; +import type { InferInsertModel } from "drizzle-orm"; +import logger from "@server/logger"; +import { and, eq, lt } from "drizzle-orm"; +import cache from "#dynamic/lib/cache"; +import { calculateCutoffTimestamp } from "@server/lib/cleanupLogs"; +import { sanitizeString } from "@server/lib/sanitize"; +import type { AiCapability } from "@server/lib/aiCapabilities"; + +// Caps how much of the request/response body we keep per row, so a single +// huge multimodal payload can't blow up buffer memory or storage. +const AI_SESSION_LOG_MAX_BODY_CHARS = 200_000; + +type AiSessionLogInsert = InferInsertModel; + +// In-memory buffer for batching AI session log inserts, mirroring the +// approach in server/routers/badger/logRequestAudit.ts. +const sessionLogBuffer: AiSessionLogInsert[] = []; + +const BATCH_SIZE = 100; // Write to DB every 100 logs +const BATCH_INTERVAL_MS = 5000; // Or every 5 seconds, whichever comes first +const MAX_BUFFER_SIZE = 10000; // Prevent unbounded memory growth +let flushTimer: NodeJS.Timeout | null = null; +let isFlushInProgress = false; + +/** + * Flush buffered logs to database + */ +async function flushSessionLogs() { + if (sessionLogBuffer.length === 0 || isFlushInProgress) { + return; + } + + isFlushInProgress = true; + + // Take all current logs and clear buffer + const logsToWrite = sessionLogBuffer.splice(0, sessionLogBuffer.length); + + try { + // Use a transaction to ensure all inserts succeed or fail together + await logsDb.transaction(async (tx) => { + // Batch insert logs in groups of 25 to avoid overwhelming the database + const BATCH_DB_SIZE = 25; + for (let i = 0; i < logsToWrite.length; i += BATCH_DB_SIZE) { + const batch = logsToWrite.slice(i, i + BATCH_DB_SIZE); + await tx.insert(aiSessionLog).values(batch); + } + }); + logger.debug( + `Flushed ${logsToWrite.length} AI session logs to database` + ); + } catch (error) { + logger.error("Error flushing AI session logs:", error); + // On transaction error, put logs back at the front of the buffer to retry + // but only if buffer isn't too large + if (sessionLogBuffer.length < MAX_BUFFER_SIZE - logsToWrite.length) { + sessionLogBuffer.unshift(...logsToWrite); + logger.info( + `Re-queued ${logsToWrite.length} AI session logs for retry` + ); + } else { + logger.error( + `Buffer full, dropped ${logsToWrite.length} AI session logs` + ); + } + } finally { + isFlushInProgress = false; + // If buffer filled up while we were flushing, flush again + if (sessionLogBuffer.length >= BATCH_SIZE) { + flushSessionLogs().catch((err) => + logger.error("Error in follow-up AI session log flush:", err) + ); + } + } +} + +/** + * Schedule a flush if not already scheduled + */ +function scheduleFlush() { + if (flushTimer === null) { + flushTimer = setTimeout(() => { + flushTimer = null; + flushSessionLogs().catch((err) => + logger.error("Error in scheduled AI session log flush:", err) + ); + }, BATCH_INTERVAL_MS); + } +} + +/** + * Gracefully flush all pending logs (call this on shutdown) + */ +export async function shutdownAiSessionLogger() { + if (flushTimer) { + clearTimeout(flushTimer); + flushTimer = null; + } + // Force flush even if one is in progress by waiting and retrying + while (isFlushInProgress) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + await flushSessionLogs(); +} + +async function getRetentionDays(orgId: string): Promise { + // check cache first + const cached = await cache.get(`org_${orgId}_aiSessionsDays`); + if (cached !== undefined) { + return cached; + } + + const [org] = await db + .select({ + settingsLogRetentionDaysAISessions: + orgs.settingsLogRetentionDaysAISessions + }) + .from(orgs) + .where(eq(orgs.orgId, orgId)) + .limit(1); + + if (!org) { + return 0; + } + + // store the result in cache + await cache.set( + `org_${orgId}_aiSessionsDays`, + org.settingsLogRetentionDaysAISessions, + 300 + ); + + return org.settingsLogRetentionDaysAISessions; +} + +export async function cleanUpOldLogs(orgId: string, retentionDays: number) { + // calculateCutoffTimestamp returns a seconds-epoch cutoff (built for + // requestAuditLog.timestamp), but aiSessionLog.createdAt is ms-epoch to + // match aiUsageRecords - convert before comparing. + const cutoffTimestampMs = calculateCutoffTimestamp(retentionDays) * 1000; + + try { + await logsDb + .delete(aiSessionLog) + .where( + and( + lt(aiSessionLog.createdAt, cutoffTimestampMs), + eq(aiSessionLog.orgId, orgId) + ) + ); + } catch (error) { + logger.error("Error cleaning up old AI session logs:", error); + } +} + +function truncateBody(value: string): { value: string; truncated: boolean } { + if (value.length <= AI_SESSION_LOG_MAX_BODY_CHARS) { + return { value, truncated: false }; + } + return { + value: value.slice(0, AI_SESSION_LOG_MAX_BODY_CHARS), + truncated: true + }; +} + +export function logAiSession(data: { + capability: AiCapability; + provider: AiProvider; + requestedModel: string | undefined; + requestBody: unknown; + responseText: string; + isStream: boolean; + statusCode: number; + orgId: string | null; + resourceId: number | null; + siteResourceId: number | null; + requestUserId: string | null; +}): void { + (async () => { + try { + // Check retention before buffering any logs + if (data.orgId) { + const retentionDays = await getRetentionDays(data.orgId); + if (retentionDays === 0) { + // do not log + return; + } + } else { + // No org resolved for this request - nothing to govern + // retention with, so don't log it. + return; + } + + const requestBodyText = truncateBody( + JSON.stringify(data.requestBody ?? "") + ); + const responseBodyText = truncateBody(data.responseText ?? ""); + + // Prevent unbounded buffer growth - drop oldest entries if buffer is too large + if (sessionLogBuffer.length >= MAX_BUFFER_SIZE) { + const dropped = sessionLogBuffer.splice(0, BATCH_SIZE); + logger.warn( + `AI session log buffer exceeded max size (${MAX_BUFFER_SIZE}), dropped ${dropped.length} oldest entries` + ); + } + + sessionLogBuffer.push({ + sessionId: randomUUID(), + orgId: sanitizeString(data.orgId), + providerId: data.provider.providerId, + capability: data.capability, + resourceId: data.resourceId ?? undefined, + siteResourceId: data.siteResourceId ?? undefined, + userId: sanitizeString(data.requestUserId ?? undefined), + requestedModel: sanitizeString(data.requestedModel), + isStream: data.isStream, + requestBody: sanitizeString(requestBodyText.value), + responseBody: sanitizeString(responseBodyText.value), + truncated: + requestBodyText.truncated || responseBodyText.truncated, + statusCode: data.statusCode, + createdAt: Date.now() + }); + + // Flush immediately if buffer is full, otherwise schedule a flush + if (sessionLogBuffer.length >= BATCH_SIZE) { + flushSessionLogs().catch((err) => + logger.error("Error flushing AI session logs:", err) + ); + } else { + scheduleFlush(); + } + } catch (error) { + logger.error("Failed to log AI session", { error }); + } + })(); +} diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 22274e049..9e1433832 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -63,10 +63,11 @@ import { isUsageEmpty, needsStreamUsageInjection, withStreamUsageOption, - stripInjectedUsageFrame, extractResponseModel, type AiUsage } from "@server/lib/aiUsageExtraction"; +import { streamAiGatewayResponse } from "@server/routers/aiGateway/streamAiGatewayResponse"; +import { logAiSession } from "@server/routers/aiGateway/logAiSession"; const EXIT_NODE_RANGES_CACHE_KEY = "aiGateway:exitNodeRanges"; const EXIT_NODE_RANGES_TTL_SEC = 6000; @@ -526,13 +527,20 @@ async function selectProvider( }; } -function logAiUsageAndCost(args: { +// Extracts usage/cost from a completed AI gateway request, records it for +// budget enforcement, and logs the aggregated prompt/response for session +// replay. Shared by both the direct-upstream path (below) and the +// "custom"/target routing-mode path (targetRouting.ts) so both get identical +// usage/cost tracking and session logging instead of only the direct-upstream +// path having it. +export function recordAiGatewayCompletion(args: { capability: AiCapability; provider: AiProvider; requestedModel: string | undefined; requestBody: unknown; responseText: string; isStream: boolean; + statusCode: number; headers: Headers; orgId: string | null; resourceId: number | null; @@ -547,6 +555,7 @@ function logAiUsageAndCost(args: { requestBody, responseText, isStream, + statusCode, headers, orgId, resourceId, @@ -608,6 +617,20 @@ function logAiUsageAndCost(args: { }); } } + + logAiSession({ + capability, + provider, + requestedModel, + requestBody, + responseText, + isStream, + statusCode, + orgId, + resourceId, + siteResourceId, + requestUserId + }); } export async function handleAiGatewayProxy( @@ -722,7 +745,14 @@ export async function handleAiGatewayProxy( res, provider, requestUser, - capability + capability, + { + orgId, + resourceId, + siteResourceId, + requestedModel, + budgets: appliedBudgets + } ); } @@ -843,84 +873,38 @@ export async function handleAiGatewayProxy( throw fetchError; } - const contentType = upstreamRes.headers.get("content-type") || ""; - const isStream = def.isStreaming(req, contentType); + const isStream = def.isStreaming( + req, + upstreamRes.headers.get("content-type") || "" + ); - res.status(upstreamRes.status); - res.setHeader("Content-Type", contentType || "application/json"); - - if (isStream && upstreamRes.body) { - res.flushHeaders(); - const reader = upstreamRes.body.getReader(); - const decoder = new TextDecoder(); - let fullText = ""; - // Frame-boundary buffer, only used when we need to filter the - // usage-only frame we injected out of what reaches the client. - let sseCarry = ""; - try { - while (!abortController.signal.aborted) { - const { done, value } = await reader.read(); - if (done) break; - const chunkText = decoder.decode(value, { stream: true }); - fullText += chunkText; - if (injectedUsageOurselves) { - sseCarry += chunkText; - const lastBoundary = sseCarry.lastIndexOf("\n\n"); - if (lastBoundary !== -1) { - const toEmit = sseCarry.slice(0, lastBoundary + 2); - sseCarry = sseCarry.slice(lastBoundary + 2); - res.write(stripInjectedUsageFrame(toEmit)); - } - } else { - res.write(value); - } - } - if (injectedUsageOurselves && sseCarry) { - res.write(stripInjectedUsageFrame(sseCarry)); - } - } finally { - await reader.cancel().catch(() => {}); - res.off("close", onClientClose); - } - if (!res.writableEnded) { - res.end(); - } - if (!abortController.signal.aborted) { - logAiUsageAndCost({ - capability, - provider, - requestedModel, - requestBody: req.body, - responseText: fullText, - isStream: true, - headers: upstreamRes.headers, - orgId, - resourceId, - siteResourceId, - requestUserId: requestUser?.userId ?? null, - budgets: appliedBudgets - }); - } - return; - } - - res.off("close", onClientClose); - const text = await upstreamRes.text(); - logAiUsageAndCost({ - capability, - provider, - requestedModel, - requestBody: req.body, - responseText: text, - isStream: false, - headers: upstreamRes.headers, - orgId, - resourceId, - siteResourceId, - requestUserId: requestUser?.userId ?? null, - budgets: appliedBudgets + const { fullText, aborted } = await streamAiGatewayResponse({ + res, + upstreamRes, + isStream, + injectedUsageOurselves, + abortController, + onClientClose }); - return res.send(text); + + if (!aborted) { + recordAiGatewayCompletion({ + capability, + provider, + requestedModel, + requestBody: outboundBody, + responseText: fullText, + isStream, + statusCode: upstreamRes.status, + headers: upstreamRes.headers, + orgId, + resourceId, + siteResourceId, + requestUserId: requestUser?.userId ?? null, + budgets: appliedBudgets + }); + } + return; } catch (error) { logger.error(error); return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ diff --git a/server/routers/aiGateway/streamAiGatewayResponse.ts b/server/routers/aiGateway/streamAiGatewayResponse.ts new file mode 100644 index 000000000..8cd369350 --- /dev/null +++ b/server/routers/aiGateway/streamAiGatewayResponse.ts @@ -0,0 +1,86 @@ +import { Response } from "express"; +import { stripInjectedUsageFrame } from "@server/lib/aiUsageExtraction"; + +/** + * Reads an upstream AI provider response, writes it through to the client + * (streaming or buffered), and returns the full response text once done, so + * the caller can extract usage/cost and log the completed session. Shared by + * both the direct-upstream path (pipeline.ts) and the "custom"/target + * routing-mode path (targetRouting.ts) so usage/cost tracking and session + * logging apply identically to both instead of each maintaining its own copy + * of this loop. + * + * Callers own fetching the upstream response and the AbortController/ + * `res.on("close", onClientClose)` wiring, since those differ meaningfully + * between the two transports (direct upstream fetch with TLS-skip support vs + * a plain fetch to gerbil) - only the "read the stream, write to the client, + * accumulate the full text" part is actually identical logic between them. + */ +export async function streamAiGatewayResponse(args: { + res: Response; + upstreamRes: globalThis.Response; + isStream: boolean; + // True when we injected stream_options.include_usage ourselves (the + // caller didn't ask for it) and need to strip the extra usage-only frame + // back out of what's forwarded to the client. + injectedUsageOurselves: boolean; + abortController: AbortController; + onClientClose: () => void; +}): Promise<{ fullText: string; aborted: boolean }> { + const { + res, + upstreamRes, + isStream, + injectedUsageOurselves, + abortController, + onClientClose + } = args; + + const contentType = upstreamRes.headers.get("content-type") || ""; + res.status(upstreamRes.status); + res.setHeader("Content-Type", contentType || "application/json"); + + if (isStream && upstreamRes.body) { + res.flushHeaders(); + const reader = upstreamRes.body.getReader(); + const decoder = new TextDecoder(); + let fullText = ""; + // Frame-boundary buffer, only used when we need to filter the + // usage-only frame we injected out of what reaches the client. + let sseCarry = ""; + try { + while (!abortController.signal.aborted) { + const { done, value } = await reader.read(); + if (done) break; + const chunkText = decoder.decode(value, { stream: true }); + fullText += chunkText; + if (injectedUsageOurselves) { + sseCarry += chunkText; + const lastBoundary = sseCarry.lastIndexOf("\n\n"); + if (lastBoundary !== -1) { + const toEmit = sseCarry.slice(0, lastBoundary + 2); + sseCarry = sseCarry.slice(lastBoundary + 2); + res.write(stripInjectedUsageFrame(toEmit)); + } + } else { + res.write(value); + } + } + if (injectedUsageOurselves && sseCarry) { + res.write(stripInjectedUsageFrame(sseCarry)); + } + } finally { + await reader.cancel().catch(() => {}); + res.off("close", onClientClose); + } + if (!res.writableEnded) { + res.end(); + } + return { fullText, aborted: abortController.signal.aborted }; + } + + res.off("close", onClientClose); + const text = await upstreamRes.text(); + res.send(text); + return { fullText: text, aborted: abortController.signal.aborted }; +} diff --git a/server/routers/aiGateway/targetRouting.ts b/server/routers/aiGateway/targetRouting.ts index a6d08d925..00ef39d12 100644 --- a/server/routers/aiGateway/targetRouting.ts +++ b/server/routers/aiGateway/targetRouting.ts @@ -1,6 +1,13 @@ import { Request, Response } from "express"; import { and, eq } from "drizzle-orm"; -import { AiProvider, db, exitNodes, sites, targets } from "@server/db"; +import { + AiBudget, + AiProvider, + db, + exitNodes, + sites, + targets +} from "@server/db"; import config from "@server/lib/config"; import { decrypt } from "@server/lib/crypto"; import { localCache } from "@server/lib/cache"; @@ -14,12 +21,18 @@ import { AI_CAPABILITY_DEFS, type AiCapability } from "@server/lib/aiCapabilities"; +import { + needsStreamUsageInjection, + withStreamUsageOption +} from "@server/lib/aiUsageExtraction"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; import { applyRequestUserHeaders, + recordAiGatewayCompletion, type RequestUser } from "@server/routers/aiGateway/pipeline"; +import { streamAiGatewayResponse } from "@server/routers/aiGateway/streamAiGatewayResponse"; // Short TTL: long enough to spare the DB on a burst of requests, short // enough that target/site changes (added, removed, exit node moved) show up @@ -157,7 +170,14 @@ export async function proxyAiGatewayToSiteTarget( res: Response, provider: AiProvider, requestUser: RequestUser | null, - capability: AiCapability + capability: AiCapability, + ctx: { + orgId: string | null; + resourceId: number | null; + siteResourceId: number | null; + requestedModel: string | undefined; + budgets: AiBudget[]; + } ): Promise { const providerTargets = await getProviderTargets(provider.providerId); if (providerTargets.length === 0) { @@ -205,7 +225,17 @@ export async function proxyAiGatewayToSiteTarget( headers[PANGOLIN_DEST_HEADER] = target.destination; headers[PANGOLIN_HOST_HEADER] = target.hostHeader; - const body = JSON.stringify(req.body); + // Same OpenAI stream_options.include_usage injection direct-upstream + // requests get (pipeline.ts) - needed here too now that target-routed + // requests get usage/cost tracking and session logging as well. + const injectedUsageOurselves = needsStreamUsageInjection( + capability, + req.body + ); + const outboundBody = injectedUsageOurselves + ? withStreamUsageOption(req.body) + : req.body; + const body = JSON.stringify(outboundBody); logger.debug("AI gateway target-routed request", { providerId: provider.providerId, @@ -214,7 +244,7 @@ export async function proxyAiGatewayToSiteTarget( hostHeader: target.hostHeader, url: gerbilUrl, headers, - body: req.body + body: outboundBody }); // Cancel the request to gerbil (which cascades to gerbil cancelling its @@ -259,35 +289,35 @@ export async function proxyAiGatewayToSiteTarget( return; } - const contentType = upstreamRes.headers.get("content-type") || ""; const isStream = AI_CAPABILITY_DEFS[capability].isStreaming( req, - contentType + upstreamRes.headers.get("content-type") || "" ); - res.status(upstreamRes.status); - res.setHeader("Content-Type", contentType || "application/json"); + const { fullText, aborted } = await streamAiGatewayResponse({ + res, + upstreamRes, + isStream, + injectedUsageOurselves, + abortController, + onClientClose + }); - if (isStream && upstreamRes.body) { - res.flushHeaders(); - const reader = upstreamRes.body.getReader(); - try { - while (!abortController.signal.aborted) { - const { done, value } = await reader.read(); - if (done) break; - res.write(value); - } - } finally { - await reader.cancel().catch(() => {}); - res.off("close", onClientClose); - } - if (!res.writableEnded) { - res.end(); - } - return; + if (!aborted) { + recordAiGatewayCompletion({ + capability, + provider, + requestedModel: ctx.requestedModel, + requestBody: outboundBody, + responseText: fullText, + isStream, + statusCode: upstreamRes.status, + headers: upstreamRes.headers, + orgId: ctx.orgId, + resourceId: ctx.resourceId, + siteResourceId: ctx.siteResourceId, + requestUserId: requestUser?.userId ?? null, + budgets: ctx.budgets + }); } - - res.off("close", onClientClose); - const text = await upstreamRes.text(); - res.send(text); } diff --git a/server/routers/org/updateOrg.ts b/server/routers/org/updateOrg.ts index f98bdec27..021448375 100644 --- a/server/routers/org/updateOrg.ts +++ b/server/routers/org/updateOrg.ts @@ -41,6 +41,10 @@ const updateOrgBodySchema = z .number() .min(build === "saas" ? 0 : -1) .optional(), + settingsLogRetentionDaysAISessions: z + .number() + .min(build === "saas" ? 0 : -1) + .optional(), settingsEnableGlobalNewtAutoUpdate: z.boolean().optional() }) .refine((data) => Object.keys(data).length > 0, { @@ -212,6 +216,19 @@ export async function updateOrg( ) ); } + if ( + parsedBody.data.settingsLogRetentionDaysAISessions !== + undefined && + parsedBody.data.settingsLogRetentionDaysAISessions > + maxRetentionDays + ) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + `You are not allowed to set log retention days greater than ${maxRetentionDays} with your current subscription` + ) + ); + } } } @@ -230,6 +247,8 @@ export async function updateOrg( parsedBody.data.settingsLogRetentionDaysAction, settingsLogRetentionDaysConnection: parsedBody.data.settingsLogRetentionDaysConnection, + settingsLogRetentionDaysAISessions: + parsedBody.data.settingsLogRetentionDaysAISessions, settingsEnableGlobalNewtAutoUpdate: parsedBody.data.settingsEnableGlobalNewtAutoUpdate }) @@ -250,6 +269,7 @@ export async function updateOrg( await cache.del(`org_${orgId}_actionDays`); await cache.del(`org_${orgId}_accessDays`); await cache.del(`org_${orgId}_connectionDays`); + await cache.del(`org_${orgId}_aiSessionsDays`); return response(res, { data: updatedOrg[0], diff --git a/src/app/[orgId]/settings/general/security/page.tsx b/src/app/[orgId]/settings/general/security/page.tsx index 51afa6077..7f83711b5 100644 --- a/src/app/[orgId]/settings/general/security/page.tsx +++ b/src/app/[orgId]/settings/general/security/page.tsx @@ -80,7 +80,8 @@ const SecurityFormSchema = z.object({ settingsLogRetentionDaysRequest: z.number(), settingsLogRetentionDaysAccess: z.number(), settingsLogRetentionDaysAction: z.number(), - settingsLogRetentionDaysConnection: z.number() + settingsLogRetentionDaysConnection: z.number(), + settingsLogRetentionDaysAISessions: z.number() }); const LOG_RETENTION_OPTIONS = [ @@ -122,7 +123,8 @@ function LogRetentionSectionForm({ org }: SectionFormProps) { settingsLogRetentionDaysRequest: true, settingsLogRetentionDaysAccess: true, settingsLogRetentionDaysAction: true, - settingsLogRetentionDaysConnection: true + settingsLogRetentionDaysConnection: true, + settingsLogRetentionDaysAISessions: true }) ), defaultValues: { @@ -133,7 +135,9 @@ function LogRetentionSectionForm({ org }: SectionFormProps) { settingsLogRetentionDaysAction: org.settingsLogRetentionDaysAction ?? 15, settingsLogRetentionDaysConnection: - org.settingsLogRetentionDaysConnection ?? 15 + org.settingsLogRetentionDaysConnection ?? 15, + settingsLogRetentionDaysAISessions: + org.settingsLogRetentionDaysAISessions ?? 15 }, mode: "onChange" }); @@ -161,7 +165,9 @@ function LogRetentionSectionForm({ org }: SectionFormProps) { settingsLogRetentionDaysAction: data.settingsLogRetentionDaysAction, settingsLogRetentionDaysConnection: - data.settingsLogRetentionDaysConnection + data.settingsLogRetentionDaysConnection, + settingsLogRetentionDaysAISessions: + data.settingsLogRetentionDaysAISessions } as any; // Update organization @@ -292,6 +298,101 @@ function LogRetentionSectionForm({ org }: SectionFormProps) { )} /> + ( + + + {t("logRetentionAISessionsLabel")} + + + + + + + )} + /> + {!env.flags.disableEnterpriseFeatures && ( <> Date: Tue, 11 Aug 2026 14:05:58 -0400 Subject: [PATCH 115/296] quiet up some more logs --- server/lib/cache.ts | 18 +++++++++--------- server/private/lib/cache.ts | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/server/lib/cache.ts b/server/lib/cache.ts index 4e2b7c782..1aece08ac 100644 --- a/server/lib/cache.ts +++ b/server/lib/cache.ts @@ -10,12 +10,12 @@ export const localCache = new NodeCache({ }); // Log cache statistics periodically for monitoring -setInterval(() => { - const stats = localCache.getStats(); - logger.debug( - `Local cache stats - Keys: ${stats.keys}, Hits: ${stats.hits}, Misses: ${stats.misses}, Hit rate: ${stats.hits > 0 ? ((stats.hits / (stats.hits + stats.misses)) * 100).toFixed(2) : 0}%` - ); -}, 300000); // Every 5 minutes +// setInterval(() => { +// const stats = localCache.getStats(); +// logger.debug( +// `Local cache stats - Keys: ${stats.keys}, Hits: ${stats.hits}, Misses: ${stats.misses}, Hit rate: ${stats.hits > 0 ? ((stats.hits / (stats.hits + stats.misses)) * 100).toFixed(2) : 0}%` +// ); +// }, 300000); // Every 5 minutes /** * Adaptive cache that uses Redis when available in multi-node environments, @@ -34,9 +34,9 @@ class AdaptiveCache { // Use local cache as fallback or primary const success = localCache.set(key, value, effectiveTtl || 0); - if (success) { - logger.debug(`Set key in local cache: ${key}`); - } + // if (success) { + // logger.debug(`Set key in local cache: ${key}`); + // } return success; } diff --git a/server/private/lib/cache.ts b/server/private/lib/cache.ts index fe7b771c9..ad2710278 100644 --- a/server/private/lib/cache.ts +++ b/server/private/lib/cache.ts @@ -24,12 +24,12 @@ export const localCache = new NodeCache({ }); // Log cache statistics periodically for monitoring -setInterval(() => { - const stats = localCache.getStats(); - logger.debug( - `Local cache stats - Keys: ${stats.keys}, Hits: ${stats.hits}, Misses: ${stats.misses}, Hit rate: ${stats.hits > 0 ? ((stats.hits / (stats.hits + stats.misses)) * 100).toFixed(2) : 0}%` - ); -}, 300000); // Every 5 minutes +// setInterval(() => { +// const stats = localCache.getStats(); +// logger.debug( +// `Local cache stats - Keys: ${stats.keys}, Hits: ${stats.hits}, Misses: ${stats.misses}, Hit rate: ${stats.hits > 0 ? ((stats.hits / (stats.hits + stats.misses)) * 100).toFixed(2) : 0}%` +// ); +// }, 300000); // Every 5 minutes /** * Adaptive cache that uses Redis when available in multi-node environments, @@ -80,9 +80,9 @@ class AdaptiveCache { // Use local cache as fallback or primary const success = localCache.set(key, value, effectiveTtl || 0); - if (success) { - logger.debug(`Set key in local cache: ${key}`); - } + // if (success) { + // logger.debug(`Set key in local cache: ${key}`); + // } return success; } From e0a66e79bba032c4d5856beafd61abbae6a853e6 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 14:07:33 -0400 Subject: [PATCH 116/296] Continue to quiet log messages --- server/private/lib/cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/private/lib/cache.ts b/server/private/lib/cache.ts index ad2710278..7ddfeb554 100644 --- a/server/private/lib/cache.ts +++ b/server/private/lib/cache.ts @@ -344,7 +344,7 @@ class RegionalAdaptiveCache { } const success = regionalLocalCache.set(key, value, effectiveTtl || 0); - if (success) logger.debug(`[regional] Set key in local cache: ${key}`); + // if (success) logger.debug(`[regional] Set key in local cache: ${key}`); return success; } From 98f5e39a7f9a6d72ce36076c9e2749387d562dad Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 11 Aug 2026 14:41:07 -0400 Subject: [PATCH 117/296] show ai gateway resource details in launcher --- messages/en-US.json | 11 + server/lib/aiInferenceResource.ts | 146 ++++++++++++ server/routers/external.ts | 14 ++ .../routers/launcher/formatLauncherAccess.ts | 23 +- server/routers/launcher/index.ts | 5 + .../launcher/launcherResourceAccess.ts | 143 +++++++---- .../routers/launcher/listLauncherAiModels.ts | 172 ++++++++++++++ server/routers/launcher/types.ts | 1 + .../resources/private/create/page.tsx | 91 +++---- src/components/PrivateResourceInfoBox.tsx | 8 +- .../LauncherGroupSection.tsx | 9 +- .../LauncherGroupTrigger.tsx | 17 +- .../LauncherInferenceApiKeysSection.tsx | 223 ++++++++++++++++++ .../LauncherInferenceModelsSection.tsx | 170 +++++++++++++ .../LauncherResourcePanel.tsx | 68 +++++- src/lib/launcherResourceAccess.ts | 20 +- src/lib/launcherResourceDetails.ts | 6 +- src/lib/queries.ts | 66 ++++++ 18 files changed, 1069 insertions(+), 124 deletions(-) create mode 100644 server/routers/launcher/listLauncherAiModels.ts create mode 100644 src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx create mode 100644 src/components/resource-launcher/LauncherInferenceModelsSection.tsx diff --git a/messages/en-US.json b/messages/en-US.json index c7db842a7..0eaf60378 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -4053,7 +4053,18 @@ "resourceLauncherTcp": "TCP", "resourceLauncherUdp": "UDP", "resourceLauncherUnlabeled": "Unlabeled", + "resourceLauncherAiGateway": "AI Gateway", "resourceLauncherNoSite": "No Site", + "resourceLauncherAvailableModels": "Available Models", + "resourceLauncherAvailableModelsDescription": "Models you can use with this inference resource.", + "resourceLauncherAvailableModelsEmpty": "No models are available for this resource.", + "resourceLauncherAvailableModelsError": "Could not load available models.", + "resourceLauncherApiKeys": "API Keys", + "resourceLauncherApiKeysDescription": "Use your identity key or an attributed key to authenticate with this resource.", + "resourceLauncherApiKeysIdentity": "Identity Key", + "resourceLauncherApiKeysManual": "Attributed Keys", + "resourceLauncherApiKeysEmpty": "No API keys are available for this resource.", + "resourceLauncherApiKeysError": "Could not load API keys.", "resourceLauncherNoResourcesInGroup": "No resources in this group", "resourceLauncherEmptyStateTitle": "No Resources Available", "resourceLauncherEmptyStateDescription": "You don't have access to any resources yet. Contact your administrator to request access.", diff --git a/server/lib/aiInferenceResource.ts b/server/lib/aiInferenceResource.ts index 92a438ac2..cfd560fce 100644 --- a/server/lib/aiInferenceResource.ts +++ b/server/lib/aiInferenceResource.ts @@ -518,6 +518,152 @@ export async function listSiteResourceAiProviders(siteResourceId: number) { .where(eq(siteResourceAiProviders.siteResourceId, siteResourceId)); } +export type EffectiveAllowModel = { + modelId: number; + modelKey: string; + name: string; + providerId: number; + providerName: string; +}; + +export async function listEffectiveAllowModels(options: { + resourceId?: number; + siteResourceId?: number; +}): Promise { + if ( + options.resourceId === undefined && + options.siteResourceId === undefined + ) { + return []; + } + + const attachments = + options.resourceId !== undefined + ? await listPublicResourceAiProviders(options.resourceId) + : await listSiteResourceAiProviders(options.siteResourceId!); + + const activeAttachments = attachments.filter( + (a) => a.enabled && a.providerEnabled + ); + if (activeAttachments.length === 0) { + return []; + } + + const inheritProviderIds = activeAttachments + .filter((a) => a.accessMode === "inherit") + .map((a) => a.providerId); + const selectProviderIds = activeAttachments + .filter((a) => a.accessMode === "select") + .map((a) => a.providerId); + + const providerNameById = new Map( + activeAttachments.map((a) => [a.providerId, a.name] as const) + ); + + const models: EffectiveAllowModel[] = []; + + if (inheritProviderIds.length > 0) { + const rows = await db + .select({ + modelId: aiModels.modelId, + modelKey: aiModels.modelKey, + name: aiModels.name, + providerId: aiModels.providerId + }) + .from(aiModels) + .where( + and( + inArray(aiModels.providerId, inheritProviderIds), + eq(aiModels.enabled, true), + eq(aiModels.listType, "allow") + ) + ); + for (const row of rows) { + models.push({ + ...row, + providerName: providerNameById.get(row.providerId) ?? "" + }); + } + } + + if (selectProviderIds.length > 0) { + if (options.resourceId !== undefined) { + const rows = await db + .select({ + modelId: aiModels.modelId, + modelKey: aiModels.modelKey, + name: aiModels.name, + providerId: aiModels.providerId + }) + .from(resourceAiModels) + .innerJoin( + aiModels, + eq(resourceAiModels.modelId, aiModels.modelId) + ) + .where( + and( + eq(resourceAiModels.resourceId, options.resourceId), + inArray(aiModels.providerId, selectProviderIds), + eq(resourceAiModels.listType, "allow"), + eq(aiModels.enabled, true) + ) + ); + for (const row of rows) { + models.push({ + ...row, + providerName: providerNameById.get(row.providerId) ?? "" + }); + } + } else if (options.siteResourceId !== undefined) { + const rows = await db + .select({ + modelId: aiModels.modelId, + modelKey: aiModels.modelKey, + name: aiModels.name, + providerId: aiModels.providerId + }) + .from(siteResourceAiModels) + .innerJoin( + aiModels, + eq(siteResourceAiModels.modelId, aiModels.modelId) + ) + .where( + and( + eq( + siteResourceAiModels.siteResourceId, + options.siteResourceId + ), + inArray(aiModels.providerId, selectProviderIds), + eq(siteResourceAiModels.listType, "allow"), + eq(aiModels.enabled, true) + ) + ); + for (const row of rows) { + models.push({ + ...row, + providerName: providerNameById.get(row.providerId) ?? "" + }); + } + } + } + + models.sort((a, b) => { + const byProvider = a.providerName.localeCompare( + b.providerName, + undefined, + { + sensitivity: "base" + } + ); + if (byProvider !== 0) { + return byProvider; + } + return a.name.localeCompare(b.name, undefined, { sensitivity: "base" }); + }); + + return models; +} + /** * Model list APIs require an inference resource with at least one select-mode * attached provider. diff --git a/server/routers/external.ts b/server/routers/external.ts index 141660f0f..9c2f38e3d 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -590,6 +590,20 @@ authenticated.get( launcher.listLauncherResources ); +authenticated.get( + "/org/:orgId/launcher/resource/:resourceId/ai-models", + verifyOrgAccess, + verifyResourceAccess, + launcher.listLauncherPublicAiModels +); + +authenticated.get( + "/org/:orgId/launcher/site-resource/:siteResourceId/ai-models", + verifyOrgAccess, + verifySiteResourceAccess, + launcher.listLauncherSiteAiModels +); + authenticated.get( "/org/:orgId/launcher/sites", verifyOrgAccess, diff --git a/server/routers/launcher/formatLauncherAccess.ts b/server/routers/launcher/formatLauncherAccess.ts index ff5a5bf37..ca2104ccd 100644 --- a/server/routers/launcher/formatLauncherAccess.ts +++ b/server/routers/launcher/formatLauncherAccess.ts @@ -98,7 +98,7 @@ function formatTcpUdpResourceAccess( export function formatPublicResourceAccess( resource: PublicResourceAccessInput ): LauncherAccessFields { - const browserModes = ["http", "ssh", "rdp", "vnc"]; + const browserModes = ["http", "ssh", "rdp", "vnc", "inference"]; if (!browserModes.includes(resource.mode)) { return formatTcpUdpResourceAccess( resource.exitNodeEndpoint, @@ -125,15 +125,10 @@ export function formatPublicResourceAccess( export function formatSiteResourceAccess( resource: SiteResourceAccessInput ): LauncherAccessFields { - if (resource.alias) { - return { - accessDisplay: resource.alias, - accessCopyValue: resource.alias, - accessUrl: null - }; - } - - if (resource.mode === "http" && resource.fullDomain) { + if ( + (resource.mode === "http" || resource.mode === "inference") && + resource.fullDomain + ) { const url = `${resource.ssl ? "https" : "http"}://${resource.fullDomain}`; return { accessDisplay: url, @@ -142,6 +137,14 @@ export function formatSiteResourceAccess( }; } + if (resource.alias) { + return { + accessDisplay: resource.alias, + accessCopyValue: resource.alias, + accessUrl: null + }; + } + const destination = formatSiteResourceDestinationDisplay({ mode: resource.mode as SiteResourceDestinationInput["mode"], destination: resource.destination, diff --git a/server/routers/launcher/index.ts b/server/routers/launcher/index.ts index 1c3fed44c..db92f783d 100644 --- a/server/routers/launcher/index.ts +++ b/server/routers/launcher/index.ts @@ -5,6 +5,11 @@ export { listLauncherResources } from "./listLauncherResources"; export { listLauncherSites } from "./listLauncherSites"; export { listLauncherLabels } from "./listLauncherLabels"; export { listLauncherViews } from "./listLauncherViews"; +export { + listLauncherPublicAiModels, + listLauncherSiteAiModels +} from "./listLauncherAiModels"; +export type { ListLauncherAiModelsResponse } from "./listLauncherAiModels"; export { createLauncherView } from "./createLauncherView"; export { updateLauncherView } from "./updateLauncherView"; export { deleteLauncherView } from "./deleteLauncherView"; diff --git a/server/routers/launcher/launcherResourceAccess.ts b/server/routers/launcher/launcherResourceAccess.ts index ce94a7a50..386a6b1a7 100644 --- a/server/routers/launcher/launcherResourceAccess.ts +++ b/server/routers/launcher/launcherResourceAccess.ts @@ -31,6 +31,7 @@ import { inArray, isNull, like, + ne, or, sql, type SQL @@ -40,6 +41,7 @@ import { formatSiteResourceAccess } from "./formatLauncherAccess"; import { + LAUNCHER_AI_GATEWAY_GROUP_KEY, LAUNCHER_FLAT_GROUP_KEY, LAUNCHER_NO_SITE_GROUP_KEY, LAUNCHER_UNLABELED_GROUP_KEY, @@ -652,6 +654,7 @@ async function listSiteGroups( } } + let aiGatewayCount = 0; let noSiteCount = 0; if (accessible.resourceIds.length > 0 && siteFilterIds.length === 0) { @@ -665,27 +668,49 @@ async function listSiteGroups( noSitePublicConditions.push(searchPublic); } - let noSitePublicQuery = db - .select({ - itemCount: countDistinct(resources.resourceId) - }) - .from(resources) - .leftJoin(targets, eq(targets.resourceId, resources.resourceId)); + const buildNoSitePublicQuery = () => { + let queryBuilder = db + .select({ + itemCount: countDistinct(resources.resourceId) + }) + .from(resources) + .leftJoin( + targets, + eq(targets.resourceId, resources.resourceId) + ); + + if (labelFilterIds.length > 0) { + queryBuilder = queryBuilder.innerJoin( + resourceLabels, + eq(resourceLabels.resourceId, resources.resourceId) + ); + } + + return queryBuilder; + }; if (labelFilterIds.length > 0) { - noSitePublicQuery = noSitePublicQuery.innerJoin( - resourceLabels, - eq(resourceLabels.resourceId, resources.resourceId) - ); noSitePublicConditions.push( inArray(resourceLabels.labelId, labelFilterIds) ); } - const [noSitePublicRow] = await noSitePublicQuery.where( - and(...noSitePublicConditions, isNull(targets.targetId)) + const [aiGatewayPublicRow] = await buildNoSitePublicQuery().where( + and( + ...noSitePublicConditions, + isNull(targets.targetId), + eq(resources.mode, "inference") + ) + ); + const [noSitePublicRow] = await buildNoSitePublicQuery().where( + and( + ...noSitePublicConditions, + isNull(targets.targetId), + ne(resources.mode, "inference") + ) ); + aiGatewayCount += Number(aiGatewayPublicRow?.itemCount ?? 0); noSiteCount += Number(noSitePublicRow?.itemCount ?? 0); } @@ -700,38 +725,57 @@ async function listSiteGroups( noSiteSiteConditions.push(searchSite); } - let noSiteSiteQuery = db - .select({ - itemCount: countDistinct(siteResources.siteResourceId) - }) - .from(siteResources) - .leftJoin( - siteNetworks, - eq(siteResources.networkId, siteNetworks.networkId) - ) - .leftJoin(sites, eq(siteNetworks.siteId, sites.siteId)); + const buildNoSiteSiteQuery = () => { + let queryBuilder = db + .select({ + itemCount: countDistinct(siteResources.siteResourceId) + }) + .from(siteResources) + .leftJoin( + siteNetworks, + eq(siteResources.networkId, siteNetworks.networkId) + ) + .leftJoin(sites, eq(siteNetworks.siteId, sites.siteId)); + + if (labelFilterIds.length > 0) { + queryBuilder = queryBuilder.innerJoin( + siteResourceLabels, + eq( + siteResourceLabels.siteResourceId, + siteResources.siteResourceId + ) + ); + } + + return queryBuilder; + }; if (labelFilterIds.length > 0) { - noSiteSiteQuery = noSiteSiteQuery.innerJoin( - siteResourceLabels, - eq( - siteResourceLabels.siteResourceId, - siteResources.siteResourceId - ) - ); noSiteSiteConditions.push( inArray(siteResourceLabels.labelId, labelFilterIds) ); } - const [noSiteSiteRow] = await noSiteSiteQuery.where( - and(...noSiteSiteConditions, isNull(sites.siteId)) + const [aiGatewaySiteRow] = await buildNoSiteSiteQuery().where( + and( + ...noSiteSiteConditions, + isNull(sites.siteId), + eq(siteResources.mode, "inference") + ) + ); + const [noSiteSiteRow] = await buildNoSiteSiteQuery().where( + and( + ...noSiteSiteConditions, + isNull(sites.siteId), + ne(siteResources.mode, "inference") + ) ); + aiGatewayCount += Number(aiGatewaySiteRow?.itemCount ?? 0); noSiteCount += Number(noSiteSiteRow?.itemCount ?? 0); } - let groups: LauncherGroup[] = Array.from(siteCountMap.values()).map( + const siteGroups: LauncherGroup[] = Array.from(siteCountMap.values()).map( (row) => ({ groupKey: String(row.siteId), name: row.name, @@ -742,8 +786,26 @@ async function listSiteGroups( }) ); + siteGroups.sort((a, b) => { + const cmp = a.name.localeCompare(b.name, undefined, { + sensitivity: "base" + }); + return query.order === "desc" ? -cmp : cmp; + }); + + const pinnedGroups: LauncherGroup[] = []; + + if (aiGatewayCount > 0 && siteFilterIds.length === 0) { + pinnedGroups.push({ + groupKey: LAUNCHER_AI_GATEWAY_GROUP_KEY, + name: "AI Gateway", + groupType: "site", + itemCount: aiGatewayCount + }); + } + if (noSiteCount > 0 && siteFilterIds.length === 0) { - groups.push({ + pinnedGroups.push({ groupKey: LAUNCHER_NO_SITE_GROUP_KEY, name: "No Site", groupType: "site", @@ -751,12 +813,7 @@ async function listSiteGroups( }); } - groups.sort((a, b) => { - const cmp = a.name.localeCompare(b.name, undefined, { - sensitivity: "base" - }); - return query.order === "desc" ? -cmp : cmp; - }); + const groups = [...pinnedGroups, ...siteGroups]; const total = groups.length; return { @@ -1189,8 +1246,11 @@ function filterResourcesBySite( items: LauncherResource[], groupKey: string ): LauncherResource[] { + if (groupKey === LAUNCHER_AI_GATEWAY_GROUP_KEY) { + return items.filter((item) => item.mode === "inference"); + } if (groupKey === LAUNCHER_NO_SITE_GROUP_KEY) { - return items.filter((item) => !item.site); + return items.filter((item) => !item.site && item.mode !== "inference"); } const siteId = Number.parseInt(groupKey, 10); if (!Number.isFinite(siteId)) { @@ -1327,7 +1387,8 @@ async function listLauncherResourcesForUserUncached( const parsedSiteId = query.groupBy === "site" && - query.groupKey !== LAUNCHER_NO_SITE_GROUP_KEY + query.groupKey !== LAUNCHER_NO_SITE_GROUP_KEY && + query.groupKey !== LAUNCHER_AI_GATEWAY_GROUP_KEY ? Number.parseInt(query.groupKey, 10) : Number.NaN; const siteIdFilter = Number.isFinite(parsedSiteId) diff --git a/server/routers/launcher/listLauncherAiModels.ts b/server/routers/launcher/listLauncherAiModels.ts new file mode 100644 index 000000000..1f4258a8c --- /dev/null +++ b/server/routers/launcher/listLauncherAiModels.ts @@ -0,0 +1,172 @@ +import { db, resources, siteResources } from "@server/db"; +import { listEffectiveAllowModels } from "@server/lib/aiInferenceResource"; +import { response } from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import { and, eq } from "drizzle-orm"; +import { NextFunction, Request, Response } from "express"; +import createHttpError from "http-errors"; +import { fromZodError } from "zod-validation-error"; +import { z } from "zod"; + +const publicParamsSchema = z.strictObject({ + orgId: z.string().min(1), + resourceId: z.coerce.number().int().positive() +}); + +const siteParamsSchema = z.strictObject({ + orgId: z.string().min(1), + siteResourceId: z.coerce.number().int().positive() +}); + +export type ListLauncherAiModelsResponse = { + models: Awaited>; +}; + +export async function listLauncherPublicAiModels( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const orgId = req.userOrgId; + if (!orgId) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") + ); + } + + const parsed = publicParamsSchema.safeParse(req.params); + if (!parsed.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromZodError(parsed.error) + ) + ); + } + + const { resourceId } = parsed.data; + + const [resource] = await db + .select({ + resourceId: resources.resourceId, + mode: resources.mode + }) + .from(resources) + .where( + and( + eq(resources.resourceId, resourceId), + eq(resources.orgId, orgId) + ) + ) + .limit(1); + + if (!resource || resource.mode !== "inference") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI models are only available for inference resources" + ) + ); + } + + const models = await listEffectiveAllowModels({ resourceId }); + return response(res, { + data: { models }, + success: true, + error: false, + message: "Launcher AI models retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + if (createHttpError.isHttpError(error)) { + return next(error); + } + console.error("Error listing launcher AI models:", error); + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Internal server error" + ) + ); + } +} + +export async function listLauncherSiteAiModels( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const orgId = req.userOrgId; + if (!orgId) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID") + ); + } + + const parsed = siteParamsSchema.safeParse(req.params); + if (!parsed.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromZodError(parsed.error) + ) + ); + } + + const { siteResourceId } = parsed.data; + + const siteResource = + req.siteResource ?? + ( + await db + .select({ + siteResourceId: siteResources.siteResourceId, + mode: siteResources.mode, + orgId: siteResources.orgId + }) + .from(siteResources) + .where( + and( + eq(siteResources.siteResourceId, siteResourceId), + eq(siteResources.orgId, orgId) + ) + ) + .limit(1) + )[0]; + + if ( + !siteResource || + siteResource.orgId !== orgId || + siteResource.mode !== "inference" + ) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "AI models are only available for inference resources" + ) + ); + } + + const models = await listEffectiveAllowModels({ siteResourceId }); + return response(res, { + data: { models }, + success: true, + error: false, + message: "Launcher AI models retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + if (createHttpError.isHttpError(error)) { + return next(error); + } + console.error("Error listing launcher AI models:", error); + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Internal server error" + ) + ); + } +} diff --git a/server/routers/launcher/types.ts b/server/routers/launcher/types.ts index f235e824d..7084781e9 100644 --- a/server/routers/launcher/types.ts +++ b/server/routers/launcher/types.ts @@ -2,6 +2,7 @@ import { z } from "zod"; export const LAUNCHER_UNLABELED_GROUP_KEY = "unlabeled"; export const LAUNCHER_NO_SITE_GROUP_KEY = "no-site"; +export const LAUNCHER_AI_GATEWAY_GROUP_KEY = "ai-gateway"; export const LAUNCHER_FLAT_GROUP_KEY = "__all__"; export const launcherViewConfigSchema = z.object({ diff --git a/src/app/[orgId]/settings/resources/private/create/page.tsx b/src/app/[orgId]/settings/resources/private/create/page.tsx index a1b04f5e4..90f115bfd 100644 --- a/src/app/[orgId]/settings/resources/private/create/page.tsx +++ b/src/app/[orgId]/settings/resources/private/create/page.tsx @@ -368,56 +368,65 @@ export default function CreatePrivateResourcePage() { /> - {mode === "http" || - (mode === "inference" && ( - - - { - if (!res) { + {(mode === "http" || + mode === "inference") && ( + + ( + + { + if (!res) { + form.setValue( + "httpConfigSubdomain", + null + ); + form.setValue( + "httpConfigDomainId", + null + ); + form.setValue( + "httpConfigFullDomain", + null + ); + return; + } form.setValue( "httpConfigSubdomain", - null + res.subdomain ?? + null ); form.setValue( "httpConfigDomainId", - null + res.domainId, + { + shouldValidate: true + } ); form.setValue( "httpConfigFullDomain", - null + res.fullDomain ); - return; - } - form.setValue( - "httpConfigSubdomain", - res.subdomain ?? - null - ); - form.setValue( - "httpConfigDomainId", - res.domainId - ); - form.setValue( - "httpConfigFullDomain", - res.fullDomain - ); - }} - /> - - - {t( - "resourceDomainDescription" - )} - - - - ))} + }} + /> + + + {t( + "resourceDomainDescription" + )} + + + )} + /> + + )} {(mode === "host" || (mode === "ssh" && !isNativeSsh)) && ( diff --git a/src/components/PrivateResourceInfoBox.tsx b/src/components/PrivateResourceInfoBox.tsx index fda9a7596..90ade3334 100644 --- a/src/components/PrivateResourceInfoBox.tsx +++ b/src/components/PrivateResourceInfoBox.tsx @@ -124,13 +124,17 @@ export function PrivateResourceInfoSections({ siteResource.fullDomain && build != "oss" ); + const showPortRestrictions = + isPanel && + siteResource.mode !== "http" && + siteResource.mode !== "inference"; const numSections = 2 + (showDestination ? 1 : 0) + (showAlias ? 1 : 0) + (showCertificate ? 1 : 0) + - (isPanel ? 1 : 0); + (showPortRestrictions ? 1 : 0); const sections = ( @@ -194,7 +198,7 @@ export function PrivateResourceInfoSections({ ) : null} - {isPanel ? ( + {showPortRestrictions ? ( {t("portRestrictions")} diff --git a/src/components/resource-launcher/LauncherGroupSection.tsx b/src/components/resource-launcher/LauncherGroupSection.tsx index 49b648032..91320b383 100644 --- a/src/components/resource-launcher/LauncherGroupSection.tsx +++ b/src/components/resource-launcher/LauncherGroupSection.tsx @@ -17,6 +17,7 @@ import type { LauncherViewConfig } from "@server/routers/launcher/types"; import { + LAUNCHER_AI_GATEWAY_GROUP_KEY, LAUNCHER_NO_SITE_GROUP_KEY, LAUNCHER_UNLABELED_GROUP_KEY } from "@server/routers/launcher/types"; @@ -148,9 +149,11 @@ export function LauncherGroupSection({ const groupTitle = group.groupKey === LAUNCHER_UNLABELED_GROUP_KEY ? t("resourceLauncherUnlabeled") - : group.groupKey === LAUNCHER_NO_SITE_GROUP_KEY - ? t("resourceLauncherNoSite") - : group.name; + : group.groupKey === LAUNCHER_AI_GATEWAY_GROUP_KEY + ? t("resourceLauncherAiGateway") + : group.groupKey === LAUNCHER_NO_SITE_GROUP_KEY + ? t("resourceLauncherNoSite") + : group.name; return ( ; + return ( - {group.groupType === "site" || group.groupType === "label" ? ( - - ) : null} + {statusDot} {title} ({group.itemCount}) diff --git a/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx b/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx new file mode 100644 index 000000000..8a71a07db --- /dev/null +++ b/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx @@ -0,0 +1,223 @@ +"use client"; + +import CopyToClipboard from "@app/components/CopyToClipboard"; +import { + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionHeader, + SettingsSectionTitle, + SettingsSubsectionDescription, + SettingsSubsectionHeader, + SettingsSubsectionTitle +} from "@app/components/Settings"; +import { Button } from "@app/components/ui/button"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { launcherQueries } from "@app/lib/queries"; +import type { + GetMyVirtualApiKeyResponse, + VirtualApiKeyWithResources +} from "@server/routers/virtualApiKey/types"; +import { useQuery } from "@tanstack/react-query"; +import type { AxiosResponse } from "axios"; +import { Loader2 } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useState } from "react"; + +type LauncherInferenceApiKeysSectionProps = { + orgId: string; + resourceGuid: string; +}; + +function keyPreview(virtualApiKeyId: string, lastChars: string): string { + return `vk-${virtualApiKeyId}••••${lastChars}`; +} + +function useRevealSecret(orgId: string, virtualApiKeyId: string) { + const t = useTranslations(); + const api = createApiClient(useEnvContext()); + const [credential, setCredential] = useState(null); + const [loading, setLoading] = useState(false); + + const revealSecret = () => { + if (credential || loading) { + return; + } + + setLoading(true); + api.get>( + `/org/${orgId}/my-virtual-api-keys/${virtualApiKeyId}` + ) + .then((res) => { + const secret = res.data.data.virtualApiKey.secret; + if (secret) { + setCredential(`vk-${virtualApiKeyId}.${secret}`); + } else { + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorFetchSecret"), + description: t( + "virtualApiKeysErrorFetchSecretDescription" + ) + }); + } + }) + .catch((e) => { + toast({ + variant: "destructive", + title: t("virtualApiKeysErrorFetchSecret"), + description: formatAxiosError( + e, + t("virtualApiKeysErrorFetchSecretDescription") + ) + }); + }) + .finally(() => { + setLoading(false); + }); + }; + + return { credential, loading, revealSecret }; +} + +function PanelKeySecret({ + orgId, + virtualApiKeyId, + lastChars +}: { + orgId: string; + virtualApiKeyId: string; + lastChars: string; +}) { + const t = useTranslations(); + const preview = keyPreview(virtualApiKeyId, lastChars); + const { credential, loading, revealSecret } = useRevealSecret( + orgId, + virtualApiKeyId + ); + const displayValue = credential ?? preview; + + return ( +
+
+ +
+ {!credential ? ( + + ) : null} +
+ ); +} + +function ManualKeyRow({ + orgId, + keyRow +}: { + orgId: string; + keyRow: VirtualApiKeyWithResources; +}) { + const t = useTranslations(); + + return ( +
+

+ {keyRow.name || t("myVirtualApiKeysUnnamed")} +

+ {keyRow.description ? ( +

+ {keyRow.description} +

+ ) : null} + +
+ ); +} + +export function LauncherInferenceApiKeysSection({ + orgId, + resourceGuid +}: LauncherInferenceApiKeysSectionProps) { + const t = useTranslations(); + const { data, isPending, isError } = useQuery( + launcherQueries.myVirtualApiKeys(orgId, resourceGuid) + ); + + return ( + + + + {t("resourceLauncherApiKeys")} + + + {t("resourceLauncherApiKeysDescription")} + + + + {isPending ? ( +
+ +
+ ) : null} + {isError ? ( +

+ {t("resourceLauncherApiKeysError")} +

+ ) : null} + {!isPending && !isError && data ? ( +
+
+

+ {t("resourceLauncherApiKeysIdentity")} +

+ +
+ {data.manualKeys.length > 0 ? ( +
+ + + {t("resourceLauncherApiKeysManual")} + + + {t( + "myVirtualApiKeysManualResourceDescription" + )} + + +
+ {data.manualKeys.map((keyRow) => ( + + ))} +
+
+ ) : null} +
+ ) : null} + + + ); +} diff --git a/src/components/resource-launcher/LauncherInferenceModelsSection.tsx b/src/components/resource-launcher/LauncherInferenceModelsSection.tsx new file mode 100644 index 000000000..510cce099 --- /dev/null +++ b/src/components/resource-launcher/LauncherInferenceModelsSection.tsx @@ -0,0 +1,170 @@ +"use client"; + +import { + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { Button } from "@app/components/ui/button"; +import { cn } from "@app/lib/cn"; +import { launcherQueries } from "@app/lib/queries"; +import { useQuery } from "@tanstack/react-query"; +import { Loader2 } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useEffect, useLayoutEffect, useRef, useState } from "react"; + +const COLLAPSED_ROWS = 5; +const GRID_COLUMNS = 2; + +type LauncherInferenceModelsSectionProps = { + orgId: string; + params: + | { + resourceType: "public"; + resourceId: number; + } + | { + resourceType: "site"; + siteResourceId: number; + }; +}; + +export function LauncherInferenceModelsSection({ + orgId, + params +}: LauncherInferenceModelsSectionProps) { + const t = useTranslations(); + const { data, isPending, isError } = useQuery( + launcherQueries.aiModels(orgId, params) + ); + const models = data?.models ?? []; + const [listExpanded, setListExpanded] = useState(false); + const [clipHeight, setClipHeight] = useState(null); + const gridRef = useRef(null); + + const collapsedLimit = GRID_COLUMNS * COLLAPSED_ROWS; + const hasOverflow = models.length > collapsedLimit; + const isCollapsed = hasOverflow && !listExpanded; + + useEffect(() => { + if (!hasOverflow) { + setListExpanded(false); + } + }, [hasOverflow]); + + useLayoutEffect(() => { + if (!isCollapsed || !gridRef.current) { + setClipHeight(null); + return; + } + + const children = Array.from(gridRef.current.children) as HTMLElement[]; + const lastVisible = children[collapsedLimit - 1]; + if (!lastVisible) { + setClipHeight(null); + return; + } + + const gridTop = gridRef.current.getBoundingClientRect().top; + const cardBottom = lastVisible.getBoundingClientRect().bottom; + // Peek slightly into the next row so the fade has content to soften. + setClipHeight(cardBottom - gridTop + 12); + }, [isCollapsed, collapsedLimit, models]); + + return ( + + + + {t("resourceLauncherAvailableModels")} + + + {t("resourceLauncherAvailableModelsDescription")} + + + + {isPending ? ( +
+ +
+ ) : null} + {isError ? ( +

+ {t("resourceLauncherAvailableModelsError")} +

+ ) : null} + {!isPending && !isError && models.length === 0 ? ( +

+ {t("resourceLauncherAvailableModelsEmpty")} +

+ ) : null} + {!isPending && !isError && models.length > 0 ? ( +
+
+
+ {models.map((model) => ( +
+ + {model.modelKey} + + {model.providerName ? ( + + {model.providerName} + + ) : null} +
+ ))} +
+ {isCollapsed ? ( +
+ ) : null} +
+ {isCollapsed ? ( +
+ +
+ ) : null} + {hasOverflow && listExpanded ? ( +
+ +
+ ) : null} +
+ ) : null} + + + ); +} diff --git a/src/components/resource-launcher/LauncherResourcePanel.tsx b/src/components/resource-launcher/LauncherResourcePanel.tsx index fb948bbcc..999d1bc03 100644 --- a/src/components/resource-launcher/LauncherResourcePanel.tsx +++ b/src/components/resource-launcher/LauncherResourcePanel.tsx @@ -8,6 +8,8 @@ import { InfoSectionTitle } from "@app/components/InfoSection"; import { PrivateResourceInfoSections } from "@app/components/PrivateResourceInfoBox"; +import { LauncherInferenceApiKeysSection } from "@app/components/resource-launcher/LauncherInferenceApiKeysSection"; +import { LauncherInferenceModelsSection } from "@app/components/resource-launcher/LauncherInferenceModelsSection"; import { SettingsSection, SettingsSectionBody, @@ -146,7 +148,8 @@ function HealthStatusDisplay({ ); } -const PUBLIC_AUTH_BROWSER_MODES = ["http", "ssh", "rdp", "vnc"]; +const PUBLIC_AUTH_METHODS_MODES = ["http", "ssh", "rdp", "vnc"]; +const PUBLIC_AUTH_BADGE_MODES = [...PUBLIC_AUTH_METHODS_MODES, "inference"]; function AuthMethodStatusDisplay({ enabled }: { enabled: boolean }) { const t = useTranslations(); @@ -227,20 +230,24 @@ function PublicResourceAuthMethods({ } function PublicResourceDetails({ + orgId, launcherResource, resource, authInfo }: { + orgId: string; launcherResource: LauncherResource; resource: GetResourceResponse; authInfo: GetResourceAuthInfoResponse; }) { const t = useTranslations(); - const supportsAuth = PUBLIC_AUTH_BROWSER_MODES.includes( - resource.mode || "" - ); + const mode = resource.mode || ""; + const isInference = mode === "inference"; + const showAuthBadge = PUBLIC_AUTH_BADGE_MODES.includes(mode); + const showAuthMethods = PUBLIC_AUTH_METHODS_MODES.includes(mode); + const showHealth = !isInference; const authState = derivePublicAuthState(resource.mode, authInfo); - const infoSectionCount = supportsAuth ? 4 : 3; + const infoSectionCount = 2 + (showAuthBadge ? 1 : 0) + (showHealth ? 1 : 0); return (
@@ -275,7 +282,7 @@ function PublicResourceDetails({ /> - {supportsAuth ? ( + {showAuthBadge ? ( {t("authentication")} @@ -295,30 +302,54 @@ function PublicResourceDetails({ ) : null} - - {t("health")} - - - - + {showHealth ? ( + + + {t("health")} + + + + + + ) : null} - {supportsAuth ? ( + {showAuthMethods ? ( ) : null} + {isInference ? ( + <> + + + + ) : null}
); } function PrivateResourceDetails({ + orgId, launcherResource, resource }: { + orgId: string; launcherResource: LauncherResource; resource: GetSiteResourceResponse; }) { const t = useTranslations(); + const isInference = resource.mode === "inference"; return (
@@ -365,6 +396,15 @@ function PrivateResourceDetails({ /> + {isInference ? ( + + ) : null}
); } @@ -405,6 +445,7 @@ function LauncherResourcePanelBody({ if (detail.resourceType === "public") { return ( diff --git a/src/lib/launcherResourceAccess.ts b/src/lib/launcherResourceAccess.ts index 6a208ef7e..f779bfec6 100644 --- a/src/lib/launcherResourceAccess.ts +++ b/src/lib/launcherResourceAccess.ts @@ -37,7 +37,7 @@ export type LauncherAccessFields = { export function formatPublicResourceAccess( resource: PublicResourceAccessInput ): LauncherAccessFields { - const browserModes = ["http", "ssh", "rdp", "vnc"]; + const browserModes = ["http", "ssh", "rdp", "vnc", "inference"]; if (!browserModes.includes(resource.mode)) { const port = resource.proxyPort?.toString() ?? ""; return { @@ -66,16 +66,8 @@ export function formatPublicResourceAccess( export function formatSiteResourceAccess( resource: SiteResourceAccessInput ): LauncherAccessFields { - if (resource.alias) { - return { - accessDisplay: resource.alias, - accessCopyValue: resource.alias, - accessUrl: null - }; - } - if ( - (resource.mode === "http" || resource.mode == "inference") && + (resource.mode === "http" || resource.mode === "inference") && resource.fullDomain ) { const url = `${resource.ssl ? "https" : "http"}://${resource.fullDomain}`; @@ -86,6 +78,14 @@ export function formatSiteResourceAccess( }; } + if (resource.alias) { + return { + accessDisplay: resource.alias, + accessCopyValue: resource.alias, + accessUrl: null + }; + } + const destination = formatSiteResourceDestinationDisplay({ mode: resource.mode as SiteResourceDestinationInput["mode"], destination: resource.destination, diff --git a/src/lib/launcherResourceDetails.ts b/src/lib/launcherResourceDetails.ts index 9920ceeef..4e77c18b0 100644 --- a/src/lib/launcherResourceDetails.ts +++ b/src/lib/launcherResourceDetails.ts @@ -4,7 +4,7 @@ import type { GetSiteResourceResponse } from "@server/routers/siteResource/getSi export type PublicAuthState = "protected" | "not_protected" | "none"; -const BROWSER_MODES = ["http", "ssh", "rdp", "vnc"]; +const BROWSER_MODES = ["http", "ssh", "rdp", "vnc", "inference"]; export function derivePublicAuthState( mode: string | null, @@ -37,6 +37,10 @@ export function formatPublicResourceType( return resource.ssl ? "HTTPS" : "HTTP"; } + if (resource.mode === "inference") { + return "Inference"; + } + const mode = (resource.mode || "").toLowerCase(); if (mode === "tcp") { return "TCP"; diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 8453022dd..02a9404e8 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -34,6 +34,8 @@ import type { ListLauncherSitesResponse, ListLauncherViewsResponse } from "@server/routers/launcher/types"; +import type { ListLauncherAiModelsResponse } from "@server/routers/launcher/listLauncherAiModels"; +import type { ListMyVirtualApiKeysResponse } from "@server/routers/virtualApiKey/types"; import type { GetResourcePolicyResponse } from "@server/routers/policy"; import type { GetResourcePoliciesResponse, @@ -1776,5 +1778,69 @@ export const launcherQueries = { data: res.data.data }; } + }), + aiModels: ( + orgId: string, + params: + | { + resourceType: "public"; + resourceId: number; + } + | { + resourceType: "site"; + siteResourceId: number; + } + | null + ) => + queryOptions({ + queryKey: ["ORG", orgId, "LAUNCHER", "AI_MODELS", params] as const, + enabled: params != null, + queryFn: async ({ signal, meta }) => { + if (!params) { + throw new Error("Resource params are required"); + } + + if (params.resourceType === "public") { + const res = await meta!.api.get< + AxiosResponse + >( + `/org/${orgId}/launcher/resource/${params.resourceId}/ai-models`, + { signal } + ); + return res.data.data; + } + + const res = await meta!.api.get< + AxiosResponse + >( + `/org/${orgId}/launcher/site-resource/${params.siteResourceId}/ai-models`, + { signal } + ); + return res.data.data; + } + }), + myVirtualApiKeys: (orgId: string, resourceGuid: string | null) => + queryOptions({ + queryKey: [ + "ORG", + orgId, + "LAUNCHER", + "MY_VIRTUAL_API_KEYS", + resourceGuid + ] as const, + enabled: Boolean(resourceGuid), + queryFn: async ({ signal, meta }) => { + if (!resourceGuid) { + throw new Error("resourceGuid is required"); + } + + const res = await meta!.api.get< + AxiosResponse + >( + `/org/${orgId}/my-virtual-api-keys?resourceGuid=${encodeURIComponent(resourceGuid)}`, + { signal } + ); + return res.data.data; + } }) }; From bc2f291352c90f3f6051dad3ea477bf9509d7c33 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 14:45:05 -0400 Subject: [PATCH 118/296] normalize the requests to also store in the log for viewing later --- server/db/pg/schema/schema.ts | 13 +- server/db/sqlite/schema/schema.ts | 13 +- server/lib/aiMessageNormalization.ts | 538 +++++++++++++++++++++++ server/lib/aiUsageExtraction.ts | 24 +- server/routers/aiGateway/logAiSession.ts | 35 +- 5 files changed, 611 insertions(+), 12 deletions(-) create mode 100644 server/lib/aiMessageNormalization.ts diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 7895b0baf..cbb1e60cb 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -69,7 +69,7 @@ export const orgs = pgTable("orgs", { "settingsLogRetentionDaysAISessions" ) // where 0 = dont keep logs and -1 = keep forever and 9001 = end of the following year .notNull() - .default(0), + .default(7), sshCaPrivateKey: text("sshCaPrivateKey"), // Encrypted SSH CA private key (PEM format) sshCaPublicKey: text("sshCaPublicKey"), // SSH CA public key (OpenSSH format) isBillingOrg: boolean("isBillingOrg"), @@ -1936,8 +1936,15 @@ export const aiSessionLog = pgTable( isStream: boolean("isStream").notNull().default(false), requestBody: text("requestBody"), responseBody: text("responseBody"), - // True if requestBody/responseBody were cut short at - // AI_SESSION_LOG_MAX_BODY_CHARS before storage. + // Capability-agnostic message transcript (JSON-encoded + // NormalizedAiMessage[] from server/lib/aiMessageNormalization.ts), + // computed at write time so search/display never need per-capability + // parsing logic. Null when normalization couldn't recognize the + // shape - callers fall back to requestBody/responseBody. + normalizedRequest: text("normalizedRequest"), + normalizedResponse: text("normalizedResponse"), + // True if any of the request/response (raw or normalized) fields + // were cut short at AI_SESSION_LOG_MAX_BODY_CHARS before storage. truncated: boolean("truncated").notNull().default(false), statusCode: integer("statusCode"), createdAt: bigint("createdAt", { mode: "number" }).notNull() // epoch ms diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 3fb15c0d6..8934638d7 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -68,7 +68,7 @@ export const orgs = sqliteTable("orgs", { "settingsLogRetentionDaysAISessions" ) // where 0 = dont keep logs and -1 = keep forever and 9001 = end of the following year .notNull() - .default(0), + .default(7), sshCaPrivateKey: text("sshCaPrivateKey"), // Encrypted SSH CA private key (PEM format) sshCaPublicKey: text("sshCaPublicKey"), // SSH CA public key (OpenSSH format) isBillingOrg: integer("isBillingOrg", { mode: "boolean" }), @@ -1928,8 +1928,15 @@ export const aiSessionLog = sqliteTable( .default(false), requestBody: text("requestBody"), responseBody: text("responseBody"), - // True if requestBody/responseBody were cut short at - // AI_SESSION_LOG_MAX_BODY_CHARS before storage. + // Capability-agnostic message transcript (JSON-encoded + // NormalizedAiMessage[] from server/lib/aiMessageNormalization.ts), + // computed at write time so search/display never need per-capability + // parsing logic. Null when normalization couldn't recognize the + // shape - callers fall back to requestBody/responseBody. + normalizedRequest: text("normalizedRequest"), + normalizedResponse: text("normalizedResponse"), + // True if any of the request/response (raw or normalized) fields + // were cut short at AI_SESSION_LOG_MAX_BODY_CHARS before storage. truncated: integer("truncated", { mode: "boolean" }) .notNull() .default(false), diff --git a/server/lib/aiMessageNormalization.ts b/server/lib/aiMessageNormalization.ts new file mode 100644 index 000000000..1153142e1 --- /dev/null +++ b/server/lib/aiMessageNormalization.ts @@ -0,0 +1,538 @@ +import type { AiCapability } from "@server/lib/aiCapabilities"; +import { sseDataFrames, tryParseJson } from "@server/lib/aiUsageExtraction"; +import logger from "@server/logger"; + +// Uniform, capability-agnostic representation of a chat message, used so +// the AI session log can be searched/displayed the same way regardless of +// which provider/capability produced it. Content is flattened to plain text +// - non-text parts (images, tool calls/results) are rendered as readable +// placeholders rather than preserved as structured data, which is enough for +// a transcript-style replay view without a per-capability renderer. +export type NormalizedRole = "system" | "user" | "assistant" | "tool"; + +export type NormalizedAiMessage = { + role: NormalizedRole; + content: string; +}; + +function normalizeRole(role: unknown): NormalizedRole { + if ( + role === "system" || + role === "user" || + role === "assistant" || + role === "tool" + ) { + return role; + } + if (role === "model") return "assistant"; // Gemini + if (role === "function") return "tool"; // OpenAI legacy function role + return "user"; +} + +function safeJsonStringify(value: unknown): string { + try { + return JSON.stringify(value ?? {}); + } catch { + return ""; + } +} + +/** + * Flattens one message "part"/"block" (OpenAI content parts, Anthropic + * content blocks, Gemini parts, Bedrock converse content blocks - they all + * follow the same rough shape) into readable text. + */ +function flattenContentPart(part: unknown): string { + if (typeof part === "string") return part; + if (part == null || typeof part !== "object") return ""; + const p = part as Record; + + if (typeof p.text === "string") return p.text; + + if ( + p.type === "image_url" || + p.type === "image" || + p.type === "input_image" || + p.type === "output_image" || + "inlineData" in p + ) { + return "[image]"; + } + + // Anthropic-style tool_use / tool_result blocks + if (p.type === "tool_use") { + const name = typeof p.name === "string" ? p.name : "tool"; + return `[tool_call: ${name}(${safeJsonStringify(p.input)})]`; + } + if (p.type === "tool_result") { + const content = p.content; + const text = + typeof content === "string" + ? content + : Array.isArray(content) + ? flattenContentParts(content) + : ""; + return `[tool_result: ${text}]`; + } + + // Gemini-style functionCall / functionResponse parts + if (p.functionCall && typeof p.functionCall === "object") { + const fc = p.functionCall as Record; + return `[tool_call: ${fc.name}(${safeJsonStringify(fc.args)})]`; + } + if (p.functionResponse && typeof p.functionResponse === "object") { + const fr = p.functionResponse as Record; + return `[tool_result: ${fr.name}(${safeJsonStringify(fr.response)})]`; + } + + // Bedrock converse-style toolUse / toolResult content blocks + if (p.toolUse && typeof p.toolUse === "object") { + const tu = p.toolUse as Record; + return `[tool_call: ${tu.name}(${safeJsonStringify(tu.input)})]`; + } + if (p.toolResult && typeof p.toolResult === "object") { + const tr = p.toolResult as Record; + const content = tr.content; + const text = Array.isArray(content) ? flattenContentParts(content) : ""; + return `[tool_result: ${text}]`; + } + + return ""; +} + +function flattenContentParts(parts: unknown[]): string { + return parts.map(flattenContentPart).join(""); +} + +function flattenContent(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) return flattenContentParts(content); + return ""; +} + +/** + * Best-effort scan for every `"text":"..."` JSON string value in raw text, + * concatenated in order. Fallback for streaming formats we can't fully parse + * as JSON/SSE (Gemini's array-JSON stream, Bedrock's binary event-stream + * framing) - same spirit as aiUsageExtraction's scanNumericFields. + */ +function scanTextFragments(text: string): string { + const out: string[] = []; + const re = /"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g; + let match: RegExpExecArray | null; + while ((match = re.exec(text)) !== null) { + try { + out.push(JSON.parse(`"${match[1]}"`)); + } catch { + out.push(match[1]); + } + } + return out.join(""); +} + +// --------------------------------------------------------------------------- +// Request (input) normalizers - operate on the already-parsed outbound body. +// --------------------------------------------------------------------------- + +function normalizeOpenAiChatRequest(body: any): NormalizedAiMessage[] { + const messages = Array.isArray(body?.messages) ? body.messages : []; + return messages.map((m: any) => ({ + role: normalizeRole(m?.role), + content: flattenContent(m?.content) + })); +} + +function normalizeOpenAiResponsesRequest(body: any): NormalizedAiMessage[] { + const out: NormalizedAiMessage[] = []; + if (typeof body?.instructions === "string" && body.instructions) { + out.push({ role: "system", content: body.instructions }); + } + const input = body?.input; + if (typeof input === "string") { + out.push({ role: "user", content: input }); + } else if (Array.isArray(input)) { + for (const item of input) { + if (item?.role) { + out.push({ + role: normalizeRole(item.role), + content: flattenContent(item.content) + }); + } else if (typeof item?.type === "string") { + out.push({ role: "tool", content: `[${item.type}]` }); + } + } + } + return out; +} + +function normalizeAnthropicRequest(body: any): NormalizedAiMessage[] { + const out: NormalizedAiMessage[] = []; + if (body?.system) { + const sys = flattenContent(body.system); + if (sys) out.push({ role: "system", content: sys }); + } + const messages = Array.isArray(body?.messages) ? body.messages : []; + for (const m of messages) { + out.push({ + role: normalizeRole(m?.role), + content: flattenContent(m?.content) + }); + } + return out; +} + +function normalizeGeminiRequest(body: any): NormalizedAiMessage[] { + const out: NormalizedAiMessage[] = []; + const sysParts = body?.systemInstruction?.parts; + if (Array.isArray(sysParts)) { + const text = flattenContentParts(sysParts); + if (text) out.push({ role: "system", content: text }); + } + const contents = Array.isArray(body?.contents) ? body.contents : []; + for (const c of contents) { + out.push({ + role: normalizeRole(c?.role), + content: Array.isArray(c?.parts) ? flattenContentParts(c.parts) : "" + }); + } + return out; +} + +function normalizeBedrockConverseRequest(body: any): NormalizedAiMessage[] { + const out: NormalizedAiMessage[] = []; + if (Array.isArray(body?.system)) { + const text = flattenContentParts(body.system); + if (text) out.push({ role: "system", content: text }); + } + const messages = Array.isArray(body?.messages) ? body.messages : []; + for (const m of messages) { + out.push({ + role: normalizeRole(m?.role), + content: Array.isArray(m?.content) + ? flattenContentParts(m.content) + : "" + }); + } + return out; +} + +/** + * bedrock_model_invoke and google_raw_predict are passthroughs - the body + * shape depends entirely on the underlying model, not the capability. Try + * the two shapes we're most likely to see (Anthropic Claude, then plain + * OpenAI-style) and give up otherwise, same fallback spirit + * aiUsageExtraction.ts uses for these two capabilities' usage extraction. + */ +function normalizeBestEffortRequest(body: any): NormalizedAiMessage[] | null { + if (!Array.isArray(body?.messages)) return null; + const looksAnthropicShaped = body.messages.some((m: any) => + Array.isArray(m?.content) + ); + return looksAnthropicShaped + ? normalizeAnthropicRequest(body) + : normalizeOpenAiChatRequest(body); +} + +// --------------------------------------------------------------------------- +// Response (output) normalizers - operate on the raw response text, which +// may be a single JSON document (non-streaming) or provider-framed streaming +// text (SSE `data:` frames, a JSON-array stream, or binary event-stream +// framing with JSON payloads embedded in it). +// --------------------------------------------------------------------------- + +function normalizeOpenAiChatResponse( + text: string, + isStream: boolean +): NormalizedAiMessage[] | null { + if (isStream) { + let role: unknown = "assistant"; + let content = ""; + let found = false; + for (const frame of sseDataFrames(text)) { + const delta = tryParseJson(frame)?.choices?.[0]?.delta; + if (!delta) continue; + found = true; + if (typeof delta.role === "string") role = delta.role; + if (typeof delta.content === "string") content += delta.content; + } + return found ? [{ role: normalizeRole(role), content }] : null; + } + + const message = tryParseJson(text)?.choices?.[0]?.message; + if (!message) return null; + return [ + { + role: normalizeRole(message.role), + content: flattenContent(message.content) + } + ]; +} + +function extractOpenAiResponsesOutputText(response: any): string | null { + if (typeof response?.output_text === "string") return response.output_text; + const output = Array.isArray(response?.output) ? response.output : []; + const pieces: string[] = []; + for (const item of output) { + if (item?.type === "message" && Array.isArray(item.content)) { + pieces.push(flattenContentParts(item.content)); + } + } + return pieces.length > 0 ? pieces.join("") : null; +} + +function normalizeOpenAiResponsesResponse( + text: string, + isStream: boolean +): NormalizedAiMessage[] | null { + if (isStream) { + let content = ""; + let found = false; + for (const frame of sseDataFrames(text)) { + const parsed = tryParseJson(frame); + if (!parsed) continue; + if ( + parsed.type === "response.output_text.delta" && + typeof parsed.delta === "string" + ) { + content += parsed.delta; + found = true; + } else if ( + parsed.type === "response.completed" && + parsed.response + ) { + const outputText = extractOpenAiResponsesOutputText( + parsed.response + ); + if (outputText != null) { + content = outputText; + found = true; + } + } + } + return found ? [{ role: "assistant", content }] : null; + } + + const parsed = tryParseJson(text); + const outputText = extractOpenAiResponsesOutputText( + parsed?.response ?? parsed + ); + return outputText != null + ? [{ role: "assistant", content: outputText }] + : null; +} + +function normalizeAnthropicResponse( + text: string, + isStream: boolean +): NormalizedAiMessage[] | null { + if (isStream) { + let role: unknown = "assistant"; + let content = ""; + let found = false; + for (const frame of sseDataFrames(text)) { + const parsed = tryParseJson(frame); + if (!parsed) continue; + if (parsed.type === "message_start" && parsed.message?.role) { + role = parsed.message.role; + } + if ( + parsed.type === "content_block_start" && + parsed.content_block?.type === "tool_use" + ) { + const name = parsed.content_block.name ?? "tool"; + content += `[tool_call: ${name}]`; + found = true; + } + if ( + parsed.type === "content_block_delta" && + typeof parsed.delta?.text === "string" + ) { + content += parsed.delta.text; + found = true; + } + } + return found ? [{ role: normalizeRole(role), content }] : null; + } + + const parsed = tryParseJson(text); + if (!parsed || !Array.isArray(parsed.content)) return null; + return [ + { + role: normalizeRole(parsed.role ?? "assistant"), + content: flattenContentParts(parsed.content) + } + ]; +} + +function geminiCandidateParts(node: any): string { + const parts = node?.candidates?.[0]?.content?.parts; + return Array.isArray(parts) ? flattenContentParts(parts) : ""; +} + +function normalizeGeminiResponse( + text: string, + _isStream: boolean +): NormalizedAiMessage[] | null { + const frames = sseDataFrames(text); + let content = ""; + let role: unknown = "model"; + let found = false; + + if (frames.length > 0) { + for (const frame of frames) { + const parsed = tryParseJson(frame); + const piece = geminiCandidateParts(parsed); + if (piece) { + content += piece; + found = true; + } + const r = parsed?.candidates?.[0]?.content?.role; + if (r) role = r; + } + } else { + const parsed = tryParseJson(text); + if (Array.isArray(parsed)) { + for (const chunk of parsed) { + const piece = geminiCandidateParts(chunk); + if (piece) { + content += piece; + found = true; + } + const r = chunk?.candidates?.[0]?.content?.role; + if (r) role = r; + } + } else if (parsed) { + const piece = geminiCandidateParts(parsed); + if (piece) { + content = piece; + found = true; + } + const r = parsed?.candidates?.[0]?.content?.role; + if (r) role = r; + } + } + + if (!found) { + const scanned = scanTextFragments(text); + return scanned + ? [{ role: normalizeRole(role), content: scanned }] + : null; + } + return [{ role: normalizeRole(role), content }]; +} + +function normalizeBedrockConverseResponse( + text: string, + isStream: boolean +): NormalizedAiMessage[] | null { + if (!isStream) { + const message = tryParseJson(text)?.output?.message; + if (!message) return null; + return [ + { + role: normalizeRole(message.role ?? "assistant"), + content: Array.isArray(message.content) + ? flattenContentParts(message.content) + : "" + } + ]; + } + // converse-stream uses AWS's binary event-stream framing, but the JSON + // payload of each event survives intact inside it (same assumption + // aiUsageExtraction.ts makes for usage) - scan for the text pieces. + const scanned = scanTextFragments(text); + return scanned ? [{ role: "assistant", content: scanned }] : null; +} + +function normalizeBedrockModelInvokeResponse( + text: string, + isStream: boolean +): NormalizedAiMessage[] | null { + const anthropicStyle = normalizeAnthropicResponse(text, isStream); + if (anthropicStyle) return anthropicStyle; + const scanned = scanTextFragments(text); + return scanned ? [{ role: "assistant", content: scanned }] : null; +} + +function normalizeGoogleRawPredictResponse( + text: string, + isStream: boolean +): NormalizedAiMessage[] | null { + const anthropicStyle = normalizeAnthropicResponse(text, isStream); + if (anthropicStyle) return anthropicStyle; + const scanned = scanTextFragments(text); + return scanned ? [{ role: "assistant", content: scanned }] : null; +} + +const REQUEST_NORMALIZERS: Record< + AiCapability, + (body: any) => NormalizedAiMessage[] | null +> = { + openai_chat: normalizeOpenAiChatRequest, + openai_responses: normalizeOpenAiResponsesRequest, + anthropic_messages: normalizeAnthropicRequest, + gemini_generate_content: normalizeGeminiRequest, + google_generate_content: normalizeGeminiRequest, + google_raw_predict: normalizeBestEffortRequest, + bedrock_model_invoke: normalizeBestEffortRequest, + bedrock_converse: normalizeBedrockConverseRequest +}; + +const RESPONSE_NORMALIZERS: Record< + AiCapability, + (text: string, isStream: boolean) => NormalizedAiMessage[] | null +> = { + openai_chat: normalizeOpenAiChatResponse, + openai_responses: normalizeOpenAiResponsesResponse, + anthropic_messages: normalizeAnthropicResponse, + gemini_generate_content: normalizeGeminiResponse, + google_generate_content: normalizeGeminiResponse, + google_raw_predict: normalizeGoogleRawPredictResponse, + bedrock_model_invoke: normalizeBedrockModelInvokeResponse, + bedrock_converse: normalizeBedrockConverseResponse +}; + +/** + * Normalizes an outbound AI gateway request body into a uniform message + * transcript, regardless of capability/provider. Returns null if the body + * doesn't contain any recognizable messages (or parsing failed) - callers + * should fall back to showing the raw request body. + */ +export function normalizeAiRequest( + capability: AiCapability, + body: unknown +): NormalizedAiMessage[] | null { + try { + const result = REQUEST_NORMALIZERS[capability](body); + return result && result.length > 0 ? result : null; + } catch (error) { + logger.debug("Failed to normalize AI request messages", { + capability, + error + }); + return null; + } +} + +/** + * Normalizes a completed (non-streaming or fully-accumulated streaming) AI + * gateway response into a uniform message transcript. Returns null if + * nothing recognizable could be extracted - callers should fall back to + * showing the raw response body. + */ +export function normalizeAiResponse( + capability: AiCapability, + responseText: string, + isStream: boolean +): NormalizedAiMessage[] | null { + try { + const result = RESPONSE_NORMALIZERS[capability](responseText, isStream); + return result && result.length > 0 ? result : null; + } catch (error) { + logger.debug("Failed to normalize AI response messages", { + capability, + error + }); + return null; + } +} diff --git a/server/lib/aiUsageExtraction.ts b/server/lib/aiUsageExtraction.ts index f3fc92580..48f19f308 100644 --- a/server/lib/aiUsageExtraction.ts +++ b/server/lib/aiUsageExtraction.ts @@ -49,7 +49,10 @@ function scanNumericFields( return out; } -function sseDataFrames(text: string): string[] { +// Exported for reuse by server/lib/aiMessageNormalization.ts, which needs +// the same SSE-frame/JSON-parsing groundwork to extract message content +// instead of usage numbers. +export function sseDataFrames(text: string): string[] { const frames: string[] = []; for (const rawFrame of text.split(/\r?\n\r?\n/)) { for (const line of rawFrame.split(/\r?\n/)) { @@ -63,7 +66,7 @@ function sseDataFrames(text: string): string[] { return frames; } -function tryParseJson(text: string): any | null { +export function tryParseJson(text: string): any | null { try { return JSON.parse(text); } catch { @@ -115,7 +118,10 @@ function extractOpenAiResponses( if (isStream) { for (const frame of sseDataFrames(text)) { const parsed = tryParseJson(frame); - if (parsed?.type === "response.completed" && parsed?.response?.usage) { + if ( + parsed?.type === "response.completed" && + parsed?.response?.usage + ) { usage = parsed.response.usage; } else if (parsed?.usage) { usage = parsed.usage; @@ -244,7 +250,10 @@ function extractBedrockConverse( if (usage) { const cacheReadTokens = usage.cacheReadInputTokens ?? 0; return { - promptTokens: Math.max(0, (usage.inputTokens ?? 0) - cacheReadTokens), + promptTokens: Math.max( + 0, + (usage.inputTokens ?? 0) - cacheReadTokens + ), cacheReadTokens, cacheWriteTokens: usage.cacheWriteInputTokens ?? 0, completionTokens: usage.outputTokens ?? 0, @@ -437,7 +446,12 @@ export function stripInjectedUsageFrame(sseText: string): string { if (dataLine) { const data = dataLine.slice("data:".length).trim(); const parsed = data !== "[DONE]" ? tryParseJson(data) : null; - if (parsed && Array.isArray(parsed.choices) && parsed.choices.length === 0 && parsed.usage) { + if ( + parsed && + Array.isArray(parsed.choices) && + parsed.choices.length === 0 && + parsed.usage + ) { continue; } } diff --git a/server/routers/aiGateway/logAiSession.ts b/server/routers/aiGateway/logAiSession.ts index 4c7384497..92723bed4 100644 --- a/server/routers/aiGateway/logAiSession.ts +++ b/server/routers/aiGateway/logAiSession.ts @@ -7,6 +7,10 @@ import cache from "#dynamic/lib/cache"; import { calculateCutoffTimestamp } from "@server/lib/cleanupLogs"; import { sanitizeString } from "@server/lib/sanitize"; import type { AiCapability } from "@server/lib/aiCapabilities"; +import { + normalizeAiRequest, + normalizeAiResponse +} from "@server/lib/aiMessageNormalization"; // Caps how much of the request/response body we keep per row, so a single // huge multimodal payload can't blow up buffer memory or storage. @@ -197,6 +201,26 @@ export function logAiSession(data: { ); const responseBodyText = truncateBody(data.responseText ?? ""); + // Uniform, capability-agnostic transcript for search/display - + // computed from the untruncated originals so normalization sees + // the full content; the normalized result gets its own + // (typically much smaller) truncation pass below. + const normalizedRequestMessages = normalizeAiRequest( + data.capability, + data.requestBody + ); + const normalizedResponseMessages = normalizeAiResponse( + data.capability, + data.responseText ?? "", + data.isStream + ); + const normalizedRequestText = normalizedRequestMessages + ? truncateBody(JSON.stringify(normalizedRequestMessages)) + : null; + const normalizedResponseText = normalizedResponseMessages + ? truncateBody(JSON.stringify(normalizedResponseMessages)) + : null; + // Prevent unbounded buffer growth - drop oldest entries if buffer is too large if (sessionLogBuffer.length >= MAX_BUFFER_SIZE) { const dropped = sessionLogBuffer.splice(0, BATCH_SIZE); @@ -217,8 +241,17 @@ export function logAiSession(data: { isStream: data.isStream, requestBody: sanitizeString(requestBodyText.value), responseBody: sanitizeString(responseBodyText.value), + normalizedRequest: normalizedRequestText + ? sanitizeString(normalizedRequestText.value) + : undefined, + normalizedResponse: normalizedResponseText + ? sanitizeString(normalizedResponseText.value) + : undefined, truncated: - requestBodyText.truncated || responseBodyText.truncated, + requestBodyText.truncated || + responseBodyText.truncated || + (normalizedRequestText?.truncated ?? false) || + (normalizedResponseText?.truncated ?? false), statusCode: data.statusCode, createdAt: Date.now() }); From 899c47e9a37263b63917ee6297d2ff4aa2d8b63f Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Tue, 11 Aug 2026 20:52:13 +0200 Subject: [PATCH 119/296] =?UTF-8?q?=F0=9F=9A=A7=20write=20process=20test?= =?UTF-8?q?=20alert=20function?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../private/lib/alerts/processTestAlerts.ts | 95 ++++++++++++------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/server/private/lib/alerts/processTestAlerts.ts b/server/private/lib/alerts/processTestAlerts.ts index f598dc633..7aa1691f0 100644 --- a/server/private/lib/alerts/processTestAlerts.ts +++ b/server/private/lib/alerts/processTestAlerts.ts @@ -1,11 +1,15 @@ +import { db, userOrgRoles, users } from "@server/db"; import logger from "@server/logger"; import type { EmailAlertAction, - TestAlertContext + TestAlertContext, + WebhookAlertConfig } from "@server/routers/alertRule/types"; +import { eq, inArray } from "drizzle-orm"; import { sendAlertEmail } from "./sendAlertEmail"; -import type { db, alertEmailRecipients, users, userOrgRoles } from "@server/db"; -import type { eq } from "drizzle-orm"; +import { decrypt } from "@server/lib/crypto"; +import config from "@server/lib/config"; +import { sendAlertWebhook } from "./sendAlertWebhook"; export async function processTestAlerts(context: TestAlertContext) { const emailActions = context.actions.filter( @@ -19,7 +23,38 @@ export async function processTestAlerts(context: TestAlertContext) { await sendAlertEmail(recipients, context); } } catch (err) { - logger.error(`processAlerts: failed to send alert email`, err); + logger.error(`processTestAlerts: failed to send alert email`, err); + } + } + + const webhookActions = context.actions.filter( + (action) => action.type === "webhook" + ); + const serverSecret = config.getRawConfig().server.secret!; + + for (const action of webhookActions) { + try { + let webhookConfig: WebhookAlertConfig = { authType: "none" }; + + if (action.config) { + try { + const decrypted = decrypt(action.config, serverSecret); + webhookConfig = JSON.parse(decrypted) as WebhookAlertConfig; + } catch (err) { + logger.error( + `processTestAlerts: failed to decrypt webhook`, + err + ); + continue; + } + } + + await sendAlertWebhook(action.webhookUrl, webhookConfig, context); + } catch (err) { + logger.error( + `processTestAlerts: failed to send alert webhook `, + err + ); } } } @@ -35,39 +70,31 @@ export async function processTestAlerts(context: TestAlertContext) { async function resolveEmailRecipients( action: EmailAlertAction ): Promise { - const emailSet = new Set(); + const emailList: string[] = []; - // for (const row of rows) { - // if (row.email) { - // emailSet.add(row.email); - // } + emailList.push(...(action.emails ?? [])); - // if (row.userId) { - // const [user] = await db - // .select({ email: users.email }) - // .from(users) - // .where(eq(users.userId, row.userId)) - // .limit(1); - // if (user?.email) { - // emailSet.add(user.email); - // } - // } + if (action.userIds && action.userIds?.length > 0) { + const userList = await db + .select({ email: users.email }) + .from(users) + .where(inArray(users.userId, action.userIds)); - // if (row.roleId) { - // // Find all users with this role via userOrgRoles - // const roleUsers = await db - // .select({ email: users.email }) - // .from(userOrgRoles) - // .innerJoin(users, eq(userOrgRoles.userId, users.userId)) - // .where(eq(userOrgRoles.roleId, Number(row.roleId))); + emailList.push( + ...userList.filter((u) => u.email !== null).map((u) => u.email!) + ); + } + if (action.roleIds && action.roleIds?.length > 0) { + const userList = await db + .select({ email: users.email }) + .from(userOrgRoles) + .innerJoin(users, eq(userOrgRoles.userId, users.userId)) + .where(inArray(userOrgRoles.roleId, action.roleIds.map(Number))); - // for (const u of roleUsers) { - // if (u.email) { - // emailSet.add(u.email); - // } - // } - // } - // } + emailList.push( + ...userList.filter((u) => u.email !== null).map((u) => u.email!) + ); + } - return Array.from(emailSet); + return [...new Set(emailList)]; } From 864ab9729393a91c4e2b45ef0a099a2b1e9642cd Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 11 Aug 2026 14:58:55 -0400 Subject: [PATCH 120/296] hide expire column --- src/components/VirtualApiKeysTable.tsx | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/components/VirtualApiKeysTable.tsx b/src/components/VirtualApiKeysTable.tsx index d5c2d3717..074d82f74 100644 --- a/src/components/VirtualApiKeysTable.tsx +++ b/src/components/VirtualApiKeysTable.tsx @@ -372,30 +372,6 @@ export default function VirtualApiKeysTable({ }, cell: ({ row }) => moment(row.original.createdAt).format("lll") }, - { - accessorKey: "expiresAt", - friendlyName: t("expires"), - header: ({ column }) => { - return ( - - ); - }, - cell: ({ row }) => { - const expiresAt = row.original.expiresAt; - if (expiresAt) { - return moment(expiresAt).format("lll"); - } - return t("never"); - } - }, { id: "actions", enableHiding: false, From 896c362dce713b24c1ea143f59b553a97aec234b Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 11 Aug 2026 15:35:28 -0400 Subject: [PATCH 121/296] only support sso on inference resource auth policy --- messages/en-US.json | 4 + .../resource-policy/PolicyAuthSsoSection.tsx | 42 +- .../PolicyAuthStackSectionEdit.tsx | 464 +++++++++++------- 3 files changed, 318 insertions(+), 192 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 0eaf60378..0742d1614 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -855,12 +855,16 @@ "authMethodsSave": "Save Settings", "policyAuthStackTitle": "Authentication", "policyAuthStackDescription": "Control which authentication methods are required to access this resource", + "policyAuthInferenceStackDescription": "Choose which users and roles can authenticate to this AI gateway", "policyAuthOrLogicTitle": "Multiple authentication methods active", "policyAuthOrLogicBanner": "Visitors may authenticate using any one of the active methods below. They do not need to complete all of them.", "policyAuthMethodActive": "Active", "policyAuthMethodOff": "Off", "policyAuthSsoTitle": "Platform SSO", "policyAuthSsoDescription": "Require sign-in through your organization's identity provider", + "policyAuthInferenceSsoDescription": "Selected users and roles can authenticate to the gateway using their identity API key", + "policyAuthInferenceIdentityKeyHelp": "Every user already has an identity API key, so you only need to create virtual API keys for non-user clients or shared access. Users can retrieve their key by signing in with their identity provider at , where it will be shown after login.", + "policyAuthInferenceIdentityKeyHelpNoUrl": "Every user already has an identity API key, so you only need to create virtual API keys for non-user clients or shared access. Users can retrieve their key by signing in with their identity provider at this resource's URL, where it will be shown after login.", "policyAuthSsoSummary": "{idp} · {users} users, {roles} roles", "policyAuthSsoDefaultIdp": "Default provider", "policyAuthAddDefaultIdentityProvider": "Add Default Identity Provider", diff --git a/src/components/resource-policy/PolicyAuthSsoSection.tsx b/src/components/resource-policy/PolicyAuthSsoSection.tsx index f7f8b3110..3cc2e54c4 100644 --- a/src/components/resource-policy/PolicyAuthSsoSection.tsx +++ b/src/components/resource-policy/PolicyAuthSsoSection.tsx @@ -29,6 +29,9 @@ export type PolicyAuthSsoSectionProps = { usersEditor: React.ReactNode; disabled?: boolean; idpDisabled?: boolean; + ssoLocked?: boolean; + title?: string; + description?: string; }; export function PolicyAuthSsoSection({ @@ -40,7 +43,10 @@ export function PolicyAuthSsoSection({ rolesEditor, usersEditor, disabled, - idpDisabled + idpDisabled, + ssoLocked, + title, + description }: PolicyAuthSsoSectionProps) { const t = useTranslations(); const [showIdpSelect, setShowIdpSelect] = useState(skipToIdpId != null); @@ -52,22 +58,34 @@ export function PolicyAuthSsoSection({ }, [skipToIdpId]); const idpSelectDisabled = idpDisabled ?? disabled; + const ssoActive = ssoLocked || sso; + const ssoTitle = title ?? t("policyAuthSsoTitle"); + const ssoDescription = description ?? t("policyAuthSsoDescription"); return ( - - - + {!ssoLocked && ( + + + + )} - {sso && ( + {ssoActive && ( <> + {ssoLocked && ssoDescription && ( + +

+ {ssoDescription} +

+ + )} {t("roles")} diff --git a/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx b/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx index aa7700948..51b718e88 100644 --- a/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx +++ b/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx @@ -15,6 +15,7 @@ import { } from "@app/components/roles-selector"; import { UsersSelector } from "@app/components/users-selector"; import { Button } from "@app/components/ui/button"; +import { Alert, AlertDescription } from "@app/components/ui/alert"; import { Form, FormField } from "@app/components/ui/form"; import { toast } from "@app/hooks/useToast"; import { useEnvContext } from "@app/hooks/useEnvContext"; @@ -28,8 +29,10 @@ import type { GetResourcePolicyResponse } from "@server/routers/policy"; import { UserType } from "@server/types/UserTypes"; import { useQuery } from "@tanstack/react-query"; import type { AxiosResponse } from "axios"; +import { ExternalLink, InfoIcon } from "lucide-react"; import { useTranslations } from "next-intl"; import { useRouter } from "next/navigation"; +import { toUnicode } from "punycode"; import { useActionState, useContext, @@ -108,9 +111,21 @@ export function PolicyAuthStackSectionEdit({ const resourceContext = useContext(ResourceContext); const api = createApiClient(useEnvContext()); + const isInferenceResource = resourceContext?.resource.mode === "inference"; const isResourceOverlay = resourceId !== undefined; const authReadonly = readonly || isResourceOverlay; + const inferenceResourceUrl = useMemo(() => { + if (!isInferenceResource || !resourceContext?.resource) { + return null; + } + const { ssl, fullDomain } = resourceContext.resource; + if (!fullDomain) { + return null; + } + return `${ssl ? "https" : "http"}://${toUnicode(fullDomain)}`; + }, [isInferenceResource, resourceContext?.resource]); + const policyRoleItems = useMemo( () => policy.roles.map((r) => ({ @@ -264,6 +279,12 @@ export function PolicyAuthStackSectionEdit({ const overlayRoles = combinedRoles.filter((r) => !r.isAdmin); const overlayUsers = combinedUsers; + useEffect(() => { + if (isInferenceResource && !form.getValues("sso")) { + form.setValue("sso", true); + } + }, [isInferenceResource, form]); + const [, formAction, isSubmitting] = useActionState(onSubmit, null); const [isSavingOverlay, setIsSavingOverlay] = useState(false); @@ -294,7 +315,7 @@ export function PolicyAuthStackSectionEdit({ .put( `/resource-policy/${policy.resourcePolicyId}/access-control`, { - sso: payload.sso, + sso: isInferenceResource ? true : payload.sso, userIds: payload.users.map((user) => user.id), roleIds: payload.roles.map((role) => Number(role.id)), skipToIdpId: payload.skipToIdpId @@ -302,94 +323,101 @@ export function PolicyAuthStackSectionEdit({ ) .catch(handleError) ); + policyUpdates.sso = isInferenceResource ? true : payload.sso; + + if (!isInferenceResource) { + if (passcodeActive && payload.password?.password) { + requests.push( + api + .put( + `/resource-policy/${policy.resourcePolicyId}/password`, + { password: payload.password.password } + ) + .catch(handleError) + ); + policyUpdates.passwordId = policy.passwordId ?? -1; + } else if (!passcodeActive && passcodeOnServerRef.current) { + requests.push( + api + .put( + `/resource-policy/${policy.resourcePolicyId}/password`, + { password: null } + ) + .catch(handleError) + ); + policyUpdates.passwordId = null; + } + + if (pinActive && payload.pincode?.pincode?.length === 6) { + requests.push( + api + .put( + `/resource-policy/${policy.resourcePolicyId}/pincode`, + { pincode: payload.pincode.pincode } + ) + .catch(handleError) + ); + policyUpdates.pincodeId = policy.pincodeId ?? -1; + } else if (!pinActive && pincodeOnServerRef.current) { + requests.push( + api + .put( + `/resource-policy/${policy.resourcePolicyId}/pincode`, + { pincode: null } + ) + .catch(handleError) + ); + policyUpdates.pincodeId = null; + } + + if ( + headerAuthActive && + payload.headerAuth?.user && + payload.headerAuth?.password + ) { + requests.push( + api + .put( + `/resource-policy/${policy.resourcePolicyId}/header-auth`, + { headerAuth: payload.headerAuth } + ) + .catch(handleError) + ); + policyUpdates.headerAuth = { + id: policy.headerAuth?.id ?? -1, + extendedCompability: + payload.headerAuth.extendedCompatibility ?? true + }; + } else if (!headerAuthActive && headerAuthOnServerRef.current) { + requests.push( + api + .put( + `/resource-policy/${policy.resourcePolicyId}/header-auth`, + { headerAuth: null } + ) + .catch(handleError) + ); + policyUpdates.headerAuth = { + id: null, + extendedCompability: null + } as unknown as GetResourcePolicyResponse["headerAuth"]; + } - if (passcodeActive && payload.password?.password) { requests.push( api .put( - `/resource-policy/${policy.resourcePolicyId}/password`, - { password: payload.password.password } + `/resource-policy/${policy.resourcePolicyId}/whitelist`, + { + emailWhitelistEnabled: + payload.emailWhitelistEnabled, + emails: payload.emails?.map((e) => e.text) ?? [] + } ) .catch(handleError) ); - policyUpdates.passwordId = policy.passwordId ?? -1; - } else if (!passcodeActive && passcodeOnServerRef.current) { - requests.push( - api - .put( - `/resource-policy/${policy.resourcePolicyId}/password`, - { password: null } - ) - .catch(handleError) - ); - policyUpdates.passwordId = null; + policyUpdates.emailWhitelistEnabled = payload.emailWhitelistEnabled; } - if (pinActive && payload.pincode?.pincode?.length === 6) { - requests.push( - api - .put( - `/resource-policy/${policy.resourcePolicyId}/pincode`, - { pincode: payload.pincode.pincode } - ) - .catch(handleError) - ); - policyUpdates.pincodeId = policy.pincodeId ?? -1; - } else if (!pinActive && pincodeOnServerRef.current) { - requests.push( - api - .put( - `/resource-policy/${policy.resourcePolicyId}/pincode`, - { pincode: null } - ) - .catch(handleError) - ); - policyUpdates.pincodeId = null; - } - - if ( - headerAuthActive && - payload.headerAuth?.user && - payload.headerAuth?.password - ) { - requests.push( - api - .put( - `/resource-policy/${policy.resourcePolicyId}/header-auth`, - { headerAuth: payload.headerAuth } - ) - .catch(handleError) - ); - policyUpdates.headerAuth = { - id: policy.headerAuth?.id ?? -1, - extendedCompability: - payload.headerAuth.extendedCompatibility ?? true - }; - } else if (!headerAuthActive && headerAuthOnServerRef.current) { - requests.push( - api - .put( - `/resource-policy/${policy.resourcePolicyId}/header-auth`, - { headerAuth: null } - ) - .catch(handleError) - ); - policyUpdates.headerAuth = { - id: null, - extendedCompability: null - } as unknown as GetResourcePolicyResponse["headerAuth"]; - } - - requests.push( - api - .put(`/resource-policy/${policy.resourcePolicyId}/whitelist`, { - emailWhitelistEnabled: payload.emailWhitelistEnabled, - emails: payload.emails?.map((e) => e.text) ?? [] - }) - .catch(handleError) - ); - policyUpdates.emailWhitelistEnabled = payload.emailWhitelistEnabled; - try { const results = await Promise.all(requests); if (results.every((res) => res && res.status === 200)) { @@ -411,13 +439,17 @@ export function PolicyAuthStackSectionEdit({ updatePolicy(policyUpdates); - resourceContext?.updateAuthInfo({ - sso: payload.sso, - whitelist: payload.emailWhitelistEnabled, - password: passcodeOnServerRef.current, - pincode: pincodeOnServerRef.current, - headerAuth: headerAuthOnServerRef.current - }); + resourceContext?.updateAuthInfo( + isInferenceResource + ? { sso: true } + : { + sso: payload.sso, + whitelist: payload.emailWhitelistEnabled, + password: passcodeOnServerRef.current, + pincode: pincodeOnServerRef.current, + headerAuth: headerAuthOnServerRef.current + } + ); toast({ title: t("success"), @@ -514,19 +546,53 @@ export function PolicyAuthStackSectionEdit({ {t("policyAuthStackTitle")} - {t("policyAuthStackDescription")} + {isInferenceResource + ? t("policyAuthInferenceStackDescription") + : t("policyAuthStackDescription")} {isResourceOverlay && ( )} + {isInferenceResource && ( + + + + {inferenceResourceUrl + ? t.rich( + "policyAuthInferenceIdentityKeyHelp", + { + resourceLink: () => ( + + {inferenceResourceUrl} + + + ) + } + ) + : t( + "policyAuthInferenceIdentityKeyHelpNoUrl" + )} + + + )} - form.setValue("sso", active) - } + sso={Boolean(sso) || isInferenceResource} + onSsoChange={(active) => { + if (isInferenceResource) { + return; + } + form.setValue("sso", active); + }} skipToIdpId={skipToIdpId} onSkipToIdpChange={(id) => form.setValue("skipToIdpId", id) @@ -534,6 +600,12 @@ export function PolicyAuthStackSectionEdit({ allIdps={allIdps} disabled={authReadonly} idpDisabled={authReadonly} + ssoLocked={isInferenceResource} + description={ + isInferenceResource + ? t("policyAuthInferenceSsoDescription") + : undefined + } rolesEditor={ isResourceOverlay ? ( - - handleToggle("pincode", active, () => { - setPinActive(false); - form.setValue("pincode", null); - }) - } - onTogglePasscode={(active) => - handleToggle("passcode", active, () => { - setPasscodeActive(false); - form.setValue("password", null); - }) - } - onToggleEmail={(active) => - handleToggle("email", active, () => - form.setValue( - "emailWhitelistEnabled", - false + {!isInferenceResource && ( + + handleToggle("pincode", active, () => { + setPinActive(false); + form.setValue("pincode", null); + }) + } + onTogglePasscode={(active) => + handleToggle("passcode", active, () => { + setPasscodeActive(false); + form.setValue("password", null); + }) + } + onToggleEmail={(active) => + handleToggle("email", active, () => + form.setValue( + "emailWhitelistEnabled", + false + ) ) - ) - } - onToggleHeaderAuth={(active) => - handleToggle("headerAuth", active, () => { - setHeaderAuthActive(false); - form.setValue("headerAuth", null); - }) - } - /> + } + onToggleHeaderAuth={(active) => + handleToggle( + "headerAuth", + active, + () => { + setHeaderAuthActive(false); + form.setValue( + "headerAuth", + null + ); + } + ) + } + /> + )} - !open && closeCredenza()} - defaultPincode={pincode?.pincode ?? ""} - onSave={(value) => { - form.setValue("pincode", { pincode: value }); - setPinActive(true); - }} - /> + {!isInferenceResource && ( + <> + + !open && closeCredenza() + } + defaultPincode={pincode?.pincode ?? ""} + onSave={(value) => { + form.setValue("pincode", { + pincode: value + }); + setPinActive(true); + }} + /> - !open && closeCredenza()} - defaultPassword={password?.password ?? ""} - existingConfigured={Boolean(policy.passwordId)} - onSave={(value) => { - form.setValue("password", { password: value }); - setPasscodeActive(true); - }} - /> + + !open && closeCredenza() + } + defaultPassword={password?.password ?? ""} + existingConfigured={Boolean( + policy.passwordId + )} + onSave={(value) => { + form.setValue("password", { + password: value + }); + setPasscodeActive(true); + }} + /> - !open && closeCredenza()} - emailEnabled={emailEnabled} - disabled={authReadonly} - emails={emails} - onSave={(value) => { - form.setValue("emails", value); - form.setValue("emailWhitelistEnabled", true); - }} - /> + + !open && closeCredenza() + } + emailEnabled={emailEnabled} + disabled={authReadonly} + emails={emails} + onSave={(value) => { + form.setValue("emails", value); + form.setValue( + "emailWhitelistEnabled", + true + ); + }} + /> - !open && closeCredenza()} - defaultValues={ - headerAuth - ? { - user: headerAuth.user, - password: headerAuth.password, - extendedCompatibility: - headerAuth.extendedCompatibility ?? - true - } - : undefined - } - existingConfigured={Boolean(policy.headerAuth?.id)} - onSave={(value) => { - form.setValue("headerAuth", value); - setHeaderAuthActive(true); - }} - /> + + !open && closeCredenza() + } + defaultValues={ + headerAuth + ? { + user: headerAuth.user, + password: headerAuth.password, + extendedCompatibility: + headerAuth.extendedCompatibility ?? + true + } + : undefined + } + existingConfigured={Boolean( + policy.headerAuth?.id + )} + onSave={(value) => { + form.setValue("headerAuth", value); + setHeaderAuthActive(true); + }} + /> + + )} + + ); + } + }, + { + accessorKey: "isStream", + header: ({ column }) => { + return ( +
+ + handleFilterChange("isStream", value) + } + searchPlaceholder={t("searchPlaceholder")} + emptyMessage={t("emptySearchOptions")} + /> +
+ ); + }, + cell: ({ row }) => { + return ( + + {row.original.isStream ? ( + <> + + {t("streaming")} + + ) : ( + + {t("nonStreaming")} + + )} + + ); + } + }, + { + accessorKey: "userEmail", + header: ({ column }) => { + return ( +
+ ({ + value: user.id, + label: user.email || user.id + }))} + selectedValue={filters.actor} + onValueChange={(value) => + handleFilterChange("actor", value) + } + label={t("actor")} + searchPlaceholder={t("searchPlaceholder")} + emptyMessage={t("emptySearchOptions")} + /> +
+ ); + }, + cell: ({ row }) => { + return ( + + {row.original.userEmail ? ( + <> + + {row.original.userEmail} + + ) : ( + <>- + )} + + ); + } + } + ]; + + const renderExpandedRow = (row: any) => { + return ( +
+
+
+ {t("aiSessionId")} +

+ {row.sessionId} +

+
+
+ {t("statusCode")} +

+ {row.statusCode ?? "N/A"} +

+
+
+ {t("model")} +

+ {row.requestedModel || "N/A"} +

+
+
+ {t("capability")} +

+ {capabilityLabels[row.capability] || row.capability} +

+
+
+ +
+ ); + }; + + return ( + <> + + + refetch()} + isRefreshing={isFetching} + onExport={() => startTransition(exportData)} + isExporting={isExporting} + onDateRangeChange={handleDateRangeChange} + dateRange={{ + start: dateRange.startDate, + end: dateRange.endDate + }} + defaultSort={{ + id: "createdAt", + desc: true + }} + totalCount={totalCount} + currentPage={currentPage} + onPageChange={handlePageChange} + onPageSizeChange={handlePageSizeChange} + isLoading={isLoading} + pageSize={pageSize} + expandable={true} + renderExpandedRow={renderExpandedRow} + /> + + ); +} + +function generateSampleAiSessionLogs(): QueryAiSessionLogResponse["log"] { + const capabilities = Object.keys(capabilityLabels); + const providers = [ + { id: 1, name: "OpenAI Production" }, + { id: 2, name: "Anthropic Default" }, + { id: 3, name: "Vertex AI" } + ]; + const resourcesSample = [ + { id: 1, niceId: "resource-1", name: "Resource 1" }, + { id: 2, niceId: "resource-2", name: "Resource 2" } + ]; + const actors = ["alice@example.com", "bob@example.com", null]; + const models = ["gpt-4o", "claude-sonnet-5", "gemini-2.5-pro"]; + + const now = Date.now(); + const sevenDaysAgoMs = now - 7 * 24 * 60 * 60 * 1000; + + return Array.from({ length: 10 }, (_, i) => { + const provider = providers[Math.floor(Math.random() * providers.length)]; + const resource = + resourcesSample[Math.floor(Math.random() * resourcesSample.length)]; + const actor = actors[Math.floor(Math.random() * actors.length)]; + + return { + id: i, + sessionId: `sample-session-${i}`, + orgId: "sample-org", + providerId: provider.id, + providerName: provider.name, + providerType: "openai", + capability: + capabilities[Math.floor(Math.random() * capabilities.length)], + resourceId: resource.id, + siteResourceId: null, + resourceName: resource.name, + resourceNiceId: resource.niceId, + resourceType: "public", + userId: actor ? `user-${i}` : null, + userEmail: actor, + requestedModel: models[Math.floor(Math.random() * models.length)], + isStream: Math.random() > 0.5, + requestBody: null, + responseBody: null, + normalizedRequest: null, + normalizedResponse: null, + truncated: false, + statusCode: 200, + createdAt: Math.floor( + sevenDaysAgoMs + Math.random() * (now - sevenDaysAgoMs) + ) + }; + }); +} diff --git a/src/app/navigation.tsx b/src/app/navigation.tsx index a9ffcca55..7181fd1c6 100644 --- a/src/app/navigation.tsx +++ b/src/app/navigation.tsx @@ -3,6 +3,7 @@ import { Env } from "@app/lib/types/env"; import { build } from "@server/build"; import { BellRing, + Bot, Boxes, Building2, Cable, @@ -227,6 +228,11 @@ export const orgNavSections = ( ) }, + { + title: "sidebarLogsAi", + href: "/{orgId}/settings/logs/ai", + icon: + }, ...(!env?.flags.disableEnterpriseFeatures ? [ { @@ -521,6 +527,11 @@ export const commandBarNavSections = ( href: "/{orgId}/settings/logs/request", icon: }, + { + title: "commandLogsAi", + href: "/{orgId}/settings/logs/ai", + icon: + }, ...(!env?.flags.disableEnterpriseFeatures ? [ { diff --git a/src/components/AiSessionChatView.tsx b/src/components/AiSessionChatView.tsx new file mode 100644 index 000000000..0a279b27e --- /dev/null +++ b/src/components/AiSessionChatView.tsx @@ -0,0 +1,176 @@ +"use client"; + +import { useMemo } from "react"; +import { useTranslations } from "next-intl"; +import { AlertTriangle, Bot, Terminal, User as UserIcon, Wrench } from "lucide-react"; +import type { NormalizedAiMessage } from "@server/lib/aiMessageNormalization"; + +type AiSessionChatViewProps = { + normalizedRequest: string | null; + normalizedResponse: string | null; + requestBody: string | null; + responseBody: string | null; + truncated: boolean; +}; + +function parseMessages(json: string | null): NormalizedAiMessage[] | null { + if (!json) return null; + try { + const parsed = JSON.parse(json); + return Array.isArray(parsed) ? (parsed as NormalizedAiMessage[]) : null; + } catch { + return null; + } +} + +function prettyRaw(raw: string | null): string | null { + if (!raw) return null; + try { + return JSON.stringify(JSON.parse(raw), null, 2); + } catch { + return raw; + } +} + +function MessageBubble({ message }: { message: NormalizedAiMessage }) { + const isUser = message.role === "user"; + const isSystem = message.role === "system"; + const isTool = message.role === "tool"; + + if (isSystem) { + return ( +
+ + 
+                    {message.content}
+
+
+ ); + } + + return ( +
+
+ {isUser ? ( + + ) : isTool ? ( + + ) : ( + + )} +
+
+ {message.content || ( +   + )} +
+
+ ); +} + +function RawFallbackBlock({ + label, + raw, + noDataLabel, + unparsedLabel +}: { + label: string; + raw: string | null; + noDataLabel: string; + unparsedLabel: string; +}) { + const pretty = prettyRaw(raw); + return ( +
+
+ {label} + {pretty && ( + + {unparsedLabel} + + )} +
+ 
+                {pretty ?? noDataLabel}
+
+
+ ); +} + +export function AiSessionChatView({ + normalizedRequest, + normalizedResponse, + requestBody, + responseBody, + truncated +}: AiSessionChatViewProps) { + const t = useTranslations(); + + const requestMessages = useMemo( + () => parseMessages(normalizedRequest), + [normalizedRequest] + ); + const responseMessages = useMemo( + () => parseMessages(normalizedResponse), + [normalizedResponse] + ); + + const hasRequestMessages = !!requestMessages && requestMessages.length > 0; + const hasResponseMessages = + !!responseMessages && responseMessages.length > 0; + + return ( +
+ {truncated && ( +
+ + {t("aiSessionLogTruncated")} +
+ )} +
+ {hasRequestMessages ? ( + requestMessages!.map((message, i) => ( + + )) + ) : ( + + )} + {hasResponseMessages ? ( + responseMessages!.map((message, i) => ( + + )) + ) : ( + + )} +
+
+ ); +} diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 02a9404e8..3a4c024d4 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -10,6 +10,7 @@ import type { QueryRequestAnalyticsResponse } from "@server/routers/auditLogs"; import type { QueryAccessAuditLogResponse, QueryActionAuditLogResponse, + QueryAiSessionLogResponse, QueryConnectionAuditLogResponse, QueryRequestAuditLogResponse } from "@server/routers/auditLogs/types"; @@ -1031,6 +1032,32 @@ export const connectionLogsFiltersSchema = z.object({ export type ConnectionLogFilters = z.output; +export const aiSessionLogsFiltersSchema = z.object({ + timeStart: z + .string() + .refine((val) => !isNaN(Date.parse(val)), { + error: "timeStart must be a valid ISO date string" + }) + .optional() + .catch(undefined), + timeEnd: z + .string() + .refine((val) => !isNaN(Date.parse(val)), { + error: "timeEnd must be a valid ISO date string" + }) + .optional() + .catch(undefined), + page: z.coerce.number().optional().catch(0).default(0), + pageSize: z.coerce.number().optional().catch(20).default(20), + providerId: z.string().optional().catch(undefined), + capability: z.string().optional().catch(undefined), + resourceId: z.string().optional().catch(undefined), + actor: z.string().optional().catch(undefined), + isStream: z.string().optional().catch(undefined) +}); + +export type AiSessionLogFilters = z.output; + export const logQueries = { requestAnalytics: ({ orgId, @@ -1180,6 +1207,37 @@ export const logQueries = { } return false; } + }), + + aiSessions: ({ + orgId, + filters + }: { + orgId: string; + filters: AiSessionLogFilters; + }) => + queryOptions({ + queryKey: ["AI_SESSION_LOGS", orgId, "ALL", filters] as const, + queryFn: async ({ signal, meta }) => { + const { page, pageSize, ...rest } = filters; + const res = await meta!.api.get< + AxiosResponse + >(`/org/${orgId}/logs/ai`, { + params: { + ...rest, + limit: pageSize, + offset: page * pageSize + }, + signal + }); + return res.data.data; + }, + refetchInterval: (query) => { + if (query.state.data) { + return durationToMs(30, "seconds"); + } + return false; + } }) }; From e734cc93a10bddb2e833a29a981d302cee4363d4 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 15:57:57 -0400 Subject: [PATCH 123/296] small visual adjustments and chat button --- messages/en-US.json | 2 + server/routers/auditLogs/queryAiSessionLog.ts | 19 +++- server/routers/auditLogs/types.ts | 1 + src/app/[orgId]/settings/logs/ai/page.tsx | 45 +++++---- src/components/AiSessionChatView.tsx | 95 +++++++++++++------ src/lib/queries.ts | 1 + 6 files changed, 113 insertions(+), 50 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 0784fdf2e..164f110e7 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -3405,6 +3405,8 @@ "aiSessionNoData": "No data captured", "aiSessionCouldNotParse": "(raw, could not parse transcript)", "aiSessionLogTruncated": "This session was truncated before storage and may be incomplete.", + "aiSessionViewRaw": "View Raw JSON", + "aiSessionViewChat": "View Chat", "requestAnalyticsDescription": "View detailed request analytics for resources in this organization", "logRetentionRequestLabel": "HTTP Request Log Retention", "logRetentionRequestDescription": "How long to retain request logs", diff --git a/server/routers/auditLogs/queryAiSessionLog.ts b/server/routers/auditLogs/queryAiSessionLog.ts index 913eacf65..2412f8980 100644 --- a/server/routers/auditLogs/queryAiSessionLog.ts +++ b/server/routers/auditLogs/queryAiSessionLog.ts @@ -66,6 +66,7 @@ export const queryAiSessionLogsQuery = z.strictObject({ .pipe(z.int().positive()) .optional(), actor: z.string().optional(), + model: z.string().optional(), isStream: z .union([z.boolean(), z.string()]) .transform((val) => (typeof val === "string" ? val === "true" : val)) @@ -123,6 +124,9 @@ function getWhere(data: Q) { ) : undefined, data.actor ? eq(aiSessionLog.userId, data.actor) : undefined, + data.model + ? eq(aiSessionLog.requestedModel, data.model) + : undefined, data.isStream !== undefined ? eq(aiSessionLog.isStream, data.isStream) : undefined @@ -333,7 +337,8 @@ async function queryUniqueFilterAttributes( uniqueProviders, uniqueUsers, uniqueResources, - uniqueSiteResources + uniqueSiteResources, + uniqueModels ] = await Promise.all([ logsDb .selectDistinct({ id: aiSessionLog.providerId }) @@ -354,9 +359,18 @@ async function queryUniqueFilterAttributes( .selectDistinct({ id: aiSessionLog.siteResourceId }) .from(aiSessionLog) .where(and(baseConditions, isNull(aiSessionLog.resourceId))) + .limit(DISTINCT_LIMIT + 1), + logsDb + .selectDistinct({ model: aiSessionLog.requestedModel }) + .from(aiSessionLog) + .where(baseConditions) .limit(DISTINCT_LIMIT + 1) ]); + const models = uniqueModels + .map((row) => row.model) + .filter((model): model is string => model !== null); + const providerIds = uniqueProviders .map((row) => row.id) .filter((id): id is number => id !== null); @@ -440,7 +454,8 @@ async function queryUniqueFilterAttributes( return { providers: sortNamedFilterOptions(providers), resources: sortNamedFilterOptions(resourcesWithNames), - users: userList + users: userList, + models: models.sort() }; } diff --git a/server/routers/auditLogs/types.ts b/server/routers/auditLogs/types.ts index cf85e5a2f..42b93904e 100644 --- a/server/routers/auditLogs/types.ts +++ b/server/routers/auditLogs/types.ts @@ -138,6 +138,7 @@ export type QueryAiSessionLogResponse = { id: string; email: string | null; }[]; + models: string[]; }; }; diff --git a/src/app/[orgId]/settings/logs/ai/page.tsx b/src/app/[orgId]/settings/logs/ai/page.tsx index 2c4c2ed32..67d18b94c 100644 --- a/src/app/[orgId]/settings/logs/ai/page.tsx +++ b/src/app/[orgId]/settings/logs/ai/page.tsx @@ -50,12 +50,14 @@ export default function AiSessionLogsPage() { capability?: string; resourceId?: string; actor?: string; + model?: string; isStream?: string; }>({ providerId: searchParams.get("providerId") || undefined, capability: searchParams.get("capability") || undefined, resourceId: searchParams.get("resourceId") || undefined, actor: searchParams.get("actor") || undefined, + model: searchParams.get("model") || undefined, isStream: searchParams.get("isStream") || undefined }); @@ -132,7 +134,8 @@ export default function AiSessionLogsPage() { const filterAttributes = data?.filterAttributes ?? { providers: [], resources: [], - users: [] + users: [], + models: [] }; const handleDateRangeChange = ( @@ -309,15 +312,27 @@ export default function AiSessionLogsPage() { }, { accessorKey: "requestedModel", - header: ({ column }) => ( - {t("model")} - ), - cell: ({ row }) => { + header: ({ column }) => { return ( - - {row.original.requestedModel || "-"} - +
+ ({ + value: model, + label: model + }))} + selectedValue={filters.model} + onValueChange={(value) => + handleFilterChange("model", value) + } + label={t("model")} + searchPlaceholder={t("searchPlaceholder")} + emptyMessage={t("emptySearchOptions")} + /> +
); + }, + cell: ({ row }) => { + return {row.original.requestedModel || "-"}; } }, { @@ -444,7 +459,7 @@ export default function AiSessionLogsPage() { const renderExpandedRow = (row: any) => { return (
-
+
{t("aiSessionId")}

@@ -457,18 +472,6 @@ export default function AiSessionLogsPage() { {row.statusCode ?? "N/A"}

-
- {t("model")} -

- {row.requestedModel || "N/A"} -

-
-
- {t("capability")} -

- {capabilityLabels[row.capability] || row.capability} -

-
{label} - {pretty && ( + {pretty && unparsedLabel && ( {unparsedLabel} @@ -123,6 +132,7 @@ export function AiSessionChatView({ truncated }: AiSessionChatViewProps) { const t = useTranslations(); + const [rawMode, setRawMode] = useState(false); const requestMessages = useMemo( () => parseMessages(normalizedRequest), @@ -139,38 +149,69 @@ export function AiSessionChatView({ return (
- {truncated && ( -
- - {t("aiSessionLogTruncated")} -
- )} -
- {hasRequestMessages ? ( - requestMessages!.map((message, i) => ( - - )) +
+ {truncated ? ( +
+ + {t("aiSessionLogTruncated")} +
) : ( +
+ )} + +
+ {rawMode ? ( +
- )} - {hasResponseMessages ? ( - responseMessages!.map((message, i) => ( - - )) - ) : ( - )} -
+
+ ) : ( +
+ {hasRequestMessages ? ( + requestMessages!.map((message, i) => ( + + )) + ) : ( + + )} + {hasResponseMessages ? ( + responseMessages!.map((message, i) => ( + + )) + ) : ( + + )} +
+ )}
); } diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 3a4c024d4..0275d108b 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -1053,6 +1053,7 @@ export const aiSessionLogsFiltersSchema = z.object({ capability: z.string().optional().catch(undefined), resourceId: z.string().optional().catch(undefined), actor: z.string().optional().catch(undefined), + model: z.string().optional().catch(undefined), isStream: z.string().optional().catch(undefined) }); From c42df737b0b737456e08932acdebde5e42db9166 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 11 Aug 2026 16:10:34 -0400 Subject: [PATCH 124/296] improve model provider selection algorithm --- docs/ai-gateway-provider-selection.md | 285 ++++++++++++++++++ server/lib/aiInferenceResource.ts | 161 ---------- server/lib/aiModelKeyMatch.ts | 23 -- server/lib/aiProviderSelection.ts | 97 ++++++ server/routers/aiGateway/pipeline.ts | 24 +- .../resource/addAiProviderToResource.ts | 3 +- .../resource/removeAiProviderFromResource.ts | 3 +- .../resource/setResourceAiProviders.ts | 3 +- .../addAiProviderToSiteResource.ts | 3 +- .../removeAiProviderFromSiteResource.ts | 3 +- .../setSiteResourceAiProviders.ts | 3 +- 11 files changed, 408 insertions(+), 200 deletions(-) create mode 100644 docs/ai-gateway-provider-selection.md create mode 100644 server/lib/aiProviderSelection.ts diff --git a/docs/ai-gateway-provider-selection.md b/docs/ai-gateway-provider-selection.md new file mode 100644 index 000000000..f955e25aa --- /dev/null +++ b/docs/ai-gateway-provider-selection.md @@ -0,0 +1,285 @@ +# AI Gateway Provider Selection + +How the AI gateway picks which attached provider handles a request when an +inference resource has more than one AI provider. + +**Code:** + +- Route → capability binding: `server/routers/aiGateway/createAiGatewayRouter.ts` +- Request pipeline: `server/routers/aiGateway/pipeline.ts` (`selectProvider`) +- Tie-break scoring: `server/lib/aiProviderSelection.ts` +- Allow/block matching: `server/lib/aiModelKeyMatch.ts` +- Model catalog: `server/lib/aiModelCatalog.ts` +- Default capabilities per provider type: `server/lib/aiProviderDefaults.ts` + +Overlapping model allows are permitted at save time. Selection happens at +request time. If the algorithm cannot confidently pick one provider, the +gateway returns `403` with an ambiguous-provider error. + +## Selection Pipeline + +Every gateway request runs through these steps in order. Each step narrows +the candidate set. Later steps only run when more than one provider remains. + +``` +1. Capability filter +2. Allow / block lists +3. Most specific allow pattern +4. Catalog ownership +5. Provider class preference +6. Ambiguous → error +``` + +### 1. Capability Filter + +The incoming path selects a capability before any provider logic runs. + +| Path | Capability | +|------|------------| +| `POST /v1/chat/completions` | `openai_chat` | +| `POST /v1/responses` | `openai_responses` | +| `POST /v1/messages` | `anthropic_messages` | +| Gemini / Vertex / Bedrock routes | their respective capability ids | + +Only attached providers that advertise that capability stay in the candidate +set. Default capabilities do not overlap for native OpenAI vs Anthropic: + +| Provider type | Default capabilities | +|---------------|----------------------| +| `openai` | `openai_chat`, `openai_responses` | +| `anthropic` | `anthropic_messages` | +| `openRouter` | `openai_chat` | +| `vercelAiGateway` | `openai_chat`, `openai_responses` | +| `microsoftFoundry` | `openai_chat`, `openai_responses`, `anthropic_messages` | +| `custom` | whatever was configured | + +### 2. Allow / Block Lists + +For each remaining provider, the gateway resolves the effective allow and +block patterns: + +- **`inherit`**: use the provider's own model lists +- **`select`**: use the resource-selected subset of those lists + +A candidate is kept only if `isAllowedByLists(requestedModel, allows, blocks)` +passes: + +1. At least one allow pattern must match +2. No block pattern may match + +Patterns support `*` and `?` globs (`gpt-*`, `claude-3-5-sonnet-?`). + +### 3. Most Specific Allow Pattern + +Among providers that allow the model, keep those whose matching allow +pattern is most specific: + +1. Exact keys beat patterns +2. Fewer wildcard characters win +3. Longer literal length wins + +Example: `gpt-4o` beats `gpt-*` beats `*`. + +### 4. Catalog Ownership + +When specificity is tied (common with multiple `*` allows), score each +provider against the known model catalog: + +| Score | Meaning | +|------:|---------| +| 2 | Typed provider whose catalog contains the model (`openai` → openai catalog, `anthropic` → anthropic, etc.) | +| 1 | Aggregator or custom (`openRouter`, `vercelAiGateway`, `custom`) and the model exists somewhere in the catalog | +| 0 | No ownership signal (typed catalog miss, or unknown model on aggregator/custom) | + +Model id lookup tries the raw id, then a stripped `vendor/model` form +(e.g. `openai/gpt-4o` → also try `gpt-4o`). + +Typed providers map to catalog providers as: + +| Provider type | Catalog | +|---------------|---------| +| `openai` | `openai` | +| `anthropic` | `anthropic` | +| `googleGemini` | `gemini` | +| `vertexAi` | `vertex` | +| `bedrock` | `bedrock` | +| `microsoftFoundry` | `azure` | +| `openRouter` / `vercelAiGateway` / `custom` | none (aggregator/custom path) | + +### 5. Provider Class Preference + +If catalog ownership is still tied, prefer: + +| Rank | Class | +|-----:|-------| +| 2 | Native typed provider (`openai`, `anthropic`, `googleGemini`, …) | +| 1 | Aggregator (`openRouter`, `vercelAiGateway`) | +| 0 | `custom` | + +### 6. Ambiguous Error + +If more than one distinct provider remains after all steps, the gateway +rejects the request: + +``` +Model "" is ambiguous across multiple AI providers on this resource +``` + +Typical remaining ties: two OpenAI-type providers both with `*`, or two +customs advertising the same capability for an unknown model. + +## Examples + +Assume each provider below is attached and enabled on the same inference +resource. + +### Example A: OpenAI + Anthropic, Both `*` + +| Provider | Allow | Capabilities | +|----------|-------|--------------| +| OpenAI | `*` | `openai_chat`, `openai_responses` | +| Anthropic | `*` | `anthropic_messages` | + +**Request:** `POST /v1/chat/completions` with `model: "gpt-4o"` + +1. Capability → only OpenAI remains +2. Allow → OpenAI matches `*` +3. Result → **OpenAI** + +Anthropic never reaches pattern or catalog scoring. Capability alone decides. + +**Request:** `POST /v1/messages` with `model: "claude-3-5-sonnet-latest"` + +1. Capability → only Anthropic remains +2. Result → **Anthropic** + +### Example B: OpenAI + OpenRouter, Both `*` + +| Provider | Allow | Capabilities | +|----------|-------|--------------| +| OpenAI | `*` | `openai_chat`, … | +| OpenRouter | `*` | `openai_chat` | + +**Request:** `POST /v1/chat/completions` with `model: "gpt-4o"` + +1. Capability → both remain (`openai_chat`) +2. Allow → both match `*` +3. Specificity → tie (`*` vs `*`) +4. Catalog → OpenAI scores `2` (owns `gpt-4o`); OpenRouter scores `1` +5. Result → **OpenAI** + +### Example C: OpenRouter Only Serving a Claude Model Over OpenAI Chat + +| Provider | Allow | Capabilities | +|----------|-------|--------------| +| OpenRouter | `*` | `openai_chat` | + +**Request:** `POST /v1/chat/completions` with `model: "anthropic/claude-3.5-sonnet"` + +1. Capability → OpenRouter remains +2. Only one candidate → **OpenRouter** + +No tie-breaking needed. + +### Example D: OpenAI (`gpt-*`) + OpenRouter (`*`) + +| Provider | Allow | +|----------|-------| +| OpenAI | `gpt-*` | +| OpenRouter | `*` | + +**Request:** `model: "gpt-4o"` on `openai_chat` + +1. Capability → both +2. Allow → both match +3. Specificity → OpenAI's `gpt-*` beats OpenRouter's `*` +4. Result → **OpenAI** + +Catalog scoring is not needed because specificity already unique'd the set. + +### Example E: OpenAI + Anthropic With Overlapping Custom Capabilities + +Someone grants Anthropic `openai_chat` as well (non-default). + +| Provider | Allow | Capabilities | +|----------|-------|--------------| +| OpenAI | `*` | `openai_chat`, … | +| Anthropic | `*` | `anthropic_messages`, `openai_chat` | + +**Request:** `POST /v1/chat/completions` with `model: "gpt-4o"` + +1. Capability → both remain +2. Allow → both match `*` +3. Specificity → tie +4. Catalog → OpenAI `2`, Anthropic `0` (`gpt-4o` is not in the anthropic catalog) +5. Result → **OpenAI** + +### Example F: Two Aggregators, Known Model + +| Provider | Allow | +|----------|-------| +| OpenRouter | `*` | +| Vercel AI Gateway | `*` | + +**Request:** `model: "gpt-4o"` on `openai_chat` + +1. Capability → both +2. Allow / specificity → tie +3. Catalog → both score `1` (known model, no typed owner in the set) +4. Class → both aggregators (rank `1`) → still tied +5. Result → **ambiguous error** + +Attach a native OpenAI provider (or narrow one aggregator's allow list) to +make this determinable. + +### Example G: Two OpenAI Providers, Both `*` + +| Provider | Type | Allow | +|----------|------|-------| +| OpenAI Prod | `openai` | `*` | +| OpenAI Staging | `openai` | `*` | + +**Request:** `model: "gpt-4o"` + +1–5 all leave both candidates (same capability, same specificity, same +catalog ownership, same class). + +Result → **ambiguous error** + +Disambiguate with different allow patterns, disable one attachment, or +split across resources. + +### Example H: Unknown Model Across Native + Aggregator + +| Provider | Allow | +|----------|-------| +| OpenAI | `*` | +| OpenRouter | `*` | + +**Request:** `model: "my-fine-tune-v3"` (not in catalog) + +1. Capability → both +2. Allow / specificity → tie +3. Catalog → both score `0` (typed miss + unknown aggregator model) +4. Class → OpenAI (`2`) beats OpenRouter (`1`) +5. Result → **OpenAI** + +## Practical Guidance + +- Native OpenAI + Anthropic with `*` is safe. Different default APIs never + collide. +- OpenAI + OpenRouter with `*` is usually fine for catalog-known OpenAI + models. Native wins. +- Prefer specific allow patterns (`gpt-4o`, `gpt-*`) when two providers share + a capability. +- Two providers of the same type both using `*` will stay ambiguous. Narrow + at least one allow list. +- Custom providers only win ties when no stronger native/aggregator signal + remains. + +## Related Behavior + +- **Saving providers on a resource does not reject overlapping allows.** + Collisions are resolved (or rejected) per request. +- Budgets, auth, and upstream URL / target routing run after a single + provider has been selected. diff --git a/server/lib/aiInferenceResource.ts b/server/lib/aiInferenceResource.ts index cfd560fce..d38683b59 100644 --- a/server/lib/aiInferenceResource.ts +++ b/server/lib/aiInferenceResource.ts @@ -10,7 +10,6 @@ import { type Transaction } from "@server/db"; import { z } from "zod"; -import { modelKeysConflict } from "@server/lib/aiModelKeyMatch"; type DbOrTrx = Transaction | typeof db; @@ -100,142 +99,6 @@ function normalizeAttachments( ); } -type EffectiveAllowRow = { - providerId: number; - modelKey: string; -}; - -/** - * Ensure effective allow modelKeys do not conflict across attached providers. - * inherit uses provider allows; select uses resource-selected allows (or the - * optional override map). Block patterns are ignored for overlap checks. - */ -export async function assertNoOverlappingModelKeys( - attachments: ResourceAiProviderAttachment[], - options: { - trx?: DbOrTrx; - resourceId?: number; - siteResourceId?: number; - selectedAllowsByProvider?: Map; - } = {} -): Promise { - const trx = options.trx ?? db; - - const activeAttachments = attachments.filter((a) => a.enabled); - - if (activeAttachments.length < 2) { - return null; - } - - const inheritProviderIds = activeAttachments - .filter((a) => a.accessMode === "inherit") - .map((a) => a.providerId); - const selectProviderIds = activeAttachments - .filter((a) => a.accessMode === "select") - .map((a) => a.providerId); - - const effectiveAllows: EffectiveAllowRow[] = []; - - if (inheritProviderIds.length > 0) { - const providerAllows = await trx - .select({ - providerId: aiModels.providerId, - modelKey: aiModels.modelKey - }) - .from(aiModels) - .where( - and( - inArray(aiModels.providerId, inheritProviderIds), - eq(aiModels.enabled, true), - eq(aiModels.listType, "allow") - ) - ); - effectiveAllows.push(...providerAllows); - } - - if (selectProviderIds.length > 0) { - if (options.selectedAllowsByProvider) { - for (const providerId of selectProviderIds) { - const keys = - options.selectedAllowsByProvider.get(providerId) ?? []; - for (const modelKey of keys) { - effectiveAllows.push({ providerId, modelKey }); - } - } - } else if (options.resourceId !== undefined) { - const rows = await trx - .select({ - providerId: aiModels.providerId, - modelKey: aiModels.modelKey - }) - .from(resourceAiModels) - .innerJoin( - aiModels, - eq(resourceAiModels.modelId, aiModels.modelId) - ) - .where( - and( - eq(resourceAiModels.resourceId, options.resourceId), - inArray(aiModels.providerId, selectProviderIds), - eq(resourceAiModels.listType, "allow"), - eq(aiModels.enabled, true) - ) - ); - effectiveAllows.push(...rows); - } else if (options.siteResourceId !== undefined) { - const rows = await trx - .select({ - providerId: aiModels.providerId, - modelKey: aiModels.modelKey - }) - .from(siteResourceAiModels) - .innerJoin( - aiModels, - eq(siteResourceAiModels.modelId, aiModels.modelId) - ) - .where( - and( - eq( - siteResourceAiModels.siteResourceId, - options.siteResourceId - ), - inArray(aiModels.providerId, selectProviderIds), - eq(siteResourceAiModels.listType, "allow"), - eq(aiModels.enabled, true) - ) - ); - effectiveAllows.push(...rows); - } - } - - const conflictPairs: string[] = []; - for (let i = 0; i < effectiveAllows.length; i++) { - for (let j = i + 1; j < effectiveAllows.length; j++) { - const left = effectiveAllows[i]; - const right = effectiveAllows[j]; - if (left.providerId === right.providerId) { - continue; - } - if (!modelKeysConflict(left.modelKey, right.modelKey)) { - continue; - } - const pair = [left.modelKey, right.modelKey].sort().join(" vs "); - if (!conflictPairs.includes(pair)) { - conflictPairs.push(pair); - } - } - } - - if (conflictPairs.length === 0) { - return null; - } - - conflictPairs.sort(); - return { - error: `Model keys must be unique across providers on a resource. Overlapping keys: ${conflictPairs.join(", ")}` - }; -} - /** * Validate provider attachments for an org. */ @@ -243,8 +106,6 @@ export async function resolveProviderAttachments(input: { orgId: string; attachments: ResourceAiProviderInput[]; requireAtLeastOne: boolean; - resourceId?: number; - siteResourceId?: number; }): Promise { const attachments = normalizeAttachments(input.attachments); @@ -286,14 +147,6 @@ export async function resolveProviderAttachments(input: { }; } - const overlapError = await assertNoOverlappingModelKeys(attachments, { - resourceId: input.resourceId, - siteResourceId: input.siteResourceId - }); - if (overlapError) { - return overlapError; - } - return attachments; } @@ -830,7 +683,6 @@ async function assertModelEntriesValid(input: { const catalogRows = await db .select({ modelId: aiModels.modelId, - modelKey: aiModels.modelKey, listType: aiModels.listType, providerId: aiModels.providerId, enabled: aiModels.enabled @@ -850,7 +702,6 @@ async function assertModelEntriesValid(input: { } const catalogById = new Map(catalogRows.map((row) => [row.modelId, row])); - const selectedAllowsByProvider = new Map(); for (const entry of input.modelEntries) { const catalog = catalogById.get(entry.modelId); if (!catalog) { @@ -862,18 +713,6 @@ async function assertModelEntriesValid(input: { if (!catalog.enabled) { return `Model ${entry.modelId} is disabled on its provider`; } - if (entry.listType === "allow") { - const keys = selectedAllowsByProvider.get(catalog.providerId) ?? []; - keys.push(catalog.modelKey); - selectedAllowsByProvider.set(catalog.providerId, keys); - } - } - - const overlapError = await assertNoOverlappingModelKeys(input.attachments, { - selectedAllowsByProvider - }); - if (overlapError) { - return overlapError.error; } return null; diff --git a/server/lib/aiModelKeyMatch.ts b/server/lib/aiModelKeyMatch.ts index 38184c3b0..8f54afb8c 100644 --- a/server/lib/aiModelKeyMatch.ts +++ b/server/lib/aiModelKeyMatch.ts @@ -61,29 +61,6 @@ export function compareModelKeySpecificity(a: string, b: string): number { return literalLength(b) - literalLength(a); } -/** - * Attach-time conflict check. Detects identical keys and exact-vs-pattern - * matches. Does not attempt full glob intersection. - */ -export function modelKeysConflict(a: string, b: string): boolean { - if (a === b) { - return true; - } - - const aIsPattern = isModelKeyPattern(a); - const bIsPattern = isModelKeyPattern(b); - - if (aIsPattern === bIsPattern) { - return false; - } - - if (aIsPattern) { - return modelKeyMatches(a, b); - } - - return modelKeyMatches(b, a); -} - /** * Provider-layer policy: empty allowlist denies all. Blocklist only applies * after an allow match. diff --git a/server/lib/aiProviderSelection.ts b/server/lib/aiProviderSelection.ts new file mode 100644 index 000000000..e35d85f1a --- /dev/null +++ b/server/lib/aiProviderSelection.ts @@ -0,0 +1,97 @@ +import { + aiModelCatalog, + getCatalogProviderForType, + type CatalogProvider +} from "@server/lib/aiModelCatalog"; +import type { AiProviderType } from "@server/lib/aiProviderDefaults"; + +function stripVendorPrefix(modelId: string): string | null { + const idx = modelId.indexOf("/"); + if (idx === -1 || idx === modelId.length - 1) { + return null; + } + return modelId.slice(idx + 1); +} + +function modelKeysToTry(modelId: string): string[] { + const keys = [modelId]; + const stripped = stripVendorPrefix(modelId); + if (stripped) { + keys.push(stripped); + } + return keys; +} + +function catalogOwnsModel( + catalogProvider: CatalogProvider, + modelId: string +): boolean { + for (const key of modelKeysToTry(modelId)) { + if (aiModelCatalog.get(catalogProvider, key)) { + return true; + } + } + return false; +} + +function modelKnownInAnyCatalog(modelId: string): boolean { + for (const key of modelKeysToTry(modelId)) { + if (aiModelCatalog.listByKey(key).length > 0) { + return true; + } + } + return false; +} + +/** + * How strongly a provider "owns" a requested model id via the known catalog. + * + * 2 - Typed provider whose catalog contains the model + * 1 - Aggregator/custom that can proxy a catalog-known model + * 0 - No ownership signal (typed miss, or unknown model on aggregator/custom) + */ +export function catalogOwnershipScore( + type: AiProviderType, + modelId: string +): number { + const catalogProvider = getCatalogProviderForType(type); + if (catalogProvider != null) { + return catalogOwnsModel(catalogProvider, modelId) ? 2 : 0; + } + return modelKnownInAnyCatalog(modelId) ? 1 : 0; +} + +/** + * Prefer native vendor providers over aggregators over custom when catalog + * ownership is tied. + * + * 2 - Native typed provider (openai, anthropic, gemini, ...) + * 1 - Aggregator gateway (openRouter, vercelAiGateway) + * 0 - Custom + */ +export function providerClassRank(type: AiProviderType): number { + if (type === "custom") { + return 0; + } + if (type === "openRouter" || type === "vercelAiGateway") { + return 1; + } + return 2; +} + +export function keepBestScored( + items: T[], + scoreFn: (item: T) => number +): T[] { + if (items.length <= 1) { + return items; + } + let best = Number.NEGATIVE_INFINITY; + for (const item of items) { + const score = scoreFn(item); + if (score > best) { + best = score; + } + } + return items.filter((item) => scoreFn(item) === best); +} diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 9e1433832..579b51019 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -50,6 +50,11 @@ import { isAllowedByLists, mostSpecificMatchingAllow } from "@server/lib/aiModelKeyMatch"; +import { + catalogOwnershipScore, + keepBestScored, + providerClassRank +} from "@server/lib/aiProviderSelection"; import { aiGatewayUpstreamFetch } from "@server/lib/aiGatewayUpstreamFetch"; import { getModelPricing, calculateAiCost } from "@server/lib/aiModelPricing"; import { @@ -502,17 +507,28 @@ async function selectProvider( }; } + // 1) Prefer the most specific allow pattern that matched the request. candidates.sort((a, b) => compareModelKeySpecificity(a.modelKey, b.modelKey) ); - const bestSpecificity = candidates[0].modelKey; - const topCandidates = candidates.filter( + let remaining = candidates.filter( (c) => compareModelKeySpecificity(c.modelKey, bestSpecificity) === 0 ); + // 2) Prefer providers whose catalog owns this model id. Aggregators only + // score when the model is known somewhere in the catalog. + remaining = keepBestScored(remaining, (c) => + catalogOwnershipScore(c.provider.type as AiProviderType, requestedModel) + ); + + // 3) Prefer native typed providers over aggregators over custom. + remaining = keepBestScored(remaining, (c) => + providerClassRank(c.provider.type as AiProviderType) + ); + const uniqueProviders = new Map(); - for (const candidate of topCandidates) { + for (const candidate of remaining) { uniqueProviders.set(candidate.provider.providerId, candidate.provider); } @@ -523,7 +539,7 @@ async function selectProvider( return { ok: false, status: HttpCode.FORBIDDEN, - message: `Model "${requestedModel}" is ambiguous across multiple AI providers on this resource` + message: `Model "${requestedModel}" is ambiguous across multiple AI providers on this resource. Ask your administrator to configure a more specific allow pattern for this model.` }; } diff --git a/server/routers/resource/addAiProviderToResource.ts b/server/routers/resource/addAiProviderToResource.ts index f2477d89d..dd0d86227 100644 --- a/server/routers/resource/addAiProviderToResource.ts +++ b/server/routers/resource/addAiProviderToResource.ts @@ -131,8 +131,7 @@ export async function addAiProviderToResource( const attachments = await resolveProviderAttachments({ orgId: resource.orgId, attachments: nextAttachments, - requireAtLeastOne: true, - resourceId + requireAtLeastOne: true }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/server/routers/resource/removeAiProviderFromResource.ts b/server/routers/resource/removeAiProviderFromResource.ts index 5b99ff089..28c877317 100644 --- a/server/routers/resource/removeAiProviderFromResource.ts +++ b/server/routers/resource/removeAiProviderFromResource.ts @@ -134,8 +134,7 @@ export async function removeAiProviderFromResource( const attachments = await resolveProviderAttachments({ orgId: resource.orgId, attachments: remaining, - requireAtLeastOne: false, - resourceId + requireAtLeastOne: false }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/server/routers/resource/setResourceAiProviders.ts b/server/routers/resource/setResourceAiProviders.ts index ef312ac70..b906b47d5 100644 --- a/server/routers/resource/setResourceAiProviders.ts +++ b/server/routers/resource/setResourceAiProviders.ts @@ -113,8 +113,7 @@ export async function setResourceAiProviders( const attachments = await resolveProviderAttachments({ orgId: resource.orgId, attachments: providers, - requireAtLeastOne: false, - resourceId + requireAtLeastOne: false }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/server/routers/siteResource/addAiProviderToSiteResource.ts b/server/routers/siteResource/addAiProviderToSiteResource.ts index 18e8cd74d..0fd93b91c 100644 --- a/server/routers/siteResource/addAiProviderToSiteResource.ts +++ b/server/routers/siteResource/addAiProviderToSiteResource.ts @@ -131,8 +131,7 @@ export async function addAiProviderToSiteResource( const attachments = await resolveProviderAttachments({ orgId: siteResource.orgId, attachments: nextAttachments, - requireAtLeastOne: true, - siteResourceId + requireAtLeastOne: true }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/server/routers/siteResource/removeAiProviderFromSiteResource.ts b/server/routers/siteResource/removeAiProviderFromSiteResource.ts index 29facaece..4e7f4e666 100644 --- a/server/routers/siteResource/removeAiProviderFromSiteResource.ts +++ b/server/routers/siteResource/removeAiProviderFromSiteResource.ts @@ -133,8 +133,7 @@ export async function removeAiProviderFromSiteResource( const attachments = await resolveProviderAttachments({ orgId: siteResource.orgId, attachments: remaining, - requireAtLeastOne: false, - siteResourceId + requireAtLeastOne: false }); if (isInferenceFieldsError(attachments)) { return next( diff --git a/server/routers/siteResource/setSiteResourceAiProviders.ts b/server/routers/siteResource/setSiteResourceAiProviders.ts index a6c8a52c7..c7762adb6 100644 --- a/server/routers/siteResource/setSiteResourceAiProviders.ts +++ b/server/routers/siteResource/setSiteResourceAiProviders.ts @@ -115,8 +115,7 @@ export async function setSiteResourceAiProviders( const attachments = await resolveProviderAttachments({ orgId: siteResource.orgId, attachments: providers, - requireAtLeastOne: false, - siteResourceId + requireAtLeastOne: false }); if (isInferenceFieldsError(attachments)) { return next( From f7aca854171a6b325e40be9de72ee4299e2f44f0 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 16:14:46 -0400 Subject: [PATCH 125/296] link the usage with the session to display together --- messages/en-US.json | 3 ++ server/db/pg/schema/schema.ts | 10 +++- server/db/sqlite/schema/schema.ts | 10 +++- server/lib/aiBudgetEnforcement.ts | 6 +++ server/routers/aiGateway/logAiSession.ts | 4 +- server/routers/aiGateway/pipeline.ts | 11 ++++- server/routers/auditLogs/queryAiSessionLog.ts | 49 ++++++++++++++++++- server/routers/auditLogs/types.ts | 10 ++++ src/app/[orgId]/settings/logs/ai/page.tsx | 34 ++++++++++++- 9 files changed, 129 insertions(+), 8 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index d5ecdf780..44b1bf6a3 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -3411,6 +3411,9 @@ "aiSessionLogTruncated": "This session was truncated before storage and may be incomplete.", "aiSessionViewRaw": "View Raw JSON", "aiSessionViewChat": "View Chat", + "tokens": "Tokens", + "cost": "Cost", + "estimated": "estimated", "requestAnalyticsDescription": "View detailed request analytics for resources in this organization", "logRetentionRequestLabel": "HTTP Request Log Retention", "logRetentionRequestDescription": "How long to retain request logs", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index cbb1e60cb..afaccc480 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1831,6 +1831,13 @@ export const aiUsageRecords = pgTable( userId: varchar("userId").references(() => users.userId, { onDelete: "set null" }), + // Links this usage record back to the aiSessionLog row for the same + // request (aiSessionLog.sessionId), so token/cost usage can be shown + // alongside the session transcript. Not a DB-level FK - aiSessionLog + // lives in the separate logs database. Nullable because the session + // log may be disabled (retention set to 0) while usage tracking + // stays on. + sessionId: varchar("sessionId"), requestedModel: varchar("requestedModel").notNull(), promptTokens: integer("promptTokens").notNull().default(0), cacheReadTokens: integer("cacheReadTokens").notNull().default(0), @@ -1862,7 +1869,8 @@ export const aiUsageRecords = pgTable( t.orgId, t.userId, t.createdAt - ) + ), + index("idx_ai_usage_records_session").on(t.sessionId) ] ); diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 8934638d7..bf3be9f87 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1819,6 +1819,13 @@ export const aiUsageRecords = sqliteTable( userId: text("userId").references(() => users.userId, { onDelete: "set null" }), + // Links this usage record back to the aiSessionLog row for the same + // request (aiSessionLog.sessionId), so token/cost usage can be shown + // alongside the session transcript. Not a DB-level FK - aiSessionLog + // lives in the separate logs database. Nullable because the session + // log may be disabled (retention set to 0) while usage tracking + // stays on. + sessionId: text("sessionId"), requestedModel: text("requestedModel").notNull(), promptTokens: integer("promptTokens").notNull().default(0), cacheReadTokens: integer("cacheReadTokens").notNull().default(0), @@ -1852,7 +1859,8 @@ export const aiUsageRecords = sqliteTable( t.orgId, t.userId, t.createdAt - ) + ), + index("idx_ai_usage_records_session").on(t.sessionId) ] ); diff --git a/server/lib/aiBudgetEnforcement.ts b/server/lib/aiBudgetEnforcement.ts index 4177af9b7..d73f5ba59 100644 --- a/server/lib/aiBudgetEnforcement.ts +++ b/server/lib/aiBudgetEnforcement.ts @@ -428,6 +428,11 @@ export type UsageRecordInput = { usage: AiUsage; costUsd: number | null; createdAt?: number; + // Same id as the aiSessionLog row logged for this request, so the two + // can be joined to show token/cost usage alongside the session + // transcript. Undefined when the session wasn't logged (e.g. session + // log retention disabled for the org). + sessionId?: string; }; type AiUsageRecordInsert = InferInsertModel; @@ -539,6 +544,7 @@ export async function recordUsage(input: UsageRecordInput): Promise { resourceId: input.resourceId, siteResourceId: input.siteResourceId, userId: input.userId, + sessionId: input.sessionId, requestedModel: input.requestedModel, promptTokens: usage.promptTokens, cacheReadTokens: usage.cacheReadTokens, diff --git a/server/routers/aiGateway/logAiSession.ts b/server/routers/aiGateway/logAiSession.ts index 92723bed4..ee05517f5 100644 --- a/server/routers/aiGateway/logAiSession.ts +++ b/server/routers/aiGateway/logAiSession.ts @@ -1,4 +1,3 @@ -import { randomUUID } from "crypto"; import { logsDb, db, orgs, aiSessionLog, type AiProvider } from "@server/db"; import type { InferInsertModel } from "drizzle-orm"; import logger from "@server/logger"; @@ -169,6 +168,7 @@ function truncateBody(value: string): { value: string; truncated: boolean } { } export function logAiSession(data: { + sessionId: string; capability: AiCapability; provider: AiProvider; requestedModel: string | undefined; @@ -230,7 +230,7 @@ export function logAiSession(data: { } sessionLogBuffer.push({ - sessionId: randomUUID(), + sessionId: data.sessionId, orgId: sanitizeString(data.orgId), providerId: data.provider.providerId, capability: data.capability, diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 579b51019..7648e2355 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "crypto"; import { Request, Response } from "express"; import { and, eq, inArray } from "drizzle-orm"; import { @@ -594,6 +595,12 @@ export function recordAiGatewayCompletion(args: { const pricing = getModelPricing(provider.type as AiProviderType, model); const cost = calculateAiCost(pricing, usage); + // Shared by the usage record and the session log so the two can be + // joined later to show token/cost usage alongside the transcript - + // generated up front since neither buffered insert's row id is known + // until its next batch flush. + const sessionId = randomUUID(); + logger.info("AI gateway request usage", { capability, providerId: provider.providerId, @@ -618,7 +625,8 @@ export function recordAiGatewayCompletion(args: { userId: requestUserId, requestedModel: model ?? "unknown", usage, - costUsd: cost?.totalCost ?? null + costUsd: cost?.totalCost ?? null, + sessionId }); if (budgets.length > 0) { @@ -635,6 +643,7 @@ export function recordAiGatewayCompletion(args: { } logAiSession({ + sessionId, capability, provider, requestedModel, diff --git a/server/routers/auditLogs/queryAiSessionLog.ts b/server/routers/auditLogs/queryAiSessionLog.ts index 2412f8980..d61bdf5c8 100644 --- a/server/routers/auditLogs/queryAiSessionLog.ts +++ b/server/routers/auditLogs/queryAiSessionLog.ts @@ -2,6 +2,7 @@ import { logsDb, aiSessionLog, aiProviders, + aiUsageRecords, resources, siteResources, users, @@ -253,6 +254,51 @@ async function enrichWithDetails( } } + const usageMap = new Map< + string, + { + promptTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + completionTokens: number; + reasoningTokens: number; + totalTokens: number; + costUsd: number | null; + estimated: boolean; + } + >(); + const sessionIds = logs.map((log) => log.sessionId); + if (sessionIds.length > 0) { + const usageDetails = await primaryDb + .select({ + sessionId: aiUsageRecords.sessionId, + promptTokens: aiUsageRecords.promptTokens, + cacheReadTokens: aiUsageRecords.cacheReadTokens, + cacheWriteTokens: aiUsageRecords.cacheWriteTokens, + completionTokens: aiUsageRecords.completionTokens, + reasoningTokens: aiUsageRecords.reasoningTokens, + totalTokens: aiUsageRecords.totalTokens, + costUsd: aiUsageRecords.costUsd, + estimated: aiUsageRecords.estimated + }) + .from(aiUsageRecords) + .where(inArray(aiUsageRecords.sessionId, sessionIds)); + + for (const u of usageDetails) { + if (!u.sessionId) continue; + usageMap.set(u.sessionId, { + promptTokens: u.promptTokens, + cacheReadTokens: u.cacheReadTokens, + cacheWriteTokens: u.cacheWriteTokens, + completionTokens: u.completionTokens, + reasoningTokens: u.reasoningTokens, + totalTokens: u.totalTokens, + costUsd: u.costUsd, + estimated: u.estimated + }); + } + } + return logs.map((log) => { const provider = providerMap.get(log.providerId); @@ -281,7 +327,8 @@ async function enrichWithDetails( providerType: provider?.type ?? null, resourceName, resourceNiceId, - userEmail: log.userId ? (userMap.get(log.userId) ?? null) : null + userEmail: log.userId ? (userMap.get(log.userId) ?? null) : null, + usage: usageMap.get(log.sessionId) ?? null }; }); } diff --git a/server/routers/auditLogs/types.ts b/server/routers/auditLogs/types.ts index 42b93904e..b63acbf30 100644 --- a/server/routers/auditLogs/types.ts +++ b/server/routers/auditLogs/types.ts @@ -119,6 +119,16 @@ export type QueryAiSessionLogResponse = { truncated: boolean; statusCode: number | null; createdAt: number; + usage: { + promptTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + completionTokens: number; + reasoningTokens: number; + totalTokens: number; + costUsd: number | null; + estimated: boolean; + } | null; }[]; pagination: { total: number; diff --git a/src/app/[orgId]/settings/logs/ai/page.tsx b/src/app/[orgId]/settings/logs/ai/page.tsx index 67d18b94c..7330db111 100644 --- a/src/app/[orgId]/settings/logs/ai/page.tsx +++ b/src/app/[orgId]/settings/logs/ai/page.tsx @@ -459,7 +459,7 @@ export default function AiSessionLogsPage() { const renderExpandedRow = (row: any) => { return (
-
+
{t("aiSessionId")}

@@ -472,6 +472,26 @@ export default function AiSessionLogsPage() { {row.statusCode ?? "N/A"}

+
+ {t("tokens")} +

+ {row.usage + ? `${row.usage.totalTokens.toLocaleString()}${ + row.usage.estimated + ? ` (${t("estimated")})` + : "" + }` + : "N/A"} +

+
+
+ {t("cost")} +

+ {row.usage && row.usage.costUsd != null + ? `$${row.usage.costUsd.toFixed(4)}` + : "N/A"} +

+
Date: Tue, 11 Aug 2026 22:25:26 +0200 Subject: [PATCH 126/296] =?UTF-8?q?=E2=9C=A8=20=20test=20alert=20email=20w?= =?UTF-8?q?orks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- messages/en-US.json | 2 + server/emails/templates/AlertNotification.tsx | 33 +++++- .../private/lib/alerts/processTestAlerts.ts | 5 +- server/private/lib/alerts/sendAlertEmail.ts | 49 +++++--- .../routers/alertRule/testAlertRule.ts | 111 +++++++++++++++++- server/private/routers/external.ts | 2 +- server/routers/alertRule/types.ts | 4 +- .../AlertRuleGraphEditor.tsx | 28 +++++ src/lib/getRandomItemInArray.ts | 5 + 9 files changed, 214 insertions(+), 25 deletions(-) create mode 100644 src/lib/getRandomItemInArray.ts diff --git a/messages/en-US.json b/messages/en-US.json index 2dd438a12..05b40d27d 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1695,6 +1695,8 @@ "alertingRuleSaved": "Alert rule saved", "alertingRuleSavedCreatedDescription": "Your new alert rule was created. You can keep editing it on this page.", "alertingRuleSavedUpdatedDescription": "Your changes to this alert rule were saved.", + "alertingTestAlertSent": "Test alert sent", + "alertingTestAlertSentDescription": "A test alert was sent to the actions configured on this rule.", "alertingEditRule": "Edit Alert Rule", "alertingCreateRule": "Create Alert Rule", "alertingRuleCredenzaDescription": "Choose what to watch, when to fire, and how to notify", diff --git a/server/emails/templates/AlertNotification.tsx b/server/emails/templates/AlertNotification.tsx index ce30753da..c81cf60da 100644 --- a/server/emails/templates/AlertNotification.tsx +++ b/server/emails/templates/AlertNotification.tsx @@ -31,9 +31,24 @@ export type AlertNotificationProps = { orgId: string; data: Record; dashboardLink: string; + isTestAlert?: boolean; }; -function getEventMeta(eventType: AlertEventType): { +function getEventMeta( + eventType: AlertEventType, + isTestAlert: boolean = false +): { + heading: string; + previewText: string; + summary: string; + statusLabel: string | null; + statusColor: string | null; +} { + const meta = getBaseEventMeta(eventType); + return isTestAlert ? { ...meta, heading: `[TEST] ${meta.heading}` } : meta; +} + +function getBaseEventMeta(eventType: AlertEventType): { heading: string; previewText: string; summary: string; @@ -180,8 +195,14 @@ function formatDataItems( } export const AlertNotification = (props: AlertNotificationProps) => { - const { eventType, orgId, data, dashboardLink } = props; - const meta = getEventMeta(eventType); + const { + eventType, + orgId, + data, + dashboardLink, + isTestAlert = false + } = props; + const meta = getEventMeta(eventType, isTestAlert); const dataItems = formatDataItems(data); const isToggle = @@ -242,6 +263,12 @@ export const AlertNotification = (props: AlertNotificationProps) => { Open your dashboard to view more details and manage your alert rules. + {isTestAlert && ( + + This is a test alert. No action is required, + and no real event has occurred. + + )} diff --git a/server/private/lib/alerts/processTestAlerts.ts b/server/private/lib/alerts/processTestAlerts.ts index 7aa1691f0..1a6c1d7c6 100644 --- a/server/private/lib/alerts/processTestAlerts.ts +++ b/server/private/lib/alerts/processTestAlerts.ts @@ -20,7 +20,10 @@ export async function processTestAlerts(context: TestAlertContext) { try { const recipients = await resolveEmailRecipients(action); if (recipients.length > 0) { - await sendAlertEmail(recipients, context); + await sendAlertEmail(recipients, { + ...context, + isTest: true + }); } } catch (err) { logger.error(`processTestAlerts: failed to send alert email`, err); diff --git a/server/private/lib/alerts/sendAlertEmail.ts b/server/private/lib/alerts/sendAlertEmail.ts index 6f99b102c..0eef6fb5c 100644 --- a/server/private/lib/alerts/sendAlertEmail.ts +++ b/server/private/lib/alerts/sendAlertEmail.ts @@ -15,7 +15,24 @@ import { sendEmail } from "@server/emails"; import AlertNotification from "@server/emails/templates/AlertNotification"; import config from "@server/lib/config"; import logger from "@server/logger"; -import { AlertContext } from "@server/routers/alertRule/types"; +import { + AlertContext, + type AlertEventType +} from "@server/routers/alertRule/types"; + +type EmailAlertContext = { + eventType: AlertEventType; + orgId: string; + /** Set for site_online / site_offline events */ + siteId?: number; + /** Set for health_check_* events */ + healthCheckId?: number; + /** Set for resource_* events */ + resourceId?: number; + /** Human-readable context data included in emails and webhook payloads */ + data: Record; + isTest?: boolean; +}; /** * Sends an alert notification email to every address in `recipients`. @@ -27,7 +44,7 @@ import { AlertContext } from "@server/routers/alertRule/types"; */ export async function sendAlertEmail( recipients: string[], - context: AlertContext + context: EmailAlertContext ): Promise { if (recipients.length === 0) { return; @@ -46,7 +63,8 @@ export async function sendAlertEmail( eventType: context.eventType, orgId: context.orgId, data: context.data, - dashboardLink + dashboardLink, + isTestAlert: context.isTest }), { from, @@ -70,34 +88,35 @@ export async function sendAlertEmail( // Helpers // --------------------------------------------------------------------------- -function buildSubject(context: AlertContext): string { +function buildSubject(context: EmailAlertContext): string { + const prefix = context.isTest ? "[Test Alert]" : "[Alert]"; switch (context.eventType) { case "site_online": - return "[Alert] Site Back Online"; + return `${prefix} Site Back Online`; case "site_offline": - return "[Alert] Site Offline"; + return `${prefix} Site Offline`; case "site_toggle": - return "[Alert] Site Status Changed"; + return `${prefix} Site Status Changed`; case "health_check_healthy": - return "[Alert] Health Check Recovered"; + return `${prefix} Health Check Recovered`; case "health_check_unhealthy": - return "[Alert] Health Check Failing"; + return `${prefix} Health Check Failing`; case "health_check_toggle": - return "[Alert] Health Check Status Changed"; + return `${prefix} Health Check Status Changed`; case "resource_healthy": - return "[Alert] Resource Healthy"; + return `${prefix} Resource Healthy`; case "resource_unhealthy": - return "[Alert] Resource Unhealthy"; + return `${prefix} Resource Unhealthy`; case "resource_degraded": - return "[Alert] Resource Degraded"; + return `${prefix} Resource Degraded`; case "resource_toggle": - return "[Alert] Resource Status Changed"; + return `${prefix} Resource Status Changed`; default: { // Exhaustiveness fallback – should never be reached with a // well-typed caller, but keeps runtime behaviour predictable. const _exhaustive: never = context.eventType; void _exhaustive; - return "[Alert] Event Notification"; + return `${prefix} Event Notification`; } } } diff --git a/server/private/routers/alertRule/testAlertRule.ts b/server/private/routers/alertRule/testAlertRule.ts index 39a5c28a1..6104dbc6b 100644 --- a/server/private/routers/alertRule/testAlertRule.ts +++ b/server/private/routers/alertRule/testAlertRule.ts @@ -27,7 +27,13 @@ import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { and, asc, desc, eq, inArray, like, or, sql } from "drizzle-orm"; -import { ListAlertRulesResponse } from "@server/routers/alertRule/types"; +import { + ListAlertRulesResponse, + type AlertAction, + type EmailAlertAction +} from "@server/routers/alertRule/types"; +import { processTestAlerts } from "@server/private/lib/alerts/processTestAlerts"; +import { getRandomItemInArray } from "@app/lib/getRandomItemInArray"; const paramsSchema = z.strictObject({ orgId: z.string().nonempty() @@ -51,12 +57,12 @@ export const RESOURCE_EVENT_TYPES = [ ] as const; const webhookActionSchema = z.strictObject({ - webhookUrl: z.string().url(), + webhookUrl: z.url(), config: z.string().optional(), enabled: z.boolean().optional().default(true) }); -const bodySchema = z.strictObject({ +const bodySchema = z.object({ eventType: z.enum([ ...HC_EVENT_TYPES, ...SITE_EVENT_TYPES, @@ -97,7 +103,106 @@ export async function testAlertRule( ); } + const body = parsedBody.data; + const collectedActions: AlertAction[] = []; + if ( + body.emails.length > 0 || + body.roleIds.length > 0 || + body.userIds.length > 0 + ) { + collectedActions.push({ + type: "email", + emails: body.emails, + roleIds: body.roleIds, + userIds: body.userIds + }); + } + + for (const action of body.webhookActions) { + collectedActions.push({ + type: "webhook", + ...action + }); + } + + let data: Record = {}; + switch (body.eventType) { + case "site_toggle": + data = { + status: getRandomItemInArray(["online", "offline"]), + siteName: "Test Site Alert" + }; + break; + case "site_offline": + data = { + status: "offline", + siteName: "Test Site Alert" + }; + break; + case "site_online": + data = { + status: "online", + siteName: "Test Site Alert" + }; + break; + case "resource_toggle": + data = { + status: getRandomItemInArray([ + "healthy", + "unhealthy", + "degraded" + ]), + siteName: "Test Resource Alert" + }; + break; + case "resource_healthy": + data = { + status: "healthy", + siteName: "Test Resource Alert" + }; + break; + case "resource_unhealthy": + data = { + status: "unhealthy", + siteName: "Test Resource Alert" + }; + break; + case "resource_degraded": + data = { + status: "degraded", + siteName: "Test Resource Alert" + }; + break; + case "health_check_toggle": + data = { + status: getRandomItemInArray(["healthy", "unhealthy"]), + healthCheckName: "Test Health Check Alert" + }; + break; + case "health_check_healthy": + data = { + status: "healthy", + healthCheckName: "Test Health Check Alert" + }; + break; + case "health_check_unhealthy": + data = { + status: "unhealthy", + healthCheckName: "Test Health Check Alert" + }; + break; + + default: + break; + } + // TODO: process alert rule + await processTestAlerts({ + eventType: body.eventType, + orgId, + actions: collectedActions, + data + }); } catch (error) { logger.error(error); return next( diff --git a/server/private/routers/external.ts b/server/private/routers/external.ts index 0fd4cc023..b880598cc 100644 --- a/server/private/routers/external.ts +++ b/server/private/routers/external.ts @@ -809,7 +809,7 @@ authenticated.get( ); authenticated.post( - "/org/:orgId/alert-rule/test", + "/org/:orgId/test-alert-rule", verifyValidLicense, verifyOrgAccess, verifyUserHasAction(ActionsEnum.testAlertRule), diff --git a/server/routers/alertRule/types.ts b/server/routers/alertRule/types.ts index 99057b312..21baa7b2b 100644 --- a/server/routers/alertRule/types.ts +++ b/server/routers/alertRule/types.ts @@ -128,7 +128,7 @@ export interface AlertContext { export type EmailAlertAction = { type: "email"; userIds?: string[]; - roleIds?: string[]; + roleIds?: number[]; emails?: string[]; }; @@ -139,7 +139,7 @@ export type WebhookAlertAction = { config?: string | undefined; }; -type AlertAction = EmailAlertAction | WebhookAlertAction; +export type AlertAction = EmailAlertAction | WebhookAlertAction; export interface TestAlertContext { eventType: AlertEventType; actions: AlertAction[]; diff --git a/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx b/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx index a10f8f3b3..243ce82f8 100644 --- a/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx +++ b/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx @@ -189,10 +189,38 @@ export default function AlertRuleGraphEditor({ description: t("alertingNoActionsTestDescription") }); } + return; } const values = form.getValues(); + try { + const payload = formValuesToApiPayload(values); + if (isNew) { + const res = await api.post< + AxiosResponse + >(`/org/${orgId}/test-alert-rule`, payload); + toast({ + title: t("alertingTestAlertSent"), + description: t("alertingTestAlertSentDescription") + }); + } else { + await api.post( + `/org/${orgId}/alert-rule/${alertRuleId}`, + payload + ); + toast({ + title: t("alertingTestAlertSent"), + description: t("alertingTestAlertSentDescription") + }); + } + } catch (e) { + toast({ + title: t("error"), + description: formatAxiosError(e), + variant: "destructive" + }); + } }; const [, formAction, isSaving] = useActionState(saveAlert, null); diff --git a/src/lib/getRandomItemInArray.ts b/src/lib/getRandomItemInArray.ts new file mode 100644 index 000000000..aa5a2e562 --- /dev/null +++ b/src/lib/getRandomItemInArray.ts @@ -0,0 +1,5 @@ +export function getRandomItemInArray(array: T[]) { + // Source - https://stackoverflow.com/a/4550514 + const randomElement = array[Math.floor(Math.random() * array.length)]; + return randomElement; +} From 1782f31075bd20371d08d8d39dbfe882a54d1298 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Tue, 11 Aug 2026 16:38:16 -0400 Subject: [PATCH 127/296] show link to provider settings --- messages/en-US.json | 1 + src/components/AiProviderAttachments.tsx | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/messages/en-US.json b/messages/en-US.json index 44b1bf6a3..b0743f631 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1921,6 +1921,7 @@ "aiResourceProvidersUpdated": "Providers updated", "aiResourceProvidersErrorUpdate": "Failed to update providers", "aiResourceProviderEditDescription": "Choose how this provider's models are exposed on this resource.", + "viewProviderSettings": "View Provider Settings", "aiResourceProviderMode": "Access mode", "aiResourceProviderModeInherit": "Inherit", "aiResourceProviderModeSelect": "Select", diff --git a/src/components/AiProviderAttachments.tsx b/src/components/AiProviderAttachments.tsx index 09a4243e1..87d61dd96 100644 --- a/src/components/AiProviderAttachments.tsx +++ b/src/components/AiProviderAttachments.tsx @@ -42,6 +42,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { useQuery } from "@tanstack/react-query"; import { Plus, XIcon } from "lucide-react"; import { useTranslations } from "next-intl"; +import Link from "next/link"; import { useEffect, useMemo, useRef, useState } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod"; @@ -178,6 +179,7 @@ export function AiProviderAttachments({ {editing && ( { @@ -283,11 +285,13 @@ type EditFormValues = { }; function EditAttachmentCredenza({ + orgId, attachment, open, onOpenChange, onSave }: { + orgId: string; attachment: AiProviderAttachmentValue; open: boolean; onOpenChange: (open: boolean) => void; @@ -565,6 +569,18 @@ function EditAttachmentCredenza({ + From f5b10df7cffde575ecca39ded24eebef566bd657 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 16:39:52 -0400 Subject: [PATCH 128/296] fix non-https crypto issue --- src/components/BudgetsEditor.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/components/BudgetsEditor.tsx b/src/components/BudgetsEditor.tsx index e9dbbaefb..3ccbfad3c 100644 --- a/src/components/BudgetsEditor.tsx +++ b/src/components/BudgetsEditor.tsx @@ -82,10 +82,17 @@ function nextAvailableCombo(rows: BudgetRow[]): { return { unit: "usd", period: "monthly" }; } +function newRowKey(): string { + if (typeof crypto !== "undefined" && "randomUUID" in crypto) { + return crypto.randomUUID(); + } + return `tmp-${Date.now()}-${Math.random().toString(36).slice(2)}`; +} + export function newBudgetRow(rows: BudgetRow[]): BudgetRow { const combo = nextAvailableCombo(rows); return { - key: crypto.randomUUID(), + key: newRowKey(), amount: "", unit: combo.unit, period: combo.period From 0016b8fce7c9b93c396b7202a417e1951bfccd8e Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 16:40:10 -0400 Subject: [PATCH 129/296] show other tokens on the expanded row --- messages/en-US.json | 10 +++- src/app/[orgId]/settings/logs/ai/page.tsx | 67 +++++++++++++++++++---- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index b0743f631..6444dbdd8 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -3412,9 +3412,15 @@ "aiSessionLogTruncated": "This session was truncated before storage and may be incomplete.", "aiSessionViewRaw": "View Raw JSON", "aiSessionViewChat": "View Chat", - "tokens": "Tokens", "cost": "Cost", - "estimated": "estimated", + "estimated": "Estimated", + "tokenUsage": "Token Usage", + "promptTokens": "Prompt Tokens", + "cacheReadTokens": "Cache Read Tokens", + "cacheWriteTokens": "Cache Write Tokens", + "completionTokens": "Completion Tokens", + "reasoningTokens": "Reasoning Tokens", + "totalTokens": "Total Tokens", "requestAnalyticsDescription": "View detailed request analytics for resources in this organization", "logRetentionRequestLabel": "HTTP Request Log Retention", "logRetentionRequestDescription": "How long to retain request logs", diff --git a/src/app/[orgId]/settings/logs/ai/page.tsx b/src/app/[orgId]/settings/logs/ai/page.tsx index 7330db111..5d51c118b 100644 --- a/src/app/[orgId]/settings/logs/ai/page.tsx +++ b/src/app/[orgId]/settings/logs/ai/page.tsx @@ -472,18 +472,6 @@ export default function AiSessionLogsPage() { {row.statusCode ?? "N/A"}

-
- {t("tokens")} -

- {row.usage - ? `${row.usage.totalTokens.toLocaleString()}${ - row.usage.estimated - ? ` (${t("estimated")})` - : "" - }` - : "N/A"} -

-
{t("cost")}

@@ -492,7 +480,62 @@ export default function AiSessionLogsPage() { : "N/A"}

+
+ {t("estimated")} +

+ {row.usage + ? row.usage.estimated + ? t("yes") + : t("no") + : "N/A"} +

+
+ {row.usage && ( +
+
+ {t("tokenUsage")} +
+
+
+ {t("promptTokens")} +

+ {row.usage.promptTokens.toLocaleString()} +

+
+
+ {t("cacheReadTokens")} +

+ {row.usage.cacheReadTokens.toLocaleString()} +

+
+
+ {t("cacheWriteTokens")} +

+ {row.usage.cacheWriteTokens.toLocaleString()} +

+
+
+ {t("completionTokens")} +

+ {row.usage.completionTokens.toLocaleString()} +

+
+
+ {t("reasoningTokens")} +

+ {row.usage.reasoningTokens.toLocaleString()} +

+
+
+ {t("totalTokens")} +

+ {row.usage.totalTokens.toLocaleString()} +

+
+
+
+ )} Date: Tue, 11 Aug 2026 17:52:27 -0400 Subject: [PATCH 130/296] basic ai analytics created --- messages/en-US.json | 2 + .../auditLogs/aiUsageAnalyticsShared.ts | 170 ++++++++++ server/routers/auditLogs/index.ts | 5 + .../auditLogs/queryAiUsageFilterOptions.ts | 269 ++++++++++++++++ .../routers/auditLogs/queryAiUsageOverview.ts | 228 ++++++++++++++ .../auditLogs/queryAiUsageProviders.ts | 183 +++++++++++ .../auditLogs/queryAiUsageResources.ts | 222 +++++++++++++ .../auditLogs/queryAiUsageUsersRoles.ts | 271 ++++++++++++++++ server/routers/external.ts | 35 +++ server/routers/integration.ts | 35 +++ .../[orgId]/settings/logs/ai-usage/page.tsx | 30 ++ src/app/navigation.tsx | 11 + src/components/AiUsageAnalyticsData.tsx | 297 ++++++++++++++++++ .../ai-usage-analytics/OverviewTab.tsx | 197 ++++++++++++ .../ai-usage-analytics/ProvidersTab.tsx | 91 ++++++ .../ai-usage-analytics/ResourcesTab.tsx | 99 ++++++ .../ToggleableTrendChart.tsx | 188 +++++++++++ .../ai-usage-analytics/TopEntitiesList.tsx | 92 ++++++ .../ai-usage-analytics/UsersRolesTab.tsx | 166 ++++++++++ src/components/ai-usage-analytics/shared.ts | 64 ++++ src/lib/queries.ts | 166 +++++++++- 21 files changed, 2820 insertions(+), 1 deletion(-) create mode 100644 server/routers/auditLogs/aiUsageAnalyticsShared.ts create mode 100644 server/routers/auditLogs/queryAiUsageFilterOptions.ts create mode 100644 server/routers/auditLogs/queryAiUsageOverview.ts create mode 100644 server/routers/auditLogs/queryAiUsageProviders.ts create mode 100644 server/routers/auditLogs/queryAiUsageResources.ts create mode 100644 server/routers/auditLogs/queryAiUsageUsersRoles.ts create mode 100644 src/app/[orgId]/settings/logs/ai-usage/page.tsx create mode 100644 src/components/AiUsageAnalyticsData.tsx create mode 100644 src/components/ai-usage-analytics/OverviewTab.tsx create mode 100644 src/components/ai-usage-analytics/ProvidersTab.tsx create mode 100644 src/components/ai-usage-analytics/ResourcesTab.tsx create mode 100644 src/components/ai-usage-analytics/ToggleableTrendChart.tsx create mode 100644 src/components/ai-usage-analytics/TopEntitiesList.tsx create mode 100644 src/components/ai-usage-analytics/UsersRolesTab.tsx create mode 100644 src/components/ai-usage-analytics/shared.ts diff --git a/messages/en-US.json b/messages/en-US.json index 6444dbdd8..166ca8a4b 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -3397,6 +3397,8 @@ "aiSessionLogsDescription": "View prompt and response transcripts for AI gateway requests in this organization", "sidebarLogsAi": "AI Session Logs", "commandLogsAi": "AI Session Logs", + "sidebarLogsAiUsage": "AI Usage Analytics", + "commandLogsAiUsage": "AI Usage Analytics", "provider": "Provider", "capability": "Capability", "model": "Model", diff --git a/server/routers/auditLogs/aiUsageAnalyticsShared.ts b/server/routers/auditLogs/aiUsageAnalyticsShared.ts new file mode 100644 index 000000000..95a6f825b --- /dev/null +++ b/server/routers/auditLogs/aiUsageAnalyticsShared.ts @@ -0,0 +1,170 @@ +import { and, eq, gte, lte, or, inArray, sql } from "drizzle-orm"; +import { aiUsageRecords, userOrgRoles, driver, db } from "@server/db"; +import { z } from "zod"; +import { getSevenDaysAgo } from "@app/lib/getSevenDaysAgo"; + +// Cap on how many distinct series a trend chart will plot before collapsing +// the remainder into an "other" bucket - matches the theme's 5 categorical +// chart colors (--chart-1..--chart-5). +export const TOP_N = 5; + +// Same guard used by queryRequestAnalytics/queryAiSessionLog for distinct +// breakdown lists. +export const DISTINCT_LIMIT = 500; + +export const aiUsageAnalyticsFiltersQuery = z.object({ + timeStart: z + .string() + .refine((val) => !isNaN(Date.parse(val)), { + error: "timeStart must be a valid ISO date string" + }) + .transform((val) => new Date(val).getTime()) + .prefault(() => getSevenDaysAgo().toISOString()) + .openapi({ + type: "string", + format: "date-time", + description: + "Start time as ISO date string (defaults to 7 days ago)" + }), + timeEnd: z + .string() + .refine((val) => !isNaN(Date.parse(val)), { + error: "timeEnd must be a valid ISO date string" + }) + .transform((val) => new Date(val).getTime()) + .prefault(() => new Date().toISOString()) + .openapi({ + type: "string", + format: "date-time", + description: + "End time as ISO date string (defaults to current time)" + }), + providerId: z + .string() + .optional() + .transform(Number) + .pipe(z.int().positive()) + .optional(), + model: z.string().optional(), + resourceId: z + .string() + .optional() + .transform(Number) + .pipe(z.int().positive()) + .optional(), + roleId: z + .string() + .optional() + .transform(Number) + .pipe(z.int().positive()) + .optional(), + userId: z.string().optional() +}); + +export const aiUsageAnalyticsParams = z.object({ + orgId: z.string() +}); + +export const aiUsageAnalyticsCombined = aiUsageAnalyticsFiltersQuery.merge( + aiUsageAnalyticsParams +); + +export type AiUsageAnalyticsQuery = z.infer; + +// A role has no column on aiUsageRecords - it's derived by resolving the +// role's members to userIds first, then filtering on userId. If the role has +// no members we still need the filter to exclude everything rather than be +// silently ignored, hence the sentinel value. +export async function resolveRoleUserIds( + orgId: string, + roleId?: number +): Promise { + if (!roleId) { + return undefined; + } + + const rows = await db + .select({ userId: userOrgRoles.userId }) + .from(userOrgRoles) + .where( + and(eq(userOrgRoles.orgId, orgId), eq(userOrgRoles.roleId, roleId)) + ); + + return rows.length > 0 + ? rows.map((r) => r.userId) + : ["__no_users_in_role__"]; +} + +export function buildAiUsageWhere( + data: AiUsageAnalyticsQuery, + roleUserIds?: string[] +) { + return and( + eq(aiUsageRecords.orgId, data.orgId), + gte(aiUsageRecords.createdAt, data.timeStart), + lte(aiUsageRecords.createdAt, data.timeEnd), + data.providerId + ? eq(aiUsageRecords.providerId, data.providerId) + : undefined, + data.model ? eq(aiUsageRecords.requestedModel, data.model) : undefined, + data.resourceId + ? or( + eq(aiUsageRecords.resourceId, data.resourceId), + eq(aiUsageRecords.siteResourceId, data.resourceId) + ) + : undefined, + data.userId ? eq(aiUsageRecords.userId, data.userId) : undefined, + roleUserIds ? inArray(aiUsageRecords.userId, roleUserIds) : undefined + ); +} + +// Buckets createdAt (epoch ms) down to a per-day string, dialect-aware, same +// approach as the DATE_TRUNC/DATE branch in queryRequestAnalytics.ts. +export function dayBucketExpr() { + return driver === "pg" + ? sql`DATE_TRUNC('day', TO_TIMESTAMP(${aiUsageRecords.createdAt} / 1000.0))` + : sql`DATE(${aiUsageRecords.createdAt} / 1000, 'unixepoch')`; +} + +export type DailyMetricRow = { + day: string; + key: K; + value: number; +}; + +// Ranks dimension keys by total value and returns the top N. +export function pickTopNKeys( + totals: Map, + n: number = TOP_N +): K[] { + return [...totals.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, n) + .map(([key]) => key); +} + +export interface DayValueRow { + day: string; + [seriesKey: string]: number | string; +} + +// Collapses per-day, per-key rows into a per-day series object, folding +// anything outside `topKeys` into a shared "other" series. +export function bucketTopNPerDay( + rows: DailyMetricRow[], + topKeys: K[] +): DayValueRow[] { + const topSet = new Set(topKeys); + const byDay = new Map>(); + + for (const row of rows) { + const seriesKey = topSet.has(row.key) ? row.key : "other"; + const dayEntry = byDay.get(row.day) ?? {}; + dayEntry[seriesKey] = (dayEntry[seriesKey] ?? 0) + row.value; + byDay.set(row.day, dayEntry); + } + + return [...byDay.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([day, values]) => ({ day, ...values })); +} diff --git a/server/routers/auditLogs/index.ts b/server/routers/auditLogs/index.ts index 51d970360..12735d650 100644 --- a/server/routers/auditLogs/index.ts +++ b/server/routers/auditLogs/index.ts @@ -3,3 +3,8 @@ export * from "./queryRequestAnalytics"; export * from "./exportRequestAuditLog"; export * from "./queryAiSessionLog"; export * from "./exportAiSessionLog"; +export * from "./queryAiUsageFilterOptions"; +export * from "./queryAiUsageOverview"; +export * from "./queryAiUsageProviders"; +export * from "./queryAiUsageResources"; +export * from "./queryAiUsageUsersRoles"; diff --git a/server/routers/auditLogs/queryAiUsageFilterOptions.ts b/server/routers/auditLogs/queryAiUsageFilterOptions.ts new file mode 100644 index 000000000..46cd3199e --- /dev/null +++ b/server/routers/auditLogs/queryAiUsageFilterOptions.ts @@ -0,0 +1,269 @@ +import { + db, + aiUsageRecords, + aiProviders, + resources, + siteResources, + users, + roles, + userOrgRoles +} from "@server/db"; +import { registry } from "@server/openApi"; +import { NextFunction } from "express"; +import { Request, Response } from "express"; +import { and, eq, gte, lte, inArray, isNull, not } from "drizzle-orm"; +import { OpenAPITags } from "@server/openApi"; +import { z } from "zod"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { fromError } from "zod-validation-error"; +import response from "@server/lib/response"; +import logger from "@server/logger"; +import { getSevenDaysAgo } from "@app/lib/getSevenDaysAgo"; +import { DISTINCT_LIMIT } from "./aiUsageAnalyticsShared"; + +const queryAiUsageFilterOptionsQuery = z.object({ + timeStart: z + .string() + .refine((val) => !isNaN(Date.parse(val)), { + error: "timeStart must be a valid ISO date string" + }) + .transform((val) => new Date(val).getTime()) + .prefault(() => getSevenDaysAgo().toISOString()), + timeEnd: z + .string() + .refine((val) => !isNaN(Date.parse(val)), { + error: "timeEnd must be a valid ISO date string" + }) + .transform((val) => new Date(val).getTime()) + .prefault(() => new Date().toISOString()) +}); + +const queryAiUsageFilterOptionsParams = z.object({ + orgId: z.string() +}); + +const queryAiUsageFilterOptionsCombined = + queryAiUsageFilterOptionsQuery.merge(queryAiUsageFilterOptionsParams); +type Q = z.infer; + +function sortNamedFilterOptions( + items: T[] +): T[] { + return [...items].sort((a, b) => { + const nameA = a.name ?? ""; + const nameB = b.name ?? ""; + + if (nameA < nameB) return -1; + if (nameA > nameB) return 1; + + return a.id - b.id; + }); +} + +async function query(data: Q) { + const baseConditions = and( + eq(aiUsageRecords.orgId, data.orgId), + gte(aiUsageRecords.createdAt, data.timeStart), + lte(aiUsageRecords.createdAt, data.timeEnd) + ); + + const [ + uniqueProviders, + uniqueModels, + uniqueResources, + uniqueSiteResources, + uniqueUsers + ] = await Promise.all([ + db + .selectDistinct({ id: aiUsageRecords.providerId }) + .from(aiUsageRecords) + .where(baseConditions) + .limit(DISTINCT_LIMIT + 1), + db + .selectDistinct({ model: aiUsageRecords.requestedModel }) + .from(aiUsageRecords) + .where(baseConditions) + .limit(DISTINCT_LIMIT + 1), + db + .selectDistinct({ id: aiUsageRecords.resourceId }) + .from(aiUsageRecords) + .where(and(baseConditions, not(isNull(aiUsageRecords.resourceId)))) + .limit(DISTINCT_LIMIT + 1), + db + .selectDistinct({ id: aiUsageRecords.siteResourceId }) + .from(aiUsageRecords) + .where( + and( + baseConditions, + isNull(aiUsageRecords.resourceId), + not(isNull(aiUsageRecords.siteResourceId)) + ) + ) + .limit(DISTINCT_LIMIT + 1), + db + .selectDistinct({ userId: aiUsageRecords.userId }) + .from(aiUsageRecords) + .where(and(baseConditions, not(isNull(aiUsageRecords.userId)))) + .limit(DISTINCT_LIMIT + 1) + ]); + + const models = uniqueModels + .map((row) => row.model) + .filter((model): model is string => model !== null) + .sort(); + + const providerIds = uniqueProviders + .map((row) => row.id) + .filter((id): id is number => id !== null); + + let providers: Array<{ id: number; name: string | null }> = []; + if (providerIds.length > 0) { + const providerDetails = await db + .select({ providerId: aiProviders.providerId, name: aiProviders.name }) + .from(aiProviders) + .where(inArray(aiProviders.providerId, providerIds)); + + providers = providerDetails.map((p) => ({ id: p.providerId, name: p.name })); + } + + const resourceIds = uniqueResources + .map((row) => row.id) + .filter((id): id is number => id !== null); + const siteResourceIds = uniqueSiteResources + .map((row) => row.id) + .filter((id): id is number => id !== null); + + let resourcesWithNames: Array<{ id: number; name: string | null }> = []; + if (resourceIds.length > 0) { + const resourceDetails = await db + .select({ resourceId: resources.resourceId, name: resources.name }) + .from(resources) + .where(inArray(resources.resourceId, resourceIds)); + + resourcesWithNames = resourcesWithNames.concat( + resourceDetails.map((r) => ({ id: r.resourceId, name: r.name })) + ); + } + if (siteResourceIds.length > 0) { + const siteResourceDetails = await db + .select({ + siteResourceId: siteResources.siteResourceId, + name: siteResources.name + }) + .from(siteResources) + .where(inArray(siteResources.siteResourceId, siteResourceIds)); + + resourcesWithNames = resourcesWithNames.concat( + siteResourceDetails.map((r) => ({ id: r.siteResourceId, name: r.name })) + ); + } + + const userIds = uniqueUsers + .map((row) => row.userId) + .filter((id): id is string => id !== null); + + let userList: Array<{ id: string; email: string | null }> = []; + let roleList: Array<{ id: number; name: string | null }> = []; + if (userIds.length > 0) { + const userDetails = await db + .select({ userId: users.userId, email: users.email }) + .from(users) + .where(inArray(users.userId, userIds)); + userList = userDetails.map((u) => ({ id: u.userId, email: u.email })); + + const roleRows = await db + .select({ roleId: roles.roleId, name: roles.name }) + .from(userOrgRoles) + .innerJoin(roles, eq(userOrgRoles.roleId, roles.roleId)) + .where( + and( + eq(userOrgRoles.orgId, data.orgId), + inArray(userOrgRoles.userId, userIds) + ) + ); + + const roleMap = new Map(); + for (const r of roleRows) { + roleMap.set(r.roleId, r.name); + } + roleList = [...roleMap.entries()].map(([id, name]) => ({ id, name })); + } + + return { + providers: sortNamedFilterOptions(providers), + resources: sortNamedFilterOptions(resourcesWithNames), + roles: sortNamedFilterOptions(roleList), + users: userList, + models + }; +} + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/logs/ai/usage/filters", + description: + "Query the distinct filter options (providers, models, resources, roles, users) available for AI usage analytics within a time range", + tags: [OpenAPITags.Logs], + request: { + query: queryAiUsageFilterOptionsQuery, + params: queryAiUsageFilterOptionsParams + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export type QueryAiUsageFilterOptionsResponse = Awaited>; + +export async function queryAiUsageFilterOptions( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = queryAiUsageFilterOptionsQuery.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedQuery.error)) + ); + } + + const parsedParams = queryAiUsageFilterOptionsParams.safeParse( + req.params + ); + if (!parsedParams.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedParams.error)) + ); + } + + const data = await query({ ...parsedQuery.data, ...parsedParams.data }); + + return response(res, { + data, + success: true, + error: false, + message: "AI usage filter options retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/auditLogs/queryAiUsageOverview.ts b/server/routers/auditLogs/queryAiUsageOverview.ts new file mode 100644 index 000000000..57ab55539 --- /dev/null +++ b/server/routers/auditLogs/queryAiUsageOverview.ts @@ -0,0 +1,228 @@ +import { db, aiUsageRecords } from "@server/db"; +import { registry } from "@server/openApi"; +import { NextFunction } from "express"; +import { Request, Response } from "express"; +import { and, count, desc, eq, sql } from "drizzle-orm"; +import { OpenAPITags } from "@server/openApi"; +import { z } from "zod"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { fromError } from "zod-validation-error"; +import response from "@server/lib/response"; +import logger from "@server/logger"; +import { + aiUsageAnalyticsFiltersQuery, + aiUsageAnalyticsParams, + aiUsageAnalyticsCombined, + buildAiUsageWhere, + resolveRoleUserIds, + dayBucketExpr, + pickTopNKeys, + bucketTopNPerDay, + DISTINCT_LIMIT, + type AiUsageAnalyticsQuery +} from "./aiUsageAnalyticsShared"; + +type Q = AiUsageAnalyticsQuery; + +async function query(data: Q) { + const roleUserIds = await resolveRoleUserIds(data.orgId, data.roleId); + const baseConditions = buildAiUsageWhere(data, roleUserIds); + + const [totalsRow] = await db + .select({ + requests: count(), + promptTokens: sql`COALESCE(SUM(${aiUsageRecords.promptTokens}), 0)`, + cacheReadTokens: sql`COALESCE(SUM(${aiUsageRecords.cacheReadTokens}), 0)`, + cacheWriteTokens: sql`COALESCE(SUM(${aiUsageRecords.cacheWriteTokens}), 0)`, + completionTokens: sql`COALESCE(SUM(${aiUsageRecords.completionTokens}), 0)`, + reasoningTokens: sql`COALESCE(SUM(${aiUsageRecords.reasoningTokens}), 0)`, + totalTokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)`, + costUsd: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`, + estimatedRequests: sql`SUM(CASE WHEN ${aiUsageRecords.estimated} THEN 1 ELSE 0 END)` + }) + .from(aiUsageRecords) + .where(baseConditions); + + const dayExpr = dayBucketExpr(); + + const requestsPerDay = await db + .select({ + day: dayExpr.as("day"), + requests: count() + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(dayExpr) + .orderBy(dayExpr); + + const tokensPerDay = await db + .select({ + day: dayExpr.as("day"), + promptTokens: sql`COALESCE(SUM(${aiUsageRecords.promptTokens}), 0)`, + cacheReadTokens: sql`COALESCE(SUM(${aiUsageRecords.cacheReadTokens}), 0)`, + cacheWriteTokens: sql`COALESCE(SUM(${aiUsageRecords.cacheWriteTokens}), 0)`, + completionTokens: sql`COALESCE(SUM(${aiUsageRecords.completionTokens}), 0)`, + reasoningTokens: sql`COALESCE(SUM(${aiUsageRecords.reasoningTokens}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(dayExpr) + .orderBy(dayExpr); + + const costPerDay = await db + .select({ + day: dayExpr.as("day"), + cost: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(dayExpr) + .orderBy(dayExpr); + + const modelByDay = await db + .select({ + day: dayExpr.as("day"), + model: aiUsageRecords.requestedModel, + cost: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`, + tokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(dayExpr, aiUsageRecords.requestedModel) + .orderBy(dayExpr); + + const modelCostTotals = new Map(); + const modelTokenTotals = new Map(); + for (const row of modelByDay) { + modelCostTotals.set( + row.model, + (modelCostTotals.get(row.model) ?? 0) + row.cost + ); + modelTokenTotals.set( + row.model, + (modelTokenTotals.get(row.model) ?? 0) + row.tokens + ); + } + + const topModelsByCost = pickTopNKeys(modelCostTotals); + const topModelsByTokens = pickTopNKeys(modelTokenTotals); + + const modelCostPerDay = bucketTopNPerDay( + modelByDay.map((r) => ({ day: r.day, key: r.model, value: r.cost })), + topModelsByCost + ); + const modelTokensPerDay = bucketTopNPerDay( + modelByDay.map((r) => ({ day: r.day, key: r.model, value: r.tokens })), + topModelsByTokens + ); + + const topModelsRaw = await db + .select({ + model: aiUsageRecords.requestedModel, + requests: count(), + totalTokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)`, + costUsd: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(aiUsageRecords.requestedModel) + .orderBy(desc(sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`)) + .limit(DISTINCT_LIMIT + 1); + + if (topModelsRaw.length > DISTINCT_LIMIT) { + throw createHttpError( + HttpCode.BAD_REQUEST, + "Too many distinct models. Please narrow your query." + ); + } + + return { + totalRequests: totalsRow.requests, + totalTokens: totalsRow.totalTokens, + totalCost: totalsRow.costUsd, + estimatedPercent: + totalsRow.requests > 0 + ? (totalsRow.estimatedRequests / totalsRow.requests) * 100 + : 0, + tokenBreakdown: { + promptTokens: totalsRow.promptTokens, + cacheReadTokens: totalsRow.cacheReadTokens, + cacheWriteTokens: totalsRow.cacheWriteTokens, + completionTokens: totalsRow.completionTokens, + reasoningTokens: totalsRow.reasoningTokens + }, + requestsPerDay, + tokensPerDay, + costPerDay, + modelCostPerDay, + modelTokensPerDay, + topModels: topModelsRaw + }; +} + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/logs/ai/usage/overview", + description: "Query the AI usage analytics overview for an organization", + tags: [OpenAPITags.Logs], + request: { + query: aiUsageAnalyticsFiltersQuery, + params: aiUsageAnalyticsParams + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export type QueryAiUsageOverviewResponse = Awaited>; + +export async function queryAiUsageOverview( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = aiUsageAnalyticsFiltersQuery.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedQuery.error)) + ); + } + + const parsedParams = aiUsageAnalyticsParams.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedParams.error)) + ); + } + + const data = await query({ ...parsedQuery.data, ...parsedParams.data }); + + return response(res, { + data, + success: true, + error: false, + message: "AI usage overview retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/auditLogs/queryAiUsageProviders.ts b/server/routers/auditLogs/queryAiUsageProviders.ts new file mode 100644 index 000000000..77df6330d --- /dev/null +++ b/server/routers/auditLogs/queryAiUsageProviders.ts @@ -0,0 +1,183 @@ +import { db, aiUsageRecords, aiProviders } from "@server/db"; +import { registry } from "@server/openApi"; +import { NextFunction } from "express"; +import { Request, Response } from "express"; +import { count, desc, inArray, sql } from "drizzle-orm"; +import { OpenAPITags } from "@server/openApi"; +import { z } from "zod"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { fromError } from "zod-validation-error"; +import response from "@server/lib/response"; +import logger from "@server/logger"; +import { + aiUsageAnalyticsFiltersQuery, + aiUsageAnalyticsParams, + buildAiUsageWhere, + resolveRoleUserIds, + dayBucketExpr, + pickTopNKeys, + bucketTopNPerDay, + DISTINCT_LIMIT, + type AiUsageAnalyticsQuery +} from "./aiUsageAnalyticsShared"; + +type Q = AiUsageAnalyticsQuery; + +async function query(data: Q) { + const roleUserIds = await resolveRoleUserIds(data.orgId, data.roleId); + const baseConditions = buildAiUsageWhere(data, roleUserIds); + const dayExpr = dayBucketExpr(); + + const providerByDay = await db + .select({ + day: dayExpr.as("day"), + providerId: aiUsageRecords.providerId, + cost: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`, + tokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(dayExpr, aiUsageRecords.providerId) + .orderBy(dayExpr); + + const costTotals = new Map(); + const tokenTotals = new Map(); + for (const row of providerByDay) { + const key = String(row.providerId); + costTotals.set(key, (costTotals.get(key) ?? 0) + row.cost); + tokenTotals.set(key, (tokenTotals.get(key) ?? 0) + row.tokens); + } + + const topByCost = pickTopNKeys(costTotals); + const topByTokens = pickTopNKeys(tokenTotals); + + const providerCostPerDay = bucketTopNPerDay( + providerByDay.map((r) => ({ + day: r.day, + key: String(r.providerId), + value: r.cost + })), + topByCost + ); + const providerTokensPerDay = bucketTopNPerDay( + providerByDay.map((r) => ({ + day: r.day, + key: String(r.providerId), + value: r.tokens + })), + topByTokens + ); + + const topProvidersRaw = await db + .select({ + providerId: aiUsageRecords.providerId, + requests: count(), + totalTokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)`, + costUsd: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(aiUsageRecords.providerId) + .orderBy(desc(sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`)) + .limit(DISTINCT_LIMIT + 1); + + if (topProvidersRaw.length > DISTINCT_LIMIT) { + throw createHttpError( + HttpCode.BAD_REQUEST, + "Too many distinct providers. Please narrow your query." + ); + } + + const providerIds = topProvidersRaw.map((r) => r.providerId); + const nameMap = new Map(); + if (providerIds.length > 0) { + const providerDetails = await db + .select({ providerId: aiProviders.providerId, name: aiProviders.name }) + .from(aiProviders) + .where(inArray(aiProviders.providerId, providerIds)); + for (const p of providerDetails) { + nameMap.set(p.providerId, p.name); + } + } + + const topProviders = topProvidersRaw.map((r) => ({ + providerId: r.providerId, + name: nameMap.get(r.providerId) ?? null, + requests: r.requests, + totalTokens: r.totalTokens, + costUsd: r.costUsd + })); + + return { + providerCostPerDay, + providerTokensPerDay, + topProviders + }; +} + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/logs/ai/usage/providers", + description: "Query the AI usage analytics provider breakdown for an organization", + tags: [OpenAPITags.Logs], + request: { + query: aiUsageAnalyticsFiltersQuery, + params: aiUsageAnalyticsParams + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export type QueryAiUsageProvidersResponse = Awaited>; + +export async function queryAiUsageProviders( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = aiUsageAnalyticsFiltersQuery.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedQuery.error)) + ); + } + + const parsedParams = aiUsageAnalyticsParams.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedParams.error)) + ); + } + + const data = await query({ ...parsedQuery.data, ...parsedParams.data }); + + return response(res, { + data, + success: true, + error: false, + message: "AI usage provider breakdown retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/auditLogs/queryAiUsageResources.ts b/server/routers/auditLogs/queryAiUsageResources.ts new file mode 100644 index 000000000..fb2fddbc4 --- /dev/null +++ b/server/routers/auditLogs/queryAiUsageResources.ts @@ -0,0 +1,222 @@ +import { db, aiUsageRecords, resources, siteResources } from "@server/db"; +import { registry } from "@server/openApi"; +import { NextFunction } from "express"; +import { Request, Response } from "express"; +import { count, desc, inArray, sql } from "drizzle-orm"; +import { OpenAPITags } from "@server/openApi"; +import { z } from "zod"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { fromError } from "zod-validation-error"; +import response from "@server/lib/response"; +import logger from "@server/logger"; +import { + aiUsageAnalyticsFiltersQuery, + aiUsageAnalyticsParams, + buildAiUsageWhere, + resolveRoleUserIds, + dayBucketExpr, + pickTopNKeys, + bucketTopNPerDay, + DISTINCT_LIMIT, + type AiUsageAnalyticsQuery +} from "./aiUsageAnalyticsShared"; + +type Q = AiUsageAnalyticsQuery; + +// Composite key namespacing resourceId ("r:") vs siteResourceId ("s:") since +// the two id spaces are independent and can overlap numerically. +function resourceKey(resourceId: number | null, siteResourceId: number | null) { + if (resourceId != null) return `r:${resourceId}`; + if (siteResourceId != null) return `s:${siteResourceId}`; + return "none"; +} + +async function query(data: Q) { + const roleUserIds = await resolveRoleUserIds(data.orgId, data.roleId); + const baseConditions = buildAiUsageWhere(data, roleUserIds); + const dayExpr = dayBucketExpr(); + + const resourceByDay = await db + .select({ + day: dayExpr.as("day"), + resourceId: aiUsageRecords.resourceId, + siteResourceId: aiUsageRecords.siteResourceId, + cost: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`, + tokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(dayExpr, aiUsageRecords.resourceId, aiUsageRecords.siteResourceId) + .orderBy(dayExpr); + + const costTotals = new Map(); + const tokenTotals = new Map(); + for (const row of resourceByDay) { + const key = resourceKey(row.resourceId, row.siteResourceId); + costTotals.set(key, (costTotals.get(key) ?? 0) + row.cost); + tokenTotals.set(key, (tokenTotals.get(key) ?? 0) + row.tokens); + } + + const topByCost = pickTopNKeys(costTotals); + const topByTokens = pickTopNKeys(tokenTotals); + + const resourceCostPerDay = bucketTopNPerDay( + resourceByDay.map((r) => ({ + day: r.day, + key: resourceKey(r.resourceId, r.siteResourceId), + value: r.cost + })), + topByCost + ); + const resourceTokensPerDay = bucketTopNPerDay( + resourceByDay.map((r) => ({ + day: r.day, + key: resourceKey(r.resourceId, r.siteResourceId), + value: r.tokens + })), + topByTokens + ); + + const topResourcesRaw = await db + .select({ + resourceId: aiUsageRecords.resourceId, + siteResourceId: aiUsageRecords.siteResourceId, + requests: count(), + totalTokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)`, + costUsd: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(aiUsageRecords.resourceId, aiUsageRecords.siteResourceId) + .orderBy(desc(sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`)) + .limit(DISTINCT_LIMIT + 1); + + if (topResourcesRaw.length > DISTINCT_LIMIT) { + throw createHttpError( + HttpCode.BAD_REQUEST, + "Too many distinct resources. Please narrow your query." + ); + } + + const resourceIds = topResourcesRaw + .map((r) => r.resourceId) + .filter((id): id is number => id !== null); + const siteResourceIds = topResourcesRaw + .map((r) => r.siteResourceId) + .filter((id): id is number => id !== null); + + const nameMap = new Map(); + if (resourceIds.length > 0) { + const resourceDetails = await db + .select({ resourceId: resources.resourceId, name: resources.name }) + .from(resources) + .where(inArray(resources.resourceId, resourceIds)); + for (const r of resourceDetails) { + nameMap.set(`r:${r.resourceId}`, r.name); + } + } + if (siteResourceIds.length > 0) { + const siteResourceDetails = await db + .select({ + siteResourceId: siteResources.siteResourceId, + name: siteResources.name + }) + .from(siteResources) + .where(inArray(siteResources.siteResourceId, siteResourceIds)); + for (const r of siteResourceDetails) { + nameMap.set(`s:${r.siteResourceId}`, r.name); + } + } + + const topResources = topResourcesRaw.map((r) => { + const key = resourceKey(r.resourceId, r.siteResourceId); + return { + key, + resourceId: r.resourceId, + siteResourceId: r.siteResourceId, + type: + r.resourceId != null + ? ("public" as const) + : r.siteResourceId != null + ? ("site" as const) + : null, + name: nameMap.get(key) ?? null, + requests: r.requests, + totalTokens: r.totalTokens, + costUsd: r.costUsd + }; + }); + + return { + resourceCostPerDay, + resourceTokensPerDay, + topResources + }; +} + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/logs/ai/usage/resources", + description: "Query the AI usage analytics resource breakdown for an organization", + tags: [OpenAPITags.Logs], + request: { + query: aiUsageAnalyticsFiltersQuery, + params: aiUsageAnalyticsParams + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export type QueryAiUsageResourcesResponse = Awaited>; + +export async function queryAiUsageResources( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = aiUsageAnalyticsFiltersQuery.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedQuery.error)) + ); + } + + const parsedParams = aiUsageAnalyticsParams.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedParams.error)) + ); + } + + const data = await query({ ...parsedQuery.data, ...parsedParams.data }); + + return response(res, { + data, + success: true, + error: false, + message: "AI usage resource breakdown retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/auditLogs/queryAiUsageUsersRoles.ts b/server/routers/auditLogs/queryAiUsageUsersRoles.ts new file mode 100644 index 000000000..b98d1d847 --- /dev/null +++ b/server/routers/auditLogs/queryAiUsageUsersRoles.ts @@ -0,0 +1,271 @@ +import { db, aiUsageRecords, users, roles, userOrgRoles } from "@server/db"; +import { registry } from "@server/openApi"; +import { NextFunction } from "express"; +import { Request, Response } from "express"; +import { and, count, desc, eq, inArray, sql } from "drizzle-orm"; +import { OpenAPITags } from "@server/openApi"; +import { z } from "zod"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { fromError } from "zod-validation-error"; +import response from "@server/lib/response"; +import logger from "@server/logger"; +import { + aiUsageAnalyticsFiltersQuery, + aiUsageAnalyticsParams, + buildAiUsageWhere, + resolveRoleUserIds, + dayBucketExpr, + pickTopNKeys, + bucketTopNPerDay, + DISTINCT_LIMIT, + type AiUsageAnalyticsQuery +} from "./aiUsageAnalyticsShared"; + +type Q = AiUsageAnalyticsQuery; + +const UNKNOWN_USER_KEY = "unknown"; + +async function query(data: Q) { + const roleUserIds = await resolveRoleUserIds(data.orgId, data.roleId); + const baseConditions = buildAiUsageWhere(data, roleUserIds); + const dayExpr = dayBucketExpr(); + + // Per (day, user) is the common granularity both the user trend charts and + // the role trend charts are built from - a usage record only stores + // userId, so role totals are derived by expanding each user's usage into + // every role they hold in the org (per-role double counting for + // multi-role users is expected/accepted). + const userByDay = await db + .select({ + day: dayExpr.as("day"), + userId: aiUsageRecords.userId, + cost: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`, + tokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(dayExpr, aiUsageRecords.userId) + .orderBy(dayExpr); + + const userTotalsRaw = await db + .select({ + userId: aiUsageRecords.userId, + requests: count(), + totalTokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)`, + costUsd: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(aiUsageRecords.userId) + .orderBy(desc(sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`)) + .limit(DISTINCT_LIMIT + 1); + + if (userTotalsRaw.length > DISTINCT_LIMIT) { + throw createHttpError( + HttpCode.BAD_REQUEST, + "Too many distinct users. Please narrow your query." + ); + } + + const userIds = userTotalsRaw + .map((r) => r.userId) + .filter((id): id is string => id !== null); + + const emailMap = new Map(); + if (userIds.length > 0) { + const userDetails = await db + .select({ userId: users.userId, email: users.email }) + .from(users) + .where(inArray(users.userId, userIds)); + for (const u of userDetails) { + emailMap.set(u.userId, u.email); + } + } + + const topUsers = userTotalsRaw.map((r) => ({ + userId: r.userId, + email: r.userId ? emailMap.get(r.userId) ?? null : null, + requests: r.requests, + totalTokens: r.totalTokens, + costUsd: r.costUsd + })); + + const userCostTotals = new Map(); + const userTokenTotals = new Map(); + for (const row of userByDay) { + const key = row.userId ?? UNKNOWN_USER_KEY; + userCostTotals.set(key, (userCostTotals.get(key) ?? 0) + row.cost); + userTokenTotals.set(key, (userTokenTotals.get(key) ?? 0) + row.tokens); + } + const topUsersByCost = pickTopNKeys(userCostTotals); + const topUsersByTokens = pickTopNKeys(userTokenTotals); + + const userCostPerDay = bucketTopNPerDay( + userByDay.map((r) => ({ + day: r.day, + key: r.userId ?? UNKNOWN_USER_KEY, + value: r.cost + })), + topUsersByCost + ); + const userTokensPerDay = bucketTopNPerDay( + userByDay.map((r) => ({ + day: r.day, + key: r.userId ?? UNKNOWN_USER_KEY, + value: r.tokens + })), + topUsersByTokens + ); + + // Resolve every user's role membership(s) in this org so usage can be + // expanded into per-role totals. + const userToRoles = new Map(); + if (userIds.length > 0) { + const roleRows = await db + .select({ + userId: userOrgRoles.userId, + roleId: roles.roleId, + name: roles.name + }) + .from(userOrgRoles) + .innerJoin(roles, eq(userOrgRoles.roleId, roles.roleId)) + .where( + and( + eq(userOrgRoles.orgId, data.orgId), + inArray(userOrgRoles.userId, userIds) + ) + ); + for (const row of roleRows) { + const existing = userToRoles.get(row.userId) ?? []; + existing.push({ roleId: row.roleId, name: row.name }); + userToRoles.set(row.userId, existing); + } + } + + const roleTotals = new Map< + number, + { name: string | null; requests: number; totalTokens: number; costUsd: number } + >(); + for (const r of userTotalsRaw) { + if (!r.userId) continue; + const userRoles = userToRoles.get(r.userId) ?? []; + for (const role of userRoles) { + const existing = roleTotals.get(role.roleId) ?? { + name: role.name, + requests: 0, + totalTokens: 0, + costUsd: 0 + }; + existing.requests += r.requests; + existing.totalTokens += r.totalTokens; + existing.costUsd += r.costUsd; + roleTotals.set(role.roleId, existing); + } + } + + const topRoles = [...roleTotals.entries()] + .map(([roleId, v]) => ({ roleId, ...v })) + .sort((a, b) => b.costUsd - a.costUsd); + + const roleCostRows: { day: string; key: string; value: number }[] = []; + const roleTokenRows: { day: string; key: string; value: number }[] = []; + for (const row of userByDay) { + if (!row.userId) continue; + const userRoles = userToRoles.get(row.userId) ?? []; + for (const role of userRoles) { + roleCostRows.push({ day: row.day, key: String(role.roleId), value: row.cost }); + roleTokenRows.push({ day: row.day, key: String(role.roleId), value: row.tokens }); + } + } + + const roleCostTotals = new Map(); + const roleTokenTotals = new Map(); + for (const row of roleCostRows) { + roleCostTotals.set(row.key, (roleCostTotals.get(row.key) ?? 0) + row.value); + } + for (const row of roleTokenRows) { + roleTokenTotals.set(row.key, (roleTokenTotals.get(row.key) ?? 0) + row.value); + } + const topRolesByCost = pickTopNKeys(roleCostTotals); + const topRolesByTokens = pickTopNKeys(roleTokenTotals); + + const roleCostPerDay = bucketTopNPerDay(roleCostRows, topRolesByCost); + const roleTokensPerDay = bucketTopNPerDay(roleTokenRows, topRolesByTokens); + + return { + topUsers, + userCostPerDay, + userTokensPerDay, + topRoles, + roleCostPerDay, + roleTokensPerDay + }; +} + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/logs/ai/usage/users-roles", + description: + "Query the AI usage analytics user and role breakdown for an organization", + tags: [OpenAPITags.Logs], + request: { + query: aiUsageAnalyticsFiltersQuery, + params: aiUsageAnalyticsParams + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export type QueryAiUsageUsersRolesResponse = Awaited>; + +export async function queryAiUsageUsersRoles( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = aiUsageAnalyticsFiltersQuery.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedQuery.error)) + ); + } + + const parsedParams = aiUsageAnalyticsParams.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError(HttpCode.BAD_REQUEST, fromError(parsedParams.error)) + ); + } + + const data = await query({ ...parsedQuery.data, ...parsedParams.data }); + + return response(res, { + data, + success: true, + error: false, + message: "AI usage user/role breakdown retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/external.ts b/server/routers/external.ts index dd1167d55..b50de6bea 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1502,6 +1502,41 @@ authenticated.get( logs.exportAiSessionLogs ); +authenticated.get( + "/org/:orgId/logs/ai/usage/filters", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageFilterOptions +); + +authenticated.get( + "/org/:orgId/logs/ai/usage/overview", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageOverview +); + +authenticated.get( + "/org/:orgId/logs/ai/usage/providers", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageProviders +); + +authenticated.get( + "/org/:orgId/logs/ai/usage/resources", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageResources +); + +authenticated.get( + "/org/:orgId/logs/ai/usage/users-roles", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageUsersRoles +); + authenticated.get( "/org/:orgId/blueprints", verifyOrgAccess, diff --git a/server/routers/integration.ts b/server/routers/integration.ts index 87ec88243..02e1ab987 100644 --- a/server/routers/integration.ts +++ b/server/routers/integration.ts @@ -1545,6 +1545,41 @@ authenticated.get( logs.exportAiSessionLogs ); +authenticated.get( + "/org/:orgId/logs/ai/usage/filters", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageFilterOptions +); + +authenticated.get( + "/org/:orgId/logs/ai/usage/overview", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageOverview +); + +authenticated.get( + "/org/:orgId/logs/ai/usage/providers", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageProviders +); + +authenticated.get( + "/org/:orgId/logs/ai/usage/resources", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageResources +); + +authenticated.get( + "/org/:orgId/logs/ai/usage/users-roles", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageUsersRoles +); + authenticated.get( "/org/:orgId/logs/analytics", verifyApiKeyOrgAccess, diff --git a/src/app/[orgId]/settings/logs/ai-usage/page.tsx b/src/app/[orgId]/settings/logs/ai-usage/page.tsx new file mode 100644 index 000000000..f045aead0 --- /dev/null +++ b/src/app/[orgId]/settings/logs/ai-usage/page.tsx @@ -0,0 +1,30 @@ +import { AiUsageAnalyticsData } from "@app/components/AiUsageAnalyticsData"; +import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; +import type { Metadata } from "next"; + +export const metadata: Metadata = { + title: "AI Usage Analytics" +}; + +export interface AiUsageAnalyticsPageProps { + params: Promise<{ orgId: string }>; +} + +export default async function AiUsageAnalyticsPage( + props: AiUsageAnalyticsPageProps +) { + const orgId = (await props.params).orgId; + + return ( + <> + + +
+ +
+ + ); +} diff --git a/src/app/navigation.tsx b/src/app/navigation.tsx index 7181fd1c6..6c5e4a935 100644 --- a/src/app/navigation.tsx +++ b/src/app/navigation.tsx @@ -8,6 +8,7 @@ import { Building2, Cable, ChartLine, + Coins, Combine, CreditCard, Fingerprint, @@ -233,6 +234,11 @@ export const orgNavSections = ( href: "/{orgId}/settings/logs/ai", icon: }, + { + title: "sidebarLogsAiUsage", + href: "/{orgId}/settings/logs/ai-usage", + icon: + }, ...(!env?.flags.disableEnterpriseFeatures ? [ { @@ -532,6 +538,11 @@ export const commandBarNavSections = ( href: "/{orgId}/settings/logs/ai", icon: }, + { + title: "commandLogsAiUsage", + href: "/{orgId}/settings/logs/ai-usage", + icon: + }, ...(!env?.flags.disableEnterpriseFeatures ? [ { diff --git a/src/components/AiUsageAnalyticsData.tsx b/src/components/AiUsageAnalyticsData.tsx new file mode 100644 index 000000000..573dc96b1 --- /dev/null +++ b/src/components/AiUsageAnalyticsData.tsx @@ -0,0 +1,297 @@ +"use client"; + +import { cn } from "@app/lib/cn"; +import { + aiUsageAnalyticsFiltersSchema, + aiUsageAnalyticsQueries, + type AiUsageAnalyticsFilters +} from "@app/lib/queries"; +import { useIsFetching, useQuery, useQueryClient } from "@tanstack/react-query"; +import { RefreshCw, XIcon } from "lucide-react"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { DateRangePicker, type DateTimeValue } from "./DateTimePicker"; +import { Button } from "./ui/button"; +import { Card, CardHeader } from "./ui/card"; +import { Label } from "./ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue +} from "./ui/select"; +import { Separator } from "./ui/separator"; +import { getSevenDaysAgo } from "@app/lib/getSevenDaysAgo"; +import { HorizontalTabs, type TabItem } from "./HorizontalTabs"; +import { OverviewTab } from "./ai-usage-analytics/OverviewTab"; +import { ProvidersTab } from "./ai-usage-analytics/ProvidersTab"; +import { ResourcesTab } from "./ai-usage-analytics/ResourcesTab"; +import { UsersRolesTab } from "./ai-usage-analytics/UsersRolesTab"; + +export type AiUsageAnalyticsDataProps = { + orgId: string; +}; + +const AI_USAGE_ANALYTICS_QUERY_PREFIX = ["AI_USAGE_ANALYTICS"]; + +export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { + const searchParams = useSearchParams(); + const path = usePathname(); + const router = useRouter(); + const queryClient = useQueryClient(); + + const filters = aiUsageAnalyticsFiltersSchema.parse( + Object.fromEntries(searchParams.entries()) + ); + + const isEmptySearchParams = Object.values(filters).every( + (v) => v === undefined + ); + + const dateRange = { + startDate: filters.timeStart + ? new Date(filters.timeStart) + : getSevenDaysAgo(), + endDate: filters.timeEnd ? new Date(filters.timeEnd) : new Date() + }; + + const { data: filterOptions } = useQuery( + aiUsageAnalyticsQueries.filterOptions({ + orgId: props.orgId, + filters: { + timeStart: filters.timeStart, + timeEnd: filters.timeEnd + } + }) + ); + + const isFetching = + useIsFetching({ + queryKey: [...AI_USAGE_ANALYTICS_QUERY_PREFIX, props.orgId] + }) > 0; + + function setFilter(key: keyof AiUsageAnalyticsFilters, value?: string) { + const newSearch = new URLSearchParams(searchParams); + newSearch.delete(key); + if (value !== undefined) { + newSearch.set(key, value); + } + router.replace(`${path}?${newSearch.toString()}`); + } + + function handleTimeRangeUpdate(start: DateTimeValue, end: DateTimeValue) { + const newSearch = new URLSearchParams(searchParams); + const timeRegex = + /^(?\d{1,2})\:(?\d{1,2})(\:(?\d{1,2}))?$/; + + if (start.date) { + const startDate = new Date(start.date); + if (start.time) { + const time = timeRegex.exec(start.time); + const groups = time?.groups ?? {}; + startDate.setHours(Number(groups.hours)); + startDate.setMinutes(Number(groups.minutes)); + if (groups.seconds) { + startDate.setSeconds(Number(groups.seconds)); + } + } + newSearch.set("timeStart", startDate.toISOString()); + } + if (end.date) { + const endDate = new Date(end.date); + if (end.time) { + const time = timeRegex.exec(end.time); + const groups = time?.groups ?? {}; + endDate.setHours(Number(groups.hours)); + endDate.setMinutes(Number(groups.minutes)); + if (groups.seconds) { + endDate.setSeconds(Number(groups.seconds)); + } + } + newSearch.set("timeEnd", endDate.toISOString()); + } + router.replace(`${path}?${newSearch.toString()}`); + } + + function getDateTime(date: Date) { + return `${date.getHours()}:${date.getMinutes()}`; + } + + const providerOptions = (filterOptions?.providers ?? []).map((p) => ({ + value: String(p.id), + label: p.name ?? `Provider #${p.id}` + })); + const modelOptions = (filterOptions?.models ?? []).map((m) => ({ + value: m, + label: m + })); + const resourceOptions = (filterOptions?.resources ?? []).map((r) => ({ + value: String(r.id), + label: r.name ?? `Resource #${r.id}` + })); + const roleOptions = (filterOptions?.roles ?? []).map((r) => ({ + value: String(r.id), + label: r.name ?? `Role #${r.id}` + })); + const userOptions = (filterOptions?.users ?? []).map((u) => ({ + value: u.id, + label: u.email ?? u.id + })); + + const tabs: TabItem[] = [ + { title: "Overview", href: "#" }, + { title: "Provider usage", href: "#" }, + { title: "Resources", href: "#" }, + { title: "Users & roles", href: "#" } + ]; + + return ( +
+ + +
+ + + + +
+ setFilter("providerId", v)} + /> + setFilter("model", v)} + /> + setFilter("resourceId", v)} + /> + setFilter("roleId", v)} + /> + setFilter("userId", v)} + /> + + {!isEmptySearchParams && ( + + )} +
+
+
+ +
+ + + + + + + + + +
+ ); +} + +type FilterSelectProps = { + id: string; + label: string; + value?: string; + options: { value: string; label: string }[]; + placeholder: string; + onValueChange: (value?: string) => void; +}; + +function FilterSelect(props: FilterSelectProps) { + return ( +
+ + +
+ ); +} diff --git a/src/components/ai-usage-analytics/OverviewTab.tsx b/src/components/ai-usage-analytics/OverviewTab.tsx new file mode 100644 index 000000000..30d4478b1 --- /dev/null +++ b/src/components/ai-usage-analytics/OverviewTab.tsx @@ -0,0 +1,197 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { aiUsageAnalyticsQueries } from "@app/lib/queries"; +import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { Card, CardContent, CardHeader } from "@app/components/ui/card"; +import { + InfoSection, + InfoSectionContent, + InfoSections, + InfoSectionTitle +} from "@app/components/InfoSection"; +import { ToggleableTrendChart } from "./ToggleableTrendChart"; +import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; +import { + SERIES_COLORS, + buildSeriesFromData, + compactNumberFormatter, + formatCost +} from "./shared"; + +type OverviewTabProps = { + orgId: string; + filters: AiUsageAnalyticsFilters; +}; + +const TOKEN_TYPE_LABELS: Record = { + promptTokens: "Prompt", + cacheReadTokens: "Cache read", + cacheWriteTokens: "Cache write", + completionTokens: "Completion", + reasoningTokens: "Reasoning" +}; + +export function OverviewTab(props: OverviewTabProps) { + const { data, isLoading } = useQuery( + aiUsageAnalyticsQueries.overview({ + orgId: props.orgId, + filters: props.filters + }) + ); + + const requestsSeries = [ + { key: "requests", label: "Requests", color: SERIES_COLORS[0] } + ]; + const tokensSeries = Object.keys(TOKEN_TYPE_LABELS).map((key, i) => ({ + key, + label: TOKEN_TYPE_LABELS[key], + color: SERIES_COLORS[i % SERIES_COLORS.length] + })); + const costSeries = [ + { key: "cost", label: "Cost", color: SERIES_COLORS[0] } + ]; + + const modelCostSeries = buildSeriesFromData( + data?.modelCostPerDay ?? [], + (key) => key + ); + const modelTokensSeries = buildSeriesFromData( + data?.modelTokensPerDay ?? [], + (key) => key + ); + + const topModels: TopEntity[] = (data?.topModels ?? []).map((m) => ({ + key: m.model, + label: m.model, + requests: m.requests, + totalTokens: m.totalTokens, + costUsd: m.costUsd + })); + + return ( +
+ + + + + Total requests + + {data + ? compactNumberFormatter.format( + data.totalRequests + ) + : "--"} + + + + Total tokens + + {data + ? compactNumberFormatter.format( + data.totalTokens + ) + : "--"} + + + + Total cost + + {data ? formatCost(data.totalCost) : "--"} + + + + Estimated + + {data + ? `${Math.round(data.estimatedPercent)}%` + : "--"} + + + + + + +
+ + +

Request volume

+ + + + + + + +

Token usage

+ + + + + + + +

Cost

+ + + formatCost(v)} + /> + + +
+ +
+ + +

Model cost

+ + + formatCost(v)} + /> + + + + +

Model tokens

+ + + + + +
+ + + +

Top models

+ + + + + +
+ ); +} diff --git a/src/components/ai-usage-analytics/ProvidersTab.tsx b/src/components/ai-usage-analytics/ProvidersTab.tsx new file mode 100644 index 000000000..2187ec53f --- /dev/null +++ b/src/components/ai-usage-analytics/ProvidersTab.tsx @@ -0,0 +1,91 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { aiUsageAnalyticsQueries } from "@app/lib/queries"; +import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { Card, CardContent, CardHeader } from "@app/components/ui/card"; +import { ToggleableTrendChart } from "./ToggleableTrendChart"; +import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; +import { buildSeriesFromData, formatCost } from "./shared"; + +type ProvidersTabProps = { + orgId: string; + filters: AiUsageAnalyticsFilters; +}; + +export function ProvidersTab(props: ProvidersTabProps) { + const { data, isLoading } = useQuery( + aiUsageAnalyticsQueries.providers({ + orgId: props.orgId, + filters: props.filters + }) + ); + + const nameByKey = new Map(); + for (const p of data?.topProviders ?? []) { + nameByKey.set(String(p.providerId), p.name ?? `Provider #${p.providerId}`); + } + const labelFor = (key: string) => nameByKey.get(key) ?? `Provider #${key}`; + + const costSeries = buildSeriesFromData( + data?.providerCostPerDay ?? [], + labelFor + ); + const tokensSeries = buildSeriesFromData( + data?.providerTokensPerDay ?? [], + labelFor + ); + + const topProviders: TopEntity[] = (data?.topProviders ?? []).map((p) => ({ + key: String(p.providerId), + label: p.name ?? `Provider #${p.providerId}`, + requests: p.requests, + totalTokens: p.totalTokens, + costUsd: p.costUsd + })); + + return ( +
+ + +

Top providers

+ + + + + + +
+ + +

Provider cost

+ + + formatCost(v)} + /> + + + + +

Provider token usage

+ + + + + +
+
+ ); +} diff --git a/src/components/ai-usage-analytics/ResourcesTab.tsx b/src/components/ai-usage-analytics/ResourcesTab.tsx new file mode 100644 index 000000000..39cba68d7 --- /dev/null +++ b/src/components/ai-usage-analytics/ResourcesTab.tsx @@ -0,0 +1,99 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { aiUsageAnalyticsQueries } from "@app/lib/queries"; +import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { Card, CardContent, CardHeader } from "@app/components/ui/card"; +import { ToggleableTrendChart } from "./ToggleableTrendChart"; +import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; +import { buildSeriesFromData, formatCost } from "./shared"; + +type ResourcesTabProps = { + orgId: string; + filters: AiUsageAnalyticsFilters; +}; + +function resourceTypeLabel(type: "public" | "site" | null) { + if (type === "public") return "Resource"; + if (type === "site") return "Site resource"; + return undefined; +} + +export function ResourcesTab(props: ResourcesTabProps) { + const { data, isLoading } = useQuery( + aiUsageAnalyticsQueries.resources({ + orgId: props.orgId, + filters: props.filters + }) + ); + + const nameByKey = new Map(); + for (const r of data?.topResources ?? []) { + nameByKey.set(r.key, r.name ?? r.key); + } + const labelFor = (key: string) => + key === "none" ? "No resource" : (nameByKey.get(key) ?? key); + + const costSeries = buildSeriesFromData( + data?.resourceCostPerDay ?? [], + labelFor + ); + const tokensSeries = buildSeriesFromData( + data?.resourceTokensPerDay ?? [], + labelFor + ); + + const topResources: TopEntity[] = (data?.topResources ?? []).map((r) => ({ + key: r.key, + label: r.name ?? (r.key === "none" ? "No resource" : r.key), + sublabel: resourceTypeLabel(r.type), + requests: r.requests, + totalTokens: r.totalTokens, + costUsd: r.costUsd + })); + + return ( +
+ + +

Top resources

+ + + + + + +
+ + +

Resource cost

+ + + formatCost(v)} + /> + + + + +

Resource token usage

+ + + + + +
+
+ ); +} diff --git a/src/components/ai-usage-analytics/ToggleableTrendChart.tsx b/src/components/ai-usage-analytics/ToggleableTrendChart.tsx new file mode 100644 index 000000000..5b6c78c5e --- /dev/null +++ b/src/components/ai-usage-analytics/ToggleableTrendChart.tsx @@ -0,0 +1,188 @@ +"use client"; + +import { useState } from "react"; +import { cn } from "@app/lib/cn"; +import { BarChart3, LineChart as LineChartIcon, LoaderIcon } from "lucide-react"; +import { + Bar, + BarChart, + CartesianGrid, + Line, + LineChart, + XAxis, + YAxis +} from "recharts"; +import { Button } from "@app/components/ui/button"; +import { + ChartContainer, + ChartLegend, + ChartLegendContent, + ChartTooltip, + ChartTooltipContent, + type ChartConfig +} from "@app/components/ui/chart"; + +export type TrendSeries = { + key: string; + label: string; + color: string; +}; + +export interface TrendChartRow { + day: string; + [seriesKey: string]: number | string; +} + +type ToggleableTrendChartProps = { + data: TrendChartRow[]; + series: TrendSeries[]; + isLoading?: boolean; + valueFormatter?: (value: number) => string; + className?: string; +}; + +const compactFormatter = new Intl.NumberFormat(undefined, { + maximumFractionDigits: 1, + notation: "compact", + compactDisplay: "short" +}); + +export function ToggleableTrendChart(props: ToggleableTrendChartProps) { + const [chartType, setChartType] = useState<"bar" | "line">("bar"); + + const valueFormatter = props.valueFormatter ?? compactFormatter.format; + + const chartConfig = props.series.reduce((acc, s) => { + acc[s.key] = { label: s.label, color: s.color }; + return acc; + }, {} as ChartConfig); + + const hasData = props.data.length > 0; + + return ( +
+
+ + +
+ + {!hasData ? ( +
+ {props.isLoading ? ( + <> + + Loading... + + ) : ( + "No data" + )} +
+ ) : ( + + {chartType === "bar" ? ( + + } /> + + formatDay(payload?.[0]?.payload?.day) + } + /> + } + /> + + + + {props.series.map((s) => ( + + ))} + + ) : ( + + } /> + + formatDay(payload?.[0]?.payload?.day) + } + /> + } + /> + + + + {props.series.map((s) => ( + + ))} + + )} + + )} +
+ ); +} + +function formatDay(value: unknown) { + if (typeof value !== "string") return ""; + const date = new Date(value); + if (isNaN(date.getTime())) return value; + return date.toLocaleDateString(undefined, { dateStyle: "medium" }); +} diff --git a/src/components/ai-usage-analytics/TopEntitiesList.tsx b/src/components/ai-usage-analytics/TopEntitiesList.tsx new file mode 100644 index 000000000..9f712363f --- /dev/null +++ b/src/components/ai-usage-analytics/TopEntitiesList.tsx @@ -0,0 +1,92 @@ +"use client"; + +import { LoaderIcon } from "lucide-react"; +import { compactNumberFormatter, formatCost } from "./shared"; + +export type TopEntity = { + key: string; + label: string; + sublabel?: string | null; + requests: number; + totalTokens: number; + costUsd: number | null; +}; + +type TopEntitiesListProps = { + entities: TopEntity[]; + isLoading: boolean; + nameColumnLabel: string; + emptyLabel?: string; +}; + +export function TopEntitiesList(props: TopEntitiesListProps) { + const totalCost = props.entities.reduce( + (sum, e) => sum + (e.costUsd ?? 0), + 0 + ); + + return ( +
+ {props.entities.length > 0 && ( +
+
{props.nameColumnLabel}
+
Requests
+
Tokens
+
Cost
+
%
+
+ )} +
    + {props.entities.length === 0 && ( +
    + {props.isLoading ? ( + <> + + Loading... + + ) : ( + (props.emptyLabel ?? "No data") + )} +
    + )} + {props.entities.map((entity) => { + const percent = + totalCost > 0 ? (entity.costUsd ?? 0) / totalCost : 0; + return ( +
  1. +
    +
    + {entity.label} + {entity.sublabel && ( + + {entity.sublabel} + + )} +
    +
    + {compactNumberFormatter.format(entity.requests)} +
    +
    + {compactNumberFormatter.format( + entity.totalTokens + )} +
    +
    + {formatCost(entity.costUsd)} +
    +
    + {Math.round(percent * 100)}% +
    +
    2. + ); + })} +
+
+ ); +} diff --git a/src/components/ai-usage-analytics/UsersRolesTab.tsx b/src/components/ai-usage-analytics/UsersRolesTab.tsx new file mode 100644 index 000000000..96a814fd6 --- /dev/null +++ b/src/components/ai-usage-analytics/UsersRolesTab.tsx @@ -0,0 +1,166 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { aiUsageAnalyticsQueries } from "@app/lib/queries"; +import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { Card, CardContent, CardHeader } from "@app/components/ui/card"; +import { ToggleableTrendChart } from "./ToggleableTrendChart"; +import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; +import { buildSeriesFromData, formatCost } from "./shared"; + +type UsersRolesTabProps = { + orgId: string; + filters: AiUsageAnalyticsFilters; +}; + +const UNKNOWN_USER_KEY = "unknown"; + +export function UsersRolesTab(props: UsersRolesTabProps) { + const { data, isLoading } = useQuery( + aiUsageAnalyticsQueries.usersRoles({ + orgId: props.orgId, + filters: props.filters + }) + ); + + const roleNameByKey = new Map(); + for (const r of data?.topRoles ?? []) { + roleNameByKey.set(String(r.roleId), r.name ?? `Role #${r.roleId}`); + } + const roleLabelFor = (key: string) => + roleNameByKey.get(key) ?? `Role #${key}`; + + const userEmailByKey = new Map(); + for (const u of data?.topUsers ?? []) { + if (u.userId) { + userEmailByKey.set(u.userId, u.email ?? u.userId); + } + } + const userLabelFor = (key: string) => + key === UNKNOWN_USER_KEY + ? "Unknown user" + : (userEmailByKey.get(key) ?? key); + + const roleCostSeries = buildSeriesFromData( + data?.roleCostPerDay ?? [], + roleLabelFor + ); + const roleTokensSeries = buildSeriesFromData( + data?.roleTokensPerDay ?? [], + roleLabelFor + ); + const userCostSeries = buildSeriesFromData( + data?.userCostPerDay ?? [], + userLabelFor + ); + const userTokensSeries = buildSeriesFromData( + data?.userTokensPerDay ?? [], + userLabelFor + ); + + const topRoles: TopEntity[] = (data?.topRoles ?? []).map((r) => ({ + key: String(r.roleId), + label: r.name ?? `Role #${r.roleId}`, + requests: r.requests, + totalTokens: r.totalTokens, + costUsd: r.costUsd + })); + + const topUsers: TopEntity[] = (data?.topUsers ?? []).map((u) => ({ + key: u.userId ?? UNKNOWN_USER_KEY, + label: u.email ?? u.userId ?? "Unknown user", + requests: u.requests, + totalTokens: u.totalTokens, + costUsd: u.costUsd + })); + + return ( +
+
+

Roles

+ + +

Top roles

+ + + + + +
+ + +

Role cost

+ + + formatCost(v)} + /> + + + + +

Role token usage

+ + + + + +
+
+ +
+

Users

+ + +

Top users

+ + + + + +
+ + +

User cost

+ + + formatCost(v)} + /> + + + + +

User token usage

+ + + + + +
+
+
+ ); +} diff --git a/src/components/ai-usage-analytics/shared.ts b/src/components/ai-usage-analytics/shared.ts new file mode 100644 index 000000000..1f551f8fa --- /dev/null +++ b/src/components/ai-usage-analytics/shared.ts @@ -0,0 +1,64 @@ +import type { TrendSeries } from "./ToggleableTrendChart"; + +// Matches the theme's 5 categorical chart colors (--chart-1..--chart-5 in +// src/app/globals.css) - the same ceiling RequestChart already respects. +export const SERIES_COLORS = [ + "var(--chart-1)", + "var(--chart-2)", + "var(--chart-3)", + "var(--chart-4)", + "var(--chart-5)" +]; +export const OTHER_COLOR = "var(--muted-foreground)"; +export const OTHER_KEY = "other"; + +// The server already collapsed each day's row down to the top-N dimension +// keys (already ranked) plus an optional "other" bucket - so the full set of +// series can be derived straight from the data's own keys, no separate +// top-list needed. Assigns one categorical color per key, "other" last. +export function buildSeriesFromData( + data: Array>, + labelFor: (key: string) => string +): TrendSeries[] { + const keys = new Set(); + for (const row of data) { + for (const key of Object.keys(row)) { + if (key !== "day" && key !== OTHER_KEY) { + keys.add(key); + } + } + } + + const series: TrendSeries[] = [...keys].map((key, i) => ({ + key, + label: labelFor(key), + color: SERIES_COLORS[i % SERIES_COLORS.length] + })); + + const hasOther = data.some((row) => OTHER_KEY in row); + if (hasOther) { + series.push({ key: OTHER_KEY, label: "Other", color: OTHER_COLOR }); + } + + return series; +} + +export const currencyFormatter = new Intl.NumberFormat(undefined, { + style: "currency", + currency: "USD", + maximumFractionDigits: 2 +}); + +export const compactNumberFormatter = new Intl.NumberFormat(undefined, { + maximumFractionDigits: 1, + notation: "compact", + compactDisplay: "short" +}); + +export const exactNumberFormatter = new Intl.NumberFormat(undefined, { + maximumFractionDigits: 0 +}); + +export function formatCost(value: number | null | undefined) { + return currencyFormatter.format(value ?? 0); +} diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 0275d108b..bf7f69b8f 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -6,7 +6,14 @@ import { type BatchedStatusHistoryResponse } from "@server/lib/statusHistory"; import type { ListAlertRulesResponse } from "@server/routers/alertRule/types"; -import type { QueryRequestAnalyticsResponse } from "@server/routers/auditLogs"; +import type { + QueryRequestAnalyticsResponse, + QueryAiUsageFilterOptionsResponse, + QueryAiUsageOverviewResponse, + QueryAiUsageProvidersResponse, + QueryAiUsageResourcesResponse, + QueryAiUsageUsersRolesResponse +} from "@server/routers/auditLogs"; import type { QueryAccessAuditLogResponse, QueryActionAuditLogResponse, @@ -928,6 +935,32 @@ export const logAnalyticsFiltersSchema = z.object({ export type LogAnalyticsFilters = z.output; +export const aiUsageAnalyticsFiltersSchema = z.object({ + timeStart: z + .string() + .refine((val) => !isNaN(Date.parse(val)), { + error: "timeStart must be a valid ISO date string" + }) + .optional() + .catch(undefined), + timeEnd: z + .string() + .refine((val) => !isNaN(Date.parse(val)), { + error: "timeEnd must be a valid ISO date string" + }) + .optional() + .catch(undefined), + providerId: z.coerce.number().optional().catch(undefined), + model: z.string().optional().catch(undefined), + resourceId: z.coerce.number().optional().catch(undefined), + roleId: z.coerce.number().optional().catch(undefined), + userId: z.string().optional().catch(undefined) +}); + +export type AiUsageAnalyticsFilters = z.output< + typeof aiUsageAnalyticsFiltersSchema +>; + export const httpLogsFiltersSchema = z.object({ timeStart: z .string() @@ -1242,6 +1275,137 @@ export const logQueries = { }) }; +export const aiUsageAnalyticsQueries = { + filterOptions: ({ + orgId, + filters + }: { + orgId: string; + filters: Pick; + }) => + queryOptions({ + queryKey: ["AI_USAGE_ANALYTICS", orgId, "FILTERS", filters] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/org/${orgId}/logs/ai/usage/filters`, { + params: filters, + signal + }); + return res.data.data; + } + }), + + overview: ({ + orgId, + filters + }: { + orgId: string; + filters: AiUsageAnalyticsFilters; + }) => + queryOptions({ + queryKey: ["AI_USAGE_ANALYTICS", orgId, "OVERVIEW", filters] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/org/${orgId}/logs/ai/usage/overview`, { + params: filters, + signal + }); + return res.data.data; + }, + refetchInterval: (query) => { + if (query.state.data) { + return durationToMs(30, "seconds"); + } + return false; + } + }), + + providers: ({ + orgId, + filters + }: { + orgId: string; + filters: AiUsageAnalyticsFilters; + }) => + queryOptions({ + queryKey: ["AI_USAGE_ANALYTICS", orgId, "PROVIDERS", filters] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/org/${orgId}/logs/ai/usage/providers`, { + params: filters, + signal + }); + return res.data.data; + }, + refetchInterval: (query) => { + if (query.state.data) { + return durationToMs(30, "seconds"); + } + return false; + } + }), + + resources: ({ + orgId, + filters + }: { + orgId: string; + filters: AiUsageAnalyticsFilters; + }) => + queryOptions({ + queryKey: ["AI_USAGE_ANALYTICS", orgId, "RESOURCES", filters] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/org/${orgId}/logs/ai/usage/resources`, { + params: filters, + signal + }); + return res.data.data; + }, + refetchInterval: (query) => { + if (query.state.data) { + return durationToMs(30, "seconds"); + } + return false; + } + }), + + usersRoles: ({ + orgId, + filters + }: { + orgId: string; + filters: AiUsageAnalyticsFilters; + }) => + queryOptions({ + queryKey: [ + "AI_USAGE_ANALYTICS", + orgId, + "USERS_ROLES", + filters + ] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/org/${orgId}/logs/ai/usage/users-roles`, { + params: filters, + signal + }); + return res.data.data; + }, + refetchInterval: (query) => { + if (query.state.data) { + return durationToMs(30, "seconds"); + } + return false; + } + }) +}; + export const aiProviderQueries = { providerTargets: ({ providerId }: { providerId: number }) => queryOptions({ From 11595f5f96d62aefbac57a99e089a5c99c47e4c2 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 11 Aug 2026 18:02:17 -0400 Subject: [PATCH 131/296] use title case and fix colors --- .../routers/auditLogs/queryAiUsageResources.ts | 14 ++++++++------ src/components/AiUsageAnalyticsData.tsx | 16 ++++++++-------- .../ai-usage-analytics/OverviewTab.tsx | 16 ++++++++-------- .../ai-usage-analytics/ProvidersTab.tsx | 6 +++--- .../ai-usage-analytics/ResourcesTab.tsx | 6 +++--- .../ai-usage-analytics/ToggleableTrendChart.tsx | 4 ++-- .../ai-usage-analytics/UsersRolesTab.tsx | 12 ++++++------ src/components/ui/chart.tsx | 2 +- 8 files changed, 39 insertions(+), 37 deletions(-) diff --git a/server/routers/auditLogs/queryAiUsageResources.ts b/server/routers/auditLogs/queryAiUsageResources.ts index fb2fddbc4..24bac8644 100644 --- a/server/routers/auditLogs/queryAiUsageResources.ts +++ b/server/routers/auditLogs/queryAiUsageResources.ts @@ -24,11 +24,13 @@ import { type Q = AiUsageAnalyticsQuery; -// Composite key namespacing resourceId ("r:") vs siteResourceId ("s:") since -// the two id spaces are independent and can overlap numerically. +// Composite key namespacing resourceId ("r-") vs siteResourceId ("s-") since +// the two id spaces are independent and can overlap numerically. Uses a dash +// rather than a colon so the key stays safe to use as a CSS custom-property +// name suffix (e.g. --color-r-1) on the client. function resourceKey(resourceId: number | null, siteResourceId: number | null) { - if (resourceId != null) return `r:${resourceId}`; - if (siteResourceId != null) return `s:${siteResourceId}`; + if (resourceId != null) return `r-${resourceId}`; + if (siteResourceId != null) return `s-${siteResourceId}`; return "none"; } @@ -113,7 +115,7 @@ async function query(data: Q) { .from(resources) .where(inArray(resources.resourceId, resourceIds)); for (const r of resourceDetails) { - nameMap.set(`r:${r.resourceId}`, r.name); + nameMap.set(`r-${r.resourceId}`, r.name); } } if (siteResourceIds.length > 0) { @@ -125,7 +127,7 @@ async function query(data: Q) { .from(siteResources) .where(inArray(siteResources.siteResourceId, siteResourceIds)); for (const r of siteResourceDetails) { - nameMap.set(`s:${r.siteResourceId}`, r.name); + nameMap.set(`s-${r.siteResourceId}`, r.name); } } diff --git a/src/components/AiUsageAnalyticsData.tsx b/src/components/AiUsageAnalyticsData.tsx index 573dc96b1..879aff1bc 100644 --- a/src/components/AiUsageAnalyticsData.tsx +++ b/src/components/AiUsageAnalyticsData.tsx @@ -140,9 +140,9 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { const tabs: TabItem[] = [ { title: "Overview", href: "#" }, - { title: "Provider usage", href: "#" }, + { title: "Provider Usage", href: "#" }, { title: "Resources", href: "#" }, - { title: "Users & roles", href: "#" } + { title: "Users & Roles", href: "#" } ]; return ( @@ -175,7 +175,7 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { label="Provider" value={filters.providerId?.toString()} options={providerOptions} - placeholder="All providers" + placeholder="All Providers" onValueChange={(v) => setFilter("providerId", v)} /> setFilter("model", v)} /> setFilter("resourceId", v)} /> setFilter("roleId", v)} /> setFilter("userId", v)} /> @@ -218,7 +218,7 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { className="gap-2" > - Reset filters + Reset Filters )}
diff --git a/src/components/ai-usage-analytics/OverviewTab.tsx b/src/components/ai-usage-analytics/OverviewTab.tsx index 30d4478b1..1bf4dab1d 100644 --- a/src/components/ai-usage-analytics/OverviewTab.tsx +++ b/src/components/ai-usage-analytics/OverviewTab.tsx @@ -75,7 +75,7 @@ export function OverviewTab(props: OverviewTabProps) { - Total requests + Total Requests {data ? compactNumberFormatter.format( @@ -85,7 +85,7 @@ export function OverviewTab(props: OverviewTabProps) { - Total tokens + Total Tokens {data ? compactNumberFormatter.format( @@ -95,7 +95,7 @@ export function OverviewTab(props: OverviewTabProps) { - Total cost + Total Cost {data ? formatCost(data.totalCost) : "--"} @@ -115,7 +115,7 @@ export function OverviewTab(props: OverviewTabProps) {
-

Request volume

+

Request Volume

 -

Token usage

+

Token Usage

 -

Model cost

+

Model Cost

 -

Model tokens

+

Model Tokens

 -

Top models

+

Top Models

 -

Top providers

+

Top Providers

 -

Provider cost

+

Provider Cost

 -

Provider token usage

+

Provider Token Usage

 -

Top resources

+

Top Resources

 -

Resource cost

+

Resource Cost

 -

Resource token usage

+

Resource Token Usage

 @@ -165,7 +165,7 @@ export function ToggleableTrendChart(props: ToggleableTrendChartProps) { Roles -

Top roles

+

Top Roles

 -

Role cost

+

Role Cost

 -

Role token usage

+

Role Token Usage

 Users -

Top users

+

Top Users

 -

User cost

+

User Cost

 -

User token usage

+

User Token Usage

 - + {children}
From ac3402a8b37c2e7c5366f29e78faebd18c4ad50a Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 12 Aug 2026 09:41:42 -0400 Subject: [PATCH 132/296] Adjust structure delete models --- config/models.json | 7559 ---------------------------------- server/lib/aiModelCatalog.ts | 43 +- server/lib/aiModelPricing.ts | 8 +- 3 files changed, 19 insertions(+), 7591 deletions(-) delete mode 100644 config/models.json diff --git a/config/models.json b/config/models.json deleted file mode 100644 index dccd0a370..000000000 --- a/config/models.json +++ /dev/null @@ -1,7559 +0,0 @@ -{ - "data": [ - { - "id": "ai21.j2-mid-v1", - "name": "ai21.j2-mid-v1", - "provider": "bedrock", - "input_cost_per_token": 0.0000125, - "output_cost_per_token": 0.0000125, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ai21.j2-ultra-v1", - "name": "ai21.j2-ultra-v1", - "provider": "bedrock", - "input_cost_per_token": 0.0000188, - "output_cost_per_token": 0.0000188, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ai21.jamba-1-5-large-v1:0", - "name": "ai21.jamba-1-5-large-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ai21.jamba-1-5-mini-v1:0", - "name": "ai21.jamba-1-5-mini-v1:0", - "provider": "bedrock", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ai21.jamba-instruct-v1:0", - "name": "ai21.jamba-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.writer.palmyra-x4-v1:0", - "name": "us.writer.palmyra-x4-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.writer.palmyra-x5-v1:0", - "name": "us.writer.palmyra-x5-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "writer.palmyra-x4-v1:0", - "name": "writer.palmyra-x4-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "writer.palmyra-x5-v1:0", - "name": "writer.palmyra-x5-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "amazon.nova-lite-v1:0", - "name": "amazon.nova-lite-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 6e-8, - "output_cost_per_token": 2.4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "amazon.nova-2-lite-v1:0", - "name": "amazon.nova-2-lite-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "amazon.nova-2-pro-preview-20251202-v1:0", - "name": "amazon.nova-2-pro-preview-20251202-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000021875, - "output_cost_per_token": 0.0000175, - "cache_read_input_token_cost": 5.46875e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.amazon.nova-2-lite-v1:0", - "name": "apac.amazon.nova-2-lite-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 3.3e-7, - "output_cost_per_token": 0.00000275, - "cache_read_input_token_cost": 8.25e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.amazon.nova-2-pro-preview-20251202-v1:0", - "name": "apac.amazon.nova-2-pro-preview-20251202-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000021875, - "output_cost_per_token": 0.0000175, - "cache_read_input_token_cost": 5.46875e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.amazon.nova-2-lite-v1:0", - "name": "eu.amazon.nova-2-lite-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 3.3e-7, - "output_cost_per_token": 0.00000275, - "cache_read_input_token_cost": 8.25e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.amazon.nova-2-pro-preview-20251202-v1:0", - "name": "eu.amazon.nova-2-pro-preview-20251202-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000021875, - "output_cost_per_token": 0.0000175, - "cache_read_input_token_cost": 5.46875e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.amazon.nova-2-lite-v1:0", - "name": "us.amazon.nova-2-lite-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 3.3e-7, - "output_cost_per_token": 0.00000275, - "cache_read_input_token_cost": 8.25e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.amazon.nova-2-pro-preview-20251202-v1:0", - "name": "us.amazon.nova-2-pro-preview-20251202-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000021875, - "output_cost_per_token": 0.0000175, - "cache_read_input_token_cost": 5.46875e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "amazon.nova-micro-v1:0", - "name": "amazon.nova-micro-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 3.5e-8, - "output_cost_per_token": 1.4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "amazon.nova-pro-v1:0", - "name": "amazon.nova-pro-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 8e-7, - "output_cost_per_token": 0.0000032, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "twelvelabs.pegasus-1-2-v1:0", - "name": "twelvelabs.pegasus-1-2-v1:0", - "provider": "bedrock", - "input_cost_per_token": null, - "output_cost_per_token": 0.0000075, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.twelvelabs.pegasus-1-2-v1:0", - "name": "us.twelvelabs.pegasus-1-2-v1:0", - "provider": "bedrock", - "input_cost_per_token": null, - "output_cost_per_token": 0.0000075, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.twelvelabs.pegasus-1-2-v1:0", - "name": "eu.twelvelabs.pegasus-1-2-v1:0", - "provider": "bedrock", - "input_cost_per_token": null, - "output_cost_per_token": 0.0000075, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "amazon.titan-text-express-v1", - "name": "amazon.titan-text-express-v1", - "provider": "bedrock", - "input_cost_per_token": 0.0000013, - "output_cost_per_token": 0.0000017, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "amazon.titan-text-lite-v1", - "name": "amazon.titan-text-lite-v1", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "amazon.titan-text-premier-v1:0", - "name": "amazon.titan-text-premier-v1:0", - "provider": "bedrock", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-3-5-haiku-20241022-v1:0", - "name": "anthropic.claude-3-5-haiku-20241022-v1:0", - "provider": "bedrock", - "input_cost_per_token": 8e-7, - "output_cost_per_token": 0.000004, - "cache_read_input_token_cost": 8e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-haiku-4-5-20251001-v1:0", - "name": "anthropic.claude-haiku-4-5-20251001-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-haiku-4-5@20251001", - "name": "anthropic.claude-haiku-4-5@20251001", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-3-5-sonnet-20240620-v1:0", - "name": "anthropic.claude-3-5-sonnet-20240620-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "name": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-3-7-sonnet-20240620-v1:0", - "name": "anthropic.claude-3-7-sonnet-20240620-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000036, - "output_cost_per_token": 0.000018, - "cache_read_input_token_cost": 3.6e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-3-7-sonnet-20250219-v1:0", - "name": "anthropic.claude-3-7-sonnet-20250219-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-3-haiku-20240307-v1:0", - "name": "anthropic.claude-3-haiku-20240307-v1:0", - "provider": "bedrock", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-3-opus-20240229-v1:0", - "name": "anthropic.claude-3-opus-20240229-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-3-sonnet-20240229-v1:0", - "name": "anthropic.claude-3-sonnet-20240229-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-instant-v1", - "name": "anthropic.claude-instant-v1", - "provider": "bedrock", - "input_cost_per_token": 8e-7, - "output_cost_per_token": 0.0000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-opus-4-1-20250805-v1:0", - "name": "anthropic.claude-opus-4-1-20250805-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-opus-4-20250514-v1:0", - "name": "anthropic.claude-opus-4-20250514-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-opus-4-5-20251101-v1:0", - "name": "anthropic.claude-opus-4-5-20251101-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-opus-4-6-v1", - "name": "anthropic.claude-opus-4-6-v1", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-opus-4-6-v1", - "name": "global.anthropic.claude-opus-4-6-v1", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-opus-4-6-v1", - "name": "us.anthropic.claude-opus-4-6-v1", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-opus-4-6-v1", - "name": "eu.anthropic.claude-opus-4-6-v1", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "au.anthropic.claude-opus-4-6-v1", - "name": "au.anthropic.claude-opus-4-6-v1", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-opus-4-7", - "name": "anthropic.claude-opus-4-7", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-mythos-preview", - "name": "anthropic.claude-mythos-preview", - "provider": "bedrock", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-opus-4-7", - "name": "global.anthropic.claude-opus-4-7", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-opus-4-7", - "name": "us.anthropic.claude-opus-4-7", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-opus-4-7", - "name": "eu.anthropic.claude-opus-4-7", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "au.anthropic.claude-opus-4-7", - "name": "au.anthropic.claude-opus-4-7", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-fable-5", - "name": "anthropic.claude-fable-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00005, - "cache_read_input_token_cost": 0.000001, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-fable-5", - "name": "global.anthropic.claude-fable-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00005, - "cache_read_input_token_cost": 0.000001, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-fable-5", - "name": "us.anthropic.claude-fable-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000011, - "output_cost_per_token": 0.000055, - "cache_read_input_token_cost": 0.0000011, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-fable-5", - "name": "eu.anthropic.claude-fable-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000011, - "output_cost_per_token": 0.000055, - "cache_read_input_token_cost": 0.0000011, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-opus-5", - "name": "anthropic.claude-opus-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-opus-5", - "name": "global.anthropic.claude-opus-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-opus-5", - "name": "us.anthropic.claude-opus-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-opus-5", - "name": "eu.anthropic.claude-opus-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "au.anthropic.claude-opus-5", - "name": "au.anthropic.claude-opus-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "jp.anthropic.claude-opus-5", - "name": "jp.anthropic.claude-opus-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-opus-4-8", - "name": "anthropic.claude-opus-4-8", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-opus-4-8", - "name": "global.anthropic.claude-opus-4-8", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-opus-4-8", - "name": "us.anthropic.claude-opus-4-8", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-opus-4-8", - "name": "eu.anthropic.claude-opus-4-8", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "au.anthropic.claude-opus-4-8", - "name": "au.anthropic.claude-opus-4-8", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "jp.anthropic.claude-opus-4-8", - "name": "jp.anthropic.claude-opus-4-8", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "jp.anthropic.claude-opus-4-7", - "name": "jp.anthropic.claude-opus-4-7", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-sonnet-5", - "name": "anthropic.claude-sonnet-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-sonnet-5", - "name": "global.anthropic.claude-sonnet-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-sonnet-5", - "name": "us.anthropic.claude-sonnet-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000022, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 2.2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-sonnet-5", - "name": "eu.anthropic.claude-sonnet-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000022, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 2.2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "au.anthropic.claude-sonnet-5", - "name": "au.anthropic.claude-sonnet-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000022, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 2.2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "jp.anthropic.claude-sonnet-5", - "name": "jp.anthropic.claude-sonnet-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000022, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 2.2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-sonnet-4-6", - "name": "anthropic.claude-sonnet-4-6", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-sonnet-4-6", - "name": "global.anthropic.claude-sonnet-4-6", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-sonnet-4-6", - "name": "us.anthropic.claude-sonnet-4-6", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000033, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 3.3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-sonnet-4-6", - "name": "eu.anthropic.claude-sonnet-4-6", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000033, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 3.3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "au.anthropic.claude-sonnet-4-6", - "name": "au.anthropic.claude-sonnet-4-6", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000033, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 3.3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "jp.anthropic.claude-sonnet-4-6", - "name": "jp.anthropic.claude-sonnet-4-6", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000033, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 3.3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-sonnet-4-20250514-v1:0", - "name": "anthropic.claude-sonnet-4-20250514-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-sonnet-4-5-20250929-v1:0", - "name": "anthropic.claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-v1", - "name": "anthropic.claude-v1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "anthropic.claude-v2:1", - "name": "anthropic.claude-v2:1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.amazon.nova-lite-v1:0", - "name": "apac.amazon.nova-lite-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 6.3e-8, - "output_cost_per_token": 2.52e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.amazon.nova-micro-v1:0", - "name": "apac.amazon.nova-micro-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 3.7e-8, - "output_cost_per_token": 1.48e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.amazon.nova-pro-v1:0", - "name": "apac.amazon.nova-pro-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 8.4e-7, - "output_cost_per_token": 0.00000336, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.anthropic.claude-3-5-sonnet-20240620-v1:0", - "name": "apac.anthropic.claude-3-5-sonnet-20240620-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.anthropic.claude-3-5-sonnet-20241022-v2:0", - "name": "apac.anthropic.claude-3-5-sonnet-20241022-v2:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.anthropic.claude-3-haiku-20240307-v1:0", - "name": "apac.anthropic.claude-3-haiku-20240307-v1:0", - "provider": "bedrock", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.anthropic.claude-haiku-4-5-20251001-v1:0", - "name": "apac.anthropic.claude-haiku-4-5-20251001-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000055, - "cache_read_input_token_cost": 1.1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.anthropic.claude-3-sonnet-20240229-v1:0", - "name": "apac.anthropic.claude-3-sonnet-20240229-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "apac.anthropic.claude-sonnet-4-20250514-v1:0", - "name": "apac.anthropic.claude-sonnet-4-20250514-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "au.anthropic.claude-sonnet-4-5-20250929-v1:0", - "name": "au.anthropic.claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000033, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 3.3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/command-r-plus", - "name": "command-r-plus", - "provider": "azure", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-haiku-4-5", - "name": "claude-haiku-4-5", - "provider": "azure_ai", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-opus-4-5", - "name": "claude-opus-4-5", - "provider": "azure_ai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-opus-4-6", - "name": "claude-opus-4-6", - "provider": "azure_ai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-opus-4-7", - "name": "claude-opus-4-7", - "provider": "azure_ai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-fable-5", - "name": "claude-fable-5", - "provider": "azure_ai", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00005, - "cache_read_input_token_cost": 0.000001, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-opus-5", - "name": "claude-opus-5", - "provider": "azure_ai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-opus-4-8", - "name": "claude-opus-4-8", - "provider": "azure_ai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-opus-4-1", - "name": "claude-opus-4-1", - "provider": "azure_ai", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-sonnet-4-5", - "name": "claude-sonnet-4-5", - "provider": "azure_ai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-sonnet-5", - "name": "claude-sonnet-5", - "provider": "azure_ai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/claude-sonnet-4-6", - "name": "claude-sonnet-4-6", - "provider": "azure_ai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/computer-use-preview", - "name": "computer-use-preview", - "provider": "azure", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/gpt-oss-120b", - "name": "gpt-oss-120b", - "provider": "azure_ai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/gpt-5.5", - "name": "gpt-5.5", - "provider": "azure_ai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/gpt-5.5-2026-04-23", - "name": "gpt-5.5-2026-04-23", - "provider": "azure_ai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/gpt-5.4", - "name": "gpt-5.4", - "provider": "azure_ai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 2.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/gpt-5.4-2026-03-05", - "name": "gpt-5.4-2026-03-05", - "provider": "azure_ai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 2.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/gpt-5.4-mini", - "name": "gpt-5.4-mini", - "provider": "azure_ai", - "input_cost_per_token": 7.5e-7, - "output_cost_per_token": 0.0000045, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/gpt-5.4-mini-2026-03-17", - "name": "gpt-5.4-mini-2026-03-17", - "provider": "azure_ai", - "input_cost_per_token": 7.5e-7, - "output_cost_per_token": 0.0000045, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/gpt-5.4-nano", - "name": "gpt-5.4-nano", - "provider": "azure_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/gpt-5.4-nano-2026-03-17", - "name": "gpt-5.4-nano-2026-03-17", - "provider": "azure_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/model_router", - "name": "model_router", - "provider": "azure_ai", - "input_cost_per_token": 1.4e-7, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-4o-2024-08-06", - "name": "eu/gpt-4o-2024-08-06", - "provider": "azure", - "input_cost_per_token": 0.00000275, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 0.000001375, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-4o-2024-11-20", - "name": "eu/gpt-4o-2024-11-20", - "provider": "azure", - "input_cost_per_token": 0.00000275, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-4o-mini-2024-07-18", - "name": "eu/gpt-4o-mini-2024-07-18", - "provider": "azure", - "input_cost_per_token": 1.65e-7, - "output_cost_per_token": 6.6e-7, - "cache_read_input_token_cost": 8.3e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5-2025-08-07", - "name": "eu/gpt-5-2025-08-07", - "provider": "azure", - "input_cost_per_token": 0.000001375, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 1.375e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5-mini-2025-08-07", - "name": "eu/gpt-5-mini-2025-08-07", - "provider": "azure", - "input_cost_per_token": 2.75e-7, - "output_cost_per_token": 0.0000022, - "cache_read_input_token_cost": 2.75e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.1", - "name": "eu/gpt-5.1", - "provider": "azure", - "input_cost_per_token": 0.00000138, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 1.4e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.1-chat", - "name": "eu/gpt-5.1-chat", - "provider": "azure", - "input_cost_per_token": 0.00000138, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 1.4e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5-nano-2025-08-07", - "name": "eu/gpt-5-nano-2025-08-07", - "provider": "azure", - "input_cost_per_token": 5.5e-8, - "output_cost_per_token": 4.4e-7, - "cache_read_input_token_cost": 5.5e-9, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/o1-2024-12-17", - "name": "eu/o1-2024-12-17", - "provider": "azure", - "input_cost_per_token": 0.0000165, - "output_cost_per_token": 0.000066, - "cache_read_input_token_cost": 0.00000825, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/o1-mini-2024-09-12", - "name": "eu/o1-mini-2024-09-12", - "provider": "azure", - "input_cost_per_token": 0.00000121, - "output_cost_per_token": 0.00000484, - "cache_read_input_token_cost": 6.05e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/o1-preview-2024-09-12", - "name": "eu/o1-preview-2024-09-12", - "provider": "azure", - "input_cost_per_token": 0.0000165, - "output_cost_per_token": 0.000066, - "cache_read_input_token_cost": 0.00000825, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/o3-mini-2025-01-31", - "name": "eu/o3-mini-2025-01-31", - "provider": "azure", - "input_cost_per_token": 0.00000121, - "output_cost_per_token": 0.00000484, - "cache_read_input_token_cost": 6.05e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/global-standard/gpt-4o-2024-08-06", - "name": "global-standard/gpt-4o-2024-08-06", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/global-standard/gpt-4o-2024-11-20", - "name": "global-standard/gpt-4o-2024-11-20", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/global-standard/gpt-4o-mini", - "name": "global-standard/gpt-4o-mini", - "provider": "azure", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/global/gpt-4o-2024-08-06", - "name": "global/gpt-4o-2024-08-06", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/global/gpt-4o-2024-11-20", - "name": "global/gpt-4o-2024-11-20", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/global/gpt-5.1", - "name": "global/gpt-5.1", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/global/gpt-5.1-chat", - "name": "global/gpt-5.1-chat", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-3.5-turbo", - "name": "gpt-3.5-turbo", - "provider": "azure", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-3.5-turbo-0125", - "name": "gpt-3.5-turbo-0125", - "provider": "azure", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-35-turbo", - "name": "gpt-35-turbo", - "provider": "azure", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-35-turbo-0125", - "name": "gpt-35-turbo-0125", - "provider": "azure", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-35-turbo-1106", - "name": "gpt-35-turbo-1106", - "provider": "azure", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-35-turbo-16k", - "name": "gpt-35-turbo-16k", - "provider": "azure", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-35-turbo-16k-0613", - "name": "gpt-35-turbo-16k-0613", - "provider": "azure", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4", - "name": "gpt-4", - "provider": "azure", - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4-0125-preview", - "name": "gpt-4-0125-preview", - "provider": "azure", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4-0613", - "name": "gpt-4-0613", - "provider": "azure", - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4-1106-preview", - "name": "gpt-4-1106-preview", - "provider": "azure", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4-32k", - "name": "gpt-4-32k", - "provider": "azure", - "input_cost_per_token": 0.00006, - "output_cost_per_token": 0.00012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4-32k-0613", - "name": "gpt-4-32k-0613", - "provider": "azure", - "input_cost_per_token": 0.00006, - "output_cost_per_token": 0.00012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4-turbo", - "name": "gpt-4-turbo", - "provider": "azure", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4-turbo-2024-04-09", - "name": "gpt-4-turbo-2024-04-09", - "provider": "azure", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4-turbo-vision-preview", - "name": "gpt-4-turbo-vision-preview", - "provider": "azure", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4.1", - "name": "gpt-4.1", - "provider": "azure", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4.1-2025-04-14", - "name": "gpt-4.1-2025-04-14", - "provider": "azure", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4.1-mini", - "name": "gpt-4.1-mini", - "provider": "azure", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.0000016, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4.1-mini-2025-04-14", - "name": "gpt-4.1-mini-2025-04-14", - "provider": "azure", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.0000016, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4.1-nano", - "name": "gpt-4.1-nano", - "provider": "azure", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4.1-nano-2025-04-14", - "name": "gpt-4.1-nano-2025-04-14", - "provider": "azure", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4.5-preview", - "name": "gpt-4.5-preview", - "provider": "azure", - "input_cost_per_token": 0.000075, - "output_cost_per_token": 0.00015, - "cache_read_input_token_cost": 0.0000375, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4o", - "name": "gpt-4o", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4o-2024-05-13", - "name": "gpt-4o-2024-05-13", - "provider": "azure", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4o-2024-08-06", - "name": "gpt-4o-2024-08-06", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4o-2024-11-20", - "name": "gpt-4o-2024-11-20", - "provider": "azure", - "input_cost_per_token": 0.00000275, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-audio-2025-08-28", - "name": "gpt-audio-2025-08-28", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-audio-1.5-2026-02-23", - "name": "gpt-audio-1.5-2026-02-23", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-audio-mini-2025-10-06", - "name": "gpt-audio-mini-2025-10-06", - "provider": "azure", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4o-audio-preview-2024-12-17", - "name": "gpt-4o-audio-preview-2024-12-17", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4o-mini", - "name": "gpt-4o-mini", - "provider": "azure", - "input_cost_per_token": 1.65e-7, - "output_cost_per_token": 6.6e-7, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4o-mini-2024-07-18", - "name": "gpt-4o-mini-2024-07-18", - "provider": "azure", - "input_cost_per_token": 1.65e-7, - "output_cost_per_token": 6.6e-7, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-4o-mini-audio-preview-2024-12-17", - "name": "gpt-4o-mini-audio-preview-2024-12-17", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.1-2025-11-13", - "name": "gpt-5.1-2025-11-13", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.1-chat-2025-11-13", - "name": "gpt-5.1-chat-2025-11-13", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5", - "name": "gpt-5", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5-2025-08-07", - "name": "gpt-5-2025-08-07", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5-chat", - "name": "gpt-5-chat", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5-chat-latest", - "name": "gpt-5-chat-latest", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5-mini", - "name": "gpt-5-mini", - "provider": "azure", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5-mini-2025-08-07", - "name": "gpt-5-mini-2025-08-07", - "provider": "azure", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5-nano", - "name": "gpt-5-nano", - "provider": "azure", - "input_cost_per_token": 5e-8, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 5e-9, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5-nano-2025-08-07", - "name": "gpt-5-nano-2025-08-07", - "provider": "azure", - "input_cost_per_token": 5e-8, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 5e-9, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.1", - "name": "gpt-5.1", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.1-chat", - "name": "gpt-5.1-chat", - "provider": "azure", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.2", - "name": "gpt-5.2", - "provider": "azure", - "input_cost_per_token": 0.00000175, - "output_cost_per_token": 0.000014, - "cache_read_input_token_cost": 1.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.2-2025-12-11", - "name": "gpt-5.2-2025-12-11", - "provider": "azure", - "input_cost_per_token": 0.00000175, - "output_cost_per_token": 0.000014, - "cache_read_input_token_cost": 1.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.2-chat", - "name": "gpt-5.2-chat", - "provider": "azure", - "input_cost_per_token": 0.00000175, - "output_cost_per_token": 0.000014, - "cache_read_input_token_cost": 1.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.2-chat-2025-12-11", - "name": "gpt-5.2-chat-2025-12-11", - "provider": "azure", - "input_cost_per_token": 0.00000175, - "output_cost_per_token": 0.000014, - "cache_read_input_token_cost": 1.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.3-chat", - "name": "gpt-5.3-chat", - "provider": "azure", - "input_cost_per_token": 0.00000175, - "output_cost_per_token": 0.000014, - "cache_read_input_token_cost": 1.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.4", - "name": "gpt-5.4", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 2.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.4", - "name": "us/gpt-5.4", - "provider": "azure", - "input_cost_per_token": 0.00000275, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 2.8e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.4", - "name": "eu/gpt-5.4", - "provider": "azure", - "input_cost_per_token": 0.00000275, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 2.8e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.4-2026-03-05", - "name": "gpt-5.4-2026-03-05", - "provider": "azure", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 2.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.4-2026-03-05", - "name": "us/gpt-5.4-2026-03-05", - "provider": "azure", - "input_cost_per_token": 0.00000275, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 2.8e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.4-2026-03-05", - "name": "eu/gpt-5.4-2026-03-05", - "provider": "azure", - "input_cost_per_token": 0.00000275, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 2.8e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.6", - "name": "gpt-5.6", - "provider": "azure", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.6-sol", - "name": "gpt-5.6-sol", - "provider": "azure", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.6-terra", - "name": "gpt-5.6-terra", - "provider": "azure", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.6-luna", - "name": "gpt-5.6-luna", - "provider": "azure", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": 2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.6", - "name": "us/gpt-5.6", - "provider": "azure", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.000033, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.6-sol", - "name": "us/gpt-5.6-sol", - "provider": "azure", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.000033, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.6-terra", - "name": "us/gpt-5.6-terra", - "provider": "azure", - "input_cost_per_token": 0.0000022, - "output_cost_per_token": 0.0000132, - "cache_read_input_token_cost": 2.2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.6-luna", - "name": "us/gpt-5.6-luna", - "provider": "azure", - "input_cost_per_token": 2.2e-7, - "output_cost_per_token": 0.00000132, - "cache_read_input_token_cost": 2.2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.6", - "name": "eu/gpt-5.6", - "provider": "azure", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.000033, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.6-sol", - "name": "eu/gpt-5.6-sol", - "provider": "azure", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.000033, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.6-terra", - "name": "eu/gpt-5.6-terra", - "provider": "azure", - "input_cost_per_token": 0.0000022, - "output_cost_per_token": 0.0000132, - "cache_read_input_token_cost": 2.2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.6-luna", - "name": "eu/gpt-5.6-luna", - "provider": "azure", - "input_cost_per_token": 2.2e-7, - "output_cost_per_token": 0.00000132, - "cache_read_input_token_cost": 2.2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.5", - "name": "gpt-5.5", - "provider": "azure", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.5", - "name": "us/gpt-5.5", - "provider": "azure", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.000033, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.5", - "name": "eu/gpt-5.5", - "provider": "azure", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.000033, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.5-2026-04-23", - "name": "gpt-5.5-2026-04-23", - "provider": "azure", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.5-2026-04-23", - "name": "us/gpt-5.5-2026-04-23", - "provider": "azure", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.000033, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/eu/gpt-5.5-2026-04-23", - "name": "eu/gpt-5.5-2026-04-23", - "provider": "azure", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.000033, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.4-mini", - "name": "gpt-5.4-mini", - "provider": "azure", - "input_cost_per_token": 7.5e-7, - "output_cost_per_token": 0.0000045, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.4-mini-2026-03-17", - "name": "gpt-5.4-mini-2026-03-17", - "provider": "azure", - "input_cost_per_token": 7.5e-7, - "output_cost_per_token": 0.0000045, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.4-nano", - "name": "gpt-5.4-nano", - "provider": "azure", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-5.4-nano-2026-03-17", - "name": "gpt-5.4-nano-2026-03-17", - "provider": "azure", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/mistral-large-2402", - "name": "mistral-large-2402", - "provider": "azure", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/mistral-large-latest", - "name": "mistral-large-latest", - "provider": "azure", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o1", - "name": "o1", - "provider": "azure", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": 0.0000075, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o1-2024-12-17", - "name": "o1-2024-12-17", - "provider": "azure", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": 0.0000075, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o1-mini", - "name": "o1-mini", - "provider": "azure", - "input_cost_per_token": 0.00000121, - "output_cost_per_token": 0.00000484, - "cache_read_input_token_cost": 6.05e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o1-mini-2024-09-12", - "name": "o1-mini-2024-09-12", - "provider": "azure", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000044, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o1-preview", - "name": "o1-preview", - "provider": "azure", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": 0.0000075, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o1-preview-2024-09-12", - "name": "o1-preview-2024-09-12", - "provider": "azure", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": 0.0000075, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o3", - "name": "o3", - "provider": "azure", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o3-2025-04-16", - "name": "o3-2025-04-16", - "provider": "azure", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o3-mini", - "name": "o3-mini", - "provider": "azure", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000044, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o3-mini-2025-01-31", - "name": "o3-mini-2025-01-31", - "provider": "azure", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000044, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o4-mini", - "name": "o4-mini", - "provider": "azure", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000044, - "cache_read_input_token_cost": 2.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/o4-mini-2025-04-16", - "name": "o4-mini-2025-04-16", - "provider": "azure", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000044, - "cache_read_input_token_cost": 2.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-4.1-2025-04-14", - "name": "us/gpt-4.1-2025-04-14", - "provider": "azure", - "input_cost_per_token": 0.0000022, - "output_cost_per_token": 0.0000088, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-4.1-mini-2025-04-14", - "name": "us/gpt-4.1-mini-2025-04-14", - "provider": "azure", - "input_cost_per_token": 4.4e-7, - "output_cost_per_token": 0.00000176, - "cache_read_input_token_cost": 1.1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-4.1-nano-2025-04-14", - "name": "us/gpt-4.1-nano-2025-04-14", - "provider": "azure", - "input_cost_per_token": 1.1e-7, - "output_cost_per_token": 4.4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-4o-2024-08-06", - "name": "us/gpt-4o-2024-08-06", - "provider": "azure", - "input_cost_per_token": 0.00000275, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 0.000001375, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-4o-2024-11-20", - "name": "us/gpt-4o-2024-11-20", - "provider": "azure", - "input_cost_per_token": 0.00000275, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-4o-mini-2024-07-18", - "name": "us/gpt-4o-mini-2024-07-18", - "provider": "azure", - "input_cost_per_token": 1.65e-7, - "output_cost_per_token": 6.6e-7, - "cache_read_input_token_cost": 8.3e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5-2025-08-07", - "name": "us/gpt-5-2025-08-07", - "provider": "azure", - "input_cost_per_token": 0.000001375, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 1.375e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5-mini-2025-08-07", - "name": "us/gpt-5-mini-2025-08-07", - "provider": "azure", - "input_cost_per_token": 2.75e-7, - "output_cost_per_token": 0.0000022, - "cache_read_input_token_cost": 2.75e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5-nano-2025-08-07", - "name": "us/gpt-5-nano-2025-08-07", - "provider": "azure", - "input_cost_per_token": 5.5e-8, - "output_cost_per_token": 4.4e-7, - "cache_read_input_token_cost": 5.5e-9, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.1", - "name": "us/gpt-5.1", - "provider": "azure", - "input_cost_per_token": 0.00000138, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 1.4e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/gpt-5.1-chat", - "name": "us/gpt-5.1-chat", - "provider": "azure", - "input_cost_per_token": 0.00000138, - "output_cost_per_token": 0.000011, - "cache_read_input_token_cost": 1.4e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/o1-2024-12-17", - "name": "us/o1-2024-12-17", - "provider": "azure", - "input_cost_per_token": 0.0000165, - "output_cost_per_token": 0.000066, - "cache_read_input_token_cost": 0.00000825, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/o1-mini-2024-09-12", - "name": "us/o1-mini-2024-09-12", - "provider": "azure", - "input_cost_per_token": 0.00000121, - "output_cost_per_token": 0.00000484, - "cache_read_input_token_cost": 6.05e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/o1-preview-2024-09-12", - "name": "us/o1-preview-2024-09-12", - "provider": "azure", - "input_cost_per_token": 0.0000165, - "output_cost_per_token": 0.000066, - "cache_read_input_token_cost": 0.00000825, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/o3-2025-04-16", - "name": "us/o3-2025-04-16", - "provider": "azure", - "input_cost_per_token": 0.0000022, - "output_cost_per_token": 0.0000088, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/o3-mini-2025-01-31", - "name": "us/o3-mini-2025-01-31", - "provider": "azure", - "input_cost_per_token": 0.00000121, - "output_cost_per_token": 0.00000484, - "cache_read_input_token_cost": 6.05e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/us/o4-mini-2025-04-16", - "name": "us/o4-mini-2025-04-16", - "provider": "azure", - "input_cost_per_token": 0.00000121, - "output_cost_per_token": 0.00000484, - "cache_read_input_token_cost": 3.1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Llama-3.2-11B-Vision-Instruct", - "name": "Llama-3.2-11B-Vision-Instruct", - "provider": "azure_ai", - "input_cost_per_token": 3.7e-7, - "output_cost_per_token": 3.7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Llama-3.2-90B-Vision-Instruct", - "name": "Llama-3.2-90B-Vision-Instruct", - "provider": "azure_ai", - "input_cost_per_token": 0.00000204, - "output_cost_per_token": 0.00000204, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Llama-3.3-70B-Instruct", - "name": "Llama-3.3-70B-Instruct", - "provider": "azure_ai", - "input_cost_per_token": 7.1e-7, - "output_cost_per_token": 7.1e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8", - "name": "Llama-4-Maverick-17B-128E-Instruct-FP8", - "provider": "azure_ai", - "input_cost_per_token": 0.00000141, - "output_cost_per_token": 3.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Llama-4-Scout-17B-16E-Instruct", - "name": "Llama-4-Scout-17B-16E-Instruct", - "provider": "azure_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 7.8e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Meta-Llama-3-70B-Instruct", - "name": "Meta-Llama-3-70B-Instruct", - "provider": "azure_ai", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 3.7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Meta-Llama-3.1-405B-Instruct", - "name": "Meta-Llama-3.1-405B-Instruct", - "provider": "azure_ai", - "input_cost_per_token": 0.00000533, - "output_cost_per_token": 0.000016, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Meta-Llama-3.1-70B-Instruct", - "name": "Meta-Llama-3.1-70B-Instruct", - "provider": "azure_ai", - "input_cost_per_token": 0.00000268, - "output_cost_per_token": 0.00000354, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Meta-Llama-3.1-8B-Instruct", - "name": "Meta-Llama-3.1-8B-Instruct", - "provider": "azure_ai", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 6.1e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-3-medium-128k-instruct", - "name": "Phi-3-medium-128k-instruct", - "provider": "azure_ai", - "input_cost_per_token": 1.7e-7, - "output_cost_per_token": 6.8e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-3-medium-4k-instruct", - "name": "Phi-3-medium-4k-instruct", - "provider": "azure_ai", - "input_cost_per_token": 1.7e-7, - "output_cost_per_token": 6.8e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-3-mini-128k-instruct", - "name": "Phi-3-mini-128k-instruct", - "provider": "azure_ai", - "input_cost_per_token": 1.3e-7, - "output_cost_per_token": 5.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-3-mini-4k-instruct", - "name": "Phi-3-mini-4k-instruct", - "provider": "azure_ai", - "input_cost_per_token": 1.3e-7, - "output_cost_per_token": 5.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-3-small-128k-instruct", - "name": "Phi-3-small-128k-instruct", - "provider": "azure_ai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-3-small-8k-instruct", - "name": "Phi-3-small-8k-instruct", - "provider": "azure_ai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-3.5-MoE-instruct", - "name": "Phi-3.5-MoE-instruct", - "provider": "azure_ai", - "input_cost_per_token": 1.6e-7, - "output_cost_per_token": 6.4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-3.5-mini-instruct", - "name": "Phi-3.5-mini-instruct", - "provider": "azure_ai", - "input_cost_per_token": 1.3e-7, - "output_cost_per_token": 5.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-3.5-vision-instruct", - "name": "Phi-3.5-vision-instruct", - "provider": "azure_ai", - "input_cost_per_token": 1.3e-7, - "output_cost_per_token": 5.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-4", - "name": "Phi-4", - "provider": "azure_ai", - "input_cost_per_token": 1.25e-7, - "output_cost_per_token": 5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-4-mini-instruct", - "name": "Phi-4-mini-instruct", - "provider": "azure_ai", - "input_cost_per_token": 7.5e-8, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-4-multimodal-instruct", - "name": "Phi-4-multimodal-instruct", - "provider": "azure_ai", - "input_cost_per_token": 8e-8, - "output_cost_per_token": 3.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-4-mini-reasoning", - "name": "Phi-4-mini-reasoning", - "provider": "azure_ai", - "input_cost_per_token": 8e-8, - "output_cost_per_token": 3.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/Phi-4-reasoning", - "name": "Phi-4-reasoning", - "provider": "azure_ai", - "input_cost_per_token": 1.25e-7, - "output_cost_per_token": 5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/MAI-DS-R1", - "name": "MAI-DS-R1", - "provider": "azure_ai", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/deepseek-v3.2", - "name": "deepseek-v3.2", - "provider": "azure_ai", - "input_cost_per_token": 5.8e-7, - "output_cost_per_token": 0.00000168, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/deepseek-v3.2-speciale", - "name": "deepseek-v3.2-speciale", - "provider": "azure_ai", - "input_cost_per_token": 5.8e-7, - "output_cost_per_token": 0.00000168, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/deepseek-r1", - "name": "deepseek-r1", - "provider": "azure_ai", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/deepseek-v3", - "name": "deepseek-v3", - "provider": "azure_ai", - "input_cost_per_token": 0.00000114, - "output_cost_per_token": 0.00000456, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/deepseek-v3-0324", - "name": "deepseek-v3-0324", - "provider": "azure_ai", - "input_cost_per_token": 0.00000114, - "output_cost_per_token": 0.00000456, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/deepseek-v3.1", - "name": "deepseek-v3.1", - "provider": "azure_ai", - "input_cost_per_token": 0.00000123, - "output_cost_per_token": 0.00000494, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/deepseek-v4-pro", - "name": "deepseek-v4-pro", - "provider": "azure_ai", - "input_cost_per_token": 0.00000174, - "output_cost_per_token": 0.00000348, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/deepseek-v4-flash", - "name": "deepseek-v4-flash", - "provider": "azure_ai", - "input_cost_per_token": 1.9e-7, - "output_cost_per_token": 5.1e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/global/grok-3", - "name": "global/grok-3", - "provider": "azure_ai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/global/grok-3-mini", - "name": "global/grok-3-mini", - "provider": "azure_ai", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000127, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/grok-3", - "name": "grok-3", - "provider": "azure_ai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/grok-3-mini", - "name": "grok-3-mini", - "provider": "azure_ai", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000127, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/grok-4", - "name": "grok-4", - "provider": "azure_ai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/grok-4-fast-non-reasoning", - "name": "grok-4-fast-non-reasoning", - "provider": "azure_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/grok-4-fast-reasoning", - "name": "grok-4-fast-reasoning", - "provider": "azure_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/grok-4-1-fast-non-reasoning", - "name": "grok-4-1-fast-non-reasoning", - "provider": "azure_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/grok-4-1-fast-reasoning", - "name": "grok-4-1-fast-reasoning", - "provider": "azure_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/grok-code-fast-1", - "name": "grok-code-fast-1", - "provider": "azure_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/jais-30b-chat", - "name": "jais-30b-chat", - "provider": "azure_ai", - "input_cost_per_token": 0.0032, - "output_cost_per_token": 0.00971, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/jamba-instruct", - "name": "jamba-instruct", - "provider": "azure_ai", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/kimi-k2.5", - "name": "kimi-k2.5", - "provider": "azure_ai", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/kimi-k2.6", - "name": "kimi-k2.6", - "provider": "azure_ai", - "input_cost_per_token": 9.5e-7, - "output_cost_per_token": 0.000004, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/ministral-3b", - "name": "ministral-3b", - "provider": "azure_ai", - "input_cost_per_token": 4e-8, - "output_cost_per_token": 4e-8, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/mistral-large", - "name": "mistral-large", - "provider": "azure_ai", - "input_cost_per_token": 0.000004, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/mistral-large-2407", - "name": "mistral-large-2407", - "provider": "azure_ai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/mistral-large-latest", - "name": "mistral-large-latest", - "provider": "azure_ai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/mistral-large-3", - "name": "mistral-large-3", - "provider": "azure_ai", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/mistral-medium-2505", - "name": "mistral-medium-2505", - "provider": "azure_ai", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/mistral-nemo", - "name": "mistral-nemo", - "provider": "azure_ai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 1.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/mistral-small", - "name": "mistral-small", - "provider": "azure_ai", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure_ai/mistral-small-2503", - "name": "mistral-small-2503", - "provider": "azure_ai", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-northeast-1/anthropic.claude-instant-v1", - "name": "ap-northeast-1/anthropic.claude-instant-v1", - "provider": "bedrock", - "input_cost_per_token": 0.00000223, - "output_cost_per_token": 0.00000755, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-northeast-1/anthropic.claude-v1", - "name": "ap-northeast-1/anthropic.claude-v1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-northeast-1/anthropic.claude-v2:1", - "name": "ap-northeast-1/anthropic.claude-v2:1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-northeast-1/deepseek.v3.2", - "name": "ap-northeast-1/deepseek.v3.2", - "provider": "bedrock", - "input_cost_per_token": 7.4e-7, - "output_cost_per_token": 0.00000222, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-northeast-1/minimax.minimax-m2.1", - "name": "ap-northeast-1/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-northeast-1/minimax.minimax-m2.5", - "name": "ap-northeast-1/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking", - "name": "ap-northeast-1/moonshotai.kimi-k2-thinking", - "provider": "bedrock", - "input_cost_per_token": 7.3e-7, - "output_cost_per_token": 0.00000303, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-northeast-1/moonshotai.kimi-k2.5", - "name": "ap-northeast-1/moonshotai.kimi-k2.5", - "provider": "bedrock", - "input_cost_per_token": 7.2e-7, - "output_cost_per_token": 0.0000036, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-northeast-1/qwen.qwen3-coder-next", - "name": "ap-northeast-1/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/moonshotai.kimi-k2-thinking", - "name": "moonshotai.kimi-k2-thinking", - "provider": "bedrock", - "input_cost_per_token": 7.3e-7, - "output_cost_per_token": 0.00000303, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/moonshotai.kimi-k2.5", - "name": "moonshotai.kimi-k2.5", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.00000303, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0", - "name": "ap-south-1/meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000318, - "output_cost_per_token": 0.0000042, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0", - "name": "ap-south-1/meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 7.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-south-1/deepseek.v3.2", - "name": "ap-south-1/deepseek.v3.2", - "provider": "bedrock", - "input_cost_per_token": 7.4e-7, - "output_cost_per_token": 0.00000222, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-south-1/minimax.minimax-m2.1", - "name": "ap-south-1/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-south-1/minimax.minimax-m2.5", - "name": "ap-south-1/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-south-1/moonshotai.kimi-k2-thinking", - "name": "ap-south-1/moonshotai.kimi-k2-thinking", - "provider": "bedrock", - "input_cost_per_token": 7.1e-7, - "output_cost_per_token": 0.00000294, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-south-1/moonshotai.kimi-k2.5", - "name": "ap-south-1/moonshotai.kimi-k2.5", - "provider": "bedrock", - "input_cost_per_token": 7.2e-7, - "output_cost_per_token": 0.0000036, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-south-1/qwen.qwen3-coder-next", - "name": "ap-south-1/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-southeast-2/minimax.minimax-m2.5", - "name": "ap-southeast-2/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3.09e-7, - "output_cost_per_token": 0.000001236, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-southeast-3/deepseek.v3.2", - "name": "ap-southeast-3/deepseek.v3.2", - "provider": "bedrock", - "input_cost_per_token": 7.4e-7, - "output_cost_per_token": 0.00000222, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-southeast-3/minimax.minimax-m2.1", - "name": "ap-southeast-3/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-southeast-3/minimax.minimax-m2.5", - "name": "ap-southeast-3/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-southeast-3/moonshotai.kimi-k2.5", - "name": "ap-southeast-3/moonshotai.kimi-k2.5", - "provider": "bedrock", - "input_cost_per_token": 7.2e-7, - "output_cost_per_token": 0.0000036, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ap-southeast-3/qwen.qwen3-coder-next", - "name": "ap-southeast-3/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0", - "name": "ca-central-1/meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000305, - "output_cost_per_token": 0.00000403, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0", - "name": "ca-central-1/meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3.5e-7, - "output_cost_per_token": 6.9e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-north-1/deepseek.v3.2", - "name": "eu-north-1/deepseek.v3.2", - "provider": "bedrock", - "input_cost_per_token": 7.4e-7, - "output_cost_per_token": 0.00000222, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-north-1/minimax.minimax-m2.1", - "name": "eu-north-1/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-north-1/minimax.minimax-m2.5", - "name": "eu-north-1/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-north-1/moonshotai.kimi-k2.5", - "name": "eu-north-1/moonshotai.kimi-k2.5", - "provider": "bedrock", - "input_cost_per_token": 7.2e-7, - "output_cost_per_token": 0.0000036, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-central-1/anthropic.claude-instant-v1", - "name": "eu-central-1/anthropic.claude-instant-v1", - "provider": "bedrock", - "input_cost_per_token": 0.00000248, - "output_cost_per_token": 0.00000838, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-central-1/anthropic.claude-v1", - "name": "eu-central-1/anthropic.claude-v1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-central-1/anthropic.claude-v2:1", - "name": "eu-central-1/anthropic.claude-v2:1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-central-1/minimax.minimax-m2.1", - "name": "eu-central-1/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-central-1/minimax.minimax-m2.5", - "name": "eu-central-1/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-central-1/qwen.qwen3-coder-next", - "name": "eu-central-1/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0", - "name": "eu-west-1/meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000286, - "output_cost_per_token": 0.00000378, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0", - "name": "eu-west-1/meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3.2e-7, - "output_cost_per_token": 6.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-1/minimax.minimax-m2.1", - "name": "eu-west-1/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-1/minimax.minimax-m2.5", - "name": "eu-west-1/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-1/qwen.qwen3-coder-next", - "name": "eu-west-1/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0", - "name": "eu-west-2/meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000345, - "output_cost_per_token": 0.00000455, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0", - "name": "eu-west-2/meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3.9e-7, - "output_cost_per_token": 7.8e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-2/minimax.minimax-m2.1", - "name": "eu-west-2/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 4.7e-7, - "output_cost_per_token": 0.00000186, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-2/minimax.minimax-m2.5", - "name": "eu-west-2/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 4.7e-7, - "output_cost_per_token": 0.00000186, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-2/qwen.qwen3-coder-next", - "name": "eu-west-2/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 7.8e-7, - "output_cost_per_token": 0.00000186, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2", - "name": "eu-west-3/mistral.mistral-7b-instruct-v0:2", - "provider": "bedrock", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 2.6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-3/mistral.mistral-large-2402-v1:0", - "name": "eu-west-3/mistral.mistral-large-2402-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000104, - "output_cost_per_token": 0.0000312, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1", - "name": "eu-west-3/mistral.mixtral-8x7b-instruct-v0:1", - "provider": "bedrock", - "input_cost_per_token": 5.9e-7, - "output_cost_per_token": 9.1e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-south-1/minimax.minimax-m2.1", - "name": "eu-south-1/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-south-1/minimax.minimax-m2.5", - "name": "eu-south-1/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/eu-south-1/qwen.qwen3-coder-next", - "name": "eu-south-1/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", - "name": "invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/sa-east-1/meta.llama3-70b-instruct-v1:0", - "name": "sa-east-1/meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000445, - "output_cost_per_token": 0.00000588, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0", - "name": "sa-east-1/meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.00000101, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/sa-east-1/deepseek.v3.2", - "name": "sa-east-1/deepseek.v3.2", - "provider": "bedrock", - "input_cost_per_token": 7.4e-7, - "output_cost_per_token": 0.00000222, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/sa-east-1/minimax.minimax-m2.1", - "name": "sa-east-1/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/sa-east-1/minimax.minimax-m2.5", - "name": "sa-east-1/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3.6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/sa-east-1/moonshotai.kimi-k2-thinking", - "name": "sa-east-1/moonshotai.kimi-k2-thinking", - "provider": "bedrock", - "input_cost_per_token": 7.3e-7, - "output_cost_per_token": 0.00000303, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/sa-east-1/moonshotai.kimi-k2.5", - "name": "sa-east-1/moonshotai.kimi-k2.5", - "provider": "bedrock", - "input_cost_per_token": 7.2e-7, - "output_cost_per_token": 0.0000036, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/sa-east-1/qwen.qwen3-coder-next", - "name": "sa-east-1/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.00000144, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/anthropic.claude-instant-v1", - "name": "us-east-1/anthropic.claude-instant-v1", - "provider": "bedrock", - "input_cost_per_token": 8e-7, - "output_cost_per_token": 0.0000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/anthropic.claude-v1", - "name": "us-east-1/anthropic.claude-v1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/anthropic.claude-v2:1", - "name": "us-east-1/anthropic.claude-v2:1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/meta.llama3-70b-instruct-v1:0", - "name": "us-east-1/meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000265, - "output_cost_per_token": 0.0000035, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/meta.llama3-8b-instruct-v1:0", - "name": "us-east-1/meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/mistral.mistral-7b-instruct-v0:2", - "name": "us-east-1/mistral.mistral-7b-instruct-v0:2", - "provider": "bedrock", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/mistral.mistral-large-2402-v1:0", - "name": "us-east-1/mistral.mistral-large-2402-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1", - "name": "us-east-1/mistral.mixtral-8x7b-instruct-v0:1", - "provider": "bedrock", - "input_cost_per_token": 4.5e-7, - "output_cost_per_token": 7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/deepseek.v3.2", - "name": "us-east-1/deepseek.v3.2", - "provider": "bedrock", - "input_cost_per_token": 6.2e-7, - "output_cost_per_token": 0.00000185, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/minimax.minimax-m2.1", - "name": "us-east-1/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/minimax.minimax-m2.5", - "name": "us-east-1/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/moonshotai.kimi-k2-thinking", - "name": "us-east-1/moonshotai.kimi-k2-thinking", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/moonshotai.kimi-k2.5", - "name": "us-east-1/moonshotai.kimi-k2.5", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/qwen.qwen3-coder-next", - "name": "us-east-1/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-2/deepseek.v3.2", - "name": "us-east-2/deepseek.v3.2", - "provider": "bedrock", - "input_cost_per_token": 6.2e-7, - "output_cost_per_token": 0.00000185, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-2/minimax.minimax-m2.1", - "name": "us-east-2/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-2/minimax.minimax-m2.5", - "name": "us-east-2/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-2/moonshotai.kimi-k2-thinking", - "name": "us-east-2/moonshotai.kimi-k2-thinking", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-2/moonshotai.kimi-k2.5", - "name": "us-east-2/moonshotai.kimi-k2.5", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-2/qwen.qwen3-coder-next", - "name": "us-east-2/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/amazon.nova-pro-v1:0", - "name": "us-gov-east-1/amazon.nova-pro-v1:0", - "provider": "bedrock", - "input_cost_per_token": 9.6e-7, - "output_cost_per_token": 0.00000384, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/amazon.titan-text-express-v1", - "name": "us-gov-east-1/amazon.titan-text-express-v1", - "provider": "bedrock", - "input_cost_per_token": 0.0000013, - "output_cost_per_token": 0.0000017, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/amazon.titan-text-lite-v1", - "name": "us-gov-east-1/amazon.titan-text-lite-v1", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/amazon.titan-text-premier-v1:0", - "name": "us-gov-east-1/amazon.titan-text-premier-v1:0", - "provider": "bedrock", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0", - "name": "us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000036, - "output_cost_per_token": 0.000018, - "cache_read_input_token_cost": 3.6e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0", - "name": "us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0", - "name": "us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000036, - "output_cost_per_token": 0.000018, - "cache_read_input_token_cost": 3.6e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0", - "name": "us-gov-east-1/claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000036, - "output_cost_per_token": 0.000018, - "cache_read_input_token_cost": 3.6e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0", - "name": "us-gov-east-1/meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000265, - "output_cost_per_token": 0.0000035, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0", - "name": "us-gov-east-1/meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.00000265, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/amazon.nova-pro-v1:0", - "name": "us-gov-west-1/amazon.nova-pro-v1:0", - "provider": "bedrock", - "input_cost_per_token": 9.6e-7, - "output_cost_per_token": 0.00000384, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/amazon.titan-text-express-v1", - "name": "us-gov-west-1/amazon.titan-text-express-v1", - "provider": "bedrock", - "input_cost_per_token": 0.0000013, - "output_cost_per_token": 0.0000017, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/amazon.titan-text-lite-v1", - "name": "us-gov-west-1/amazon.titan-text-lite-v1", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/amazon.titan-text-premier-v1:0", - "name": "us-gov-west-1/amazon.titan-text-premier-v1:0", - "provider": "bedrock", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0", - "name": "us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000036, - "output_cost_per_token": 0.000018, - "cache_read_input_token_cost": 3.6e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0", - "name": "us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000036, - "output_cost_per_token": 0.000018, - "cache_read_input_token_cost": 3.6e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0", - "name": "us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0", - "name": "us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000036, - "output_cost_per_token": 0.000018, - "cache_read_input_token_cost": 3.6e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0", - "name": "us-gov-west-1/claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000036, - "output_cost_per_token": 0.000018, - "cache_read_input_token_cost": 3.6e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0", - "name": "us-gov-west-1/meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000265, - "output_cost_per_token": 0.0000035, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0", - "name": "us-gov-west-1/meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.00000265, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0", - "name": "us-west-1/meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000265, - "output_cost_per_token": 0.0000035, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-1/meta.llama3-8b-instruct-v1:0", - "name": "us-west-1/meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/anthropic.claude-instant-v1", - "name": "us-west-2/anthropic.claude-instant-v1", - "provider": "bedrock", - "input_cost_per_token": 8e-7, - "output_cost_per_token": 0.0000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/anthropic.claude-v1", - "name": "us-west-2/anthropic.claude-v1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/anthropic.claude-v2:1", - "name": "us-west-2/anthropic.claude-v2:1", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/mistral.mistral-7b-instruct-v0:2", - "name": "us-west-2/mistral.mistral-7b-instruct-v0:2", - "provider": "bedrock", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/mistral.mistral-large-2402-v1:0", - "name": "us-west-2/mistral.mistral-large-2402-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1", - "name": "us-west-2/mistral.mixtral-8x7b-instruct-v0:1", - "provider": "bedrock", - "input_cost_per_token": 4.5e-7, - "output_cost_per_token": 7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/deepseek.v3.2", - "name": "us-west-2/deepseek.v3.2", - "provider": "bedrock", - "input_cost_per_token": 6.2e-7, - "output_cost_per_token": 0.00000185, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/minimax.minimax-m2.1", - "name": "us-west-2/minimax.minimax-m2.1", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/minimax.minimax-m2.5", - "name": "us-west-2/minimax.minimax-m2.5", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/moonshotai.kimi-k2-thinking", - "name": "us-west-2/moonshotai.kimi-k2-thinking", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/moonshotai.kimi-k2.5", - "name": "us-west-2/moonshotai.kimi-k2.5", - "provider": "bedrock", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/qwen.qwen3-coder-next", - "name": "us-west-2/qwen.qwen3-coder-next", - "provider": "bedrock", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0", - "name": "us.anthropic.claude-3-5-haiku-20241022-v1:0", - "provider": "bedrock", - "input_cost_per_token": 8e-7, - "output_cost_per_token": 0.000004, - "cache_read_input_token_cost": 8e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "chatgpt-4o-latest", - "name": "chatgpt-4o-latest", - "provider": "openai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-haiku-4-5-20251001", - "name": "claude-haiku-4-5-20251001", - "provider": "anthropic", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-haiku-4-5", - "name": "claude-haiku-4-5", - "provider": "anthropic", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-3-7-sonnet-20250219", - "name": "claude-3-7-sonnet-20250219", - "provider": "anthropic", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-3-haiku-20240307", - "name": "claude-3-haiku-20240307", - "provider": "anthropic", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-3-opus-20240229", - "name": "claude-3-opus-20240229", - "provider": "anthropic", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-4-opus-20250514", - "name": "claude-4-opus-20250514", - "provider": "anthropic", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-4-sonnet-20250514", - "name": "claude-4-sonnet-20250514", - "provider": "anthropic", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-sonnet-4-5", - "name": "claude-sonnet-4-5", - "provider": "anthropic", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-sonnet-4-5-20250929", - "name": "claude-sonnet-4-5-20250929", - "provider": "anthropic", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-sonnet-5", - "name": "claude-sonnet-5", - "provider": "anthropic", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-sonnet-4-6", - "name": "claude-sonnet-4-6", - "provider": "anthropic", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-sonnet-4-5-20250929-v1:0", - "name": "claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-1", - "name": "claude-opus-4-1", - "provider": "anthropic", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-1-20250805", - "name": "claude-opus-4-1-20250805", - "provider": "anthropic", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-20250514", - "name": "claude-opus-4-20250514", - "provider": "anthropic", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-5-20251101", - "name": "claude-opus-4-5-20251101", - "provider": "anthropic", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-5", - "name": "claude-opus-4-5", - "provider": "anthropic", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-6", - "name": "claude-opus-4-6", - "provider": "anthropic", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-6-20260205", - "name": "claude-opus-4-6-20260205", - "provider": "anthropic", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-7", - "name": "claude-opus-4-7", - "provider": "anthropic", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-7-20260416", - "name": "claude-opus-4-7-20260416", - "provider": "anthropic", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-fable-5", - "name": "claude-fable-5", - "provider": "anthropic", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00005, - "cache_read_input_token_cost": 0.000001, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-5", - "name": "claude-opus-5", - "provider": "anthropic", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-opus-4-8", - "name": "claude-opus-4-8", - "provider": "anthropic", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "claude-sonnet-4-20250514", - "name": "claude-sonnet-4-20250514", - "provider": "anthropic", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "cohere.command-light-text-v14", - "name": "cohere.command-light-text-v14", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "cohere.command-r-plus-v1:0", - "name": "cohere.command-r-plus-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "cohere.command-r-v1:0", - "name": "cohere.command-r-v1:0", - "provider": "bedrock", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "cohere.command-text-v14", - "name": "cohere.command-text-v14", - "provider": "bedrock", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "computer-use-preview", - "name": "computer-use-preview", - "provider": "azure", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "deepseek.v3-v1:0", - "name": "deepseek.v3-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 5.8e-7, - "output_cost_per_token": 0.00000168, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "deepseek.v3.2", - "name": "deepseek.v3.2", - "provider": "bedrock_converse", - "input_cost_per_token": 6.2e-7, - "output_cost_per_token": 0.00000185, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.amazon.nova-lite-v1:0", - "name": "eu.amazon.nova-lite-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 7.8e-8, - "output_cost_per_token": 3.12e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.amazon.nova-micro-v1:0", - "name": "eu.amazon.nova-micro-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 4.6e-8, - "output_cost_per_token": 1.84e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.amazon.nova-pro-v1:0", - "name": "eu.amazon.nova-pro-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.00000105, - "output_cost_per_token": 0.0000042, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-3-5-haiku-20241022-v1:0", - "name": "eu.anthropic.claude-3-5-haiku-20241022-v1:0", - "provider": "bedrock", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-haiku-4-5-20251001-v1:0", - "name": "eu.anthropic.claude-haiku-4-5-20251001-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000055, - "cache_read_input_token_cost": 1.1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-3-5-sonnet-20240620-v1:0", - "name": "eu.anthropic.claude-3-5-sonnet-20240620-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-3-5-sonnet-20241022-v2:0", - "name": "eu.anthropic.claude-3-5-sonnet-20241022-v2:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-3-7-sonnet-20250219-v1:0", - "name": "eu.anthropic.claude-3-7-sonnet-20250219-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-3-haiku-20240307-v1:0", - "name": "eu.anthropic.claude-3-haiku-20240307-v1:0", - "provider": "bedrock", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-3-opus-20240229-v1:0", - "name": "eu.anthropic.claude-3-opus-20240229-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-3-sonnet-20240229-v1:0", - "name": "eu.anthropic.claude-3-sonnet-20240229-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-opus-4-1-20250805-v1:0", - "name": "eu.anthropic.claude-opus-4-1-20250805-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-opus-4-20250514-v1:0", - "name": "eu.anthropic.claude-opus-4-20250514-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-sonnet-4-20250514-v1:0", - "name": "eu.anthropic.claude-sonnet-4-20250514-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", - "name": "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000033, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 3.3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.meta.llama3-2-1b-instruct-v1:0", - "name": "eu.meta.llama3-2-1b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 1.3e-7, - "output_cost_per_token": 1.3e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.meta.llama3-2-3b-instruct-v1:0", - "name": "eu.meta.llama3-2-3b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 1.9e-7, - "output_cost_per_token": 1.9e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.mistral.pixtral-large-2502-v1:0", - "name": "eu.mistral.pixtral-large-2502-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-3.5-turbo", - "name": "ft:gpt-3.5-turbo", - "provider": "openai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-3.5-turbo-0125", - "name": "ft:gpt-3.5-turbo-0125", - "provider": "openai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-3.5-turbo-0613", - "name": "ft:gpt-3.5-turbo-0613", - "provider": "openai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-3.5-turbo-1106", - "name": "ft:gpt-3.5-turbo-1106", - "provider": "openai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-4-0613", - "name": "ft:gpt-4-0613", - "provider": "openai", - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-4o-2024-08-06", - "name": "ft:gpt-4o-2024-08-06", - "provider": "openai", - "input_cost_per_token": 0.00000375, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.000001875, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-4o-2024-11-20", - "name": "ft:gpt-4o-2024-11-20", - "provider": "openai", - "input_cost_per_token": 0.00000375, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-4o-mini-2024-07-18", - "name": "ft:gpt-4o-mini-2024-07-18", - "provider": "openai", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": 1.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-4.1-2025-04-14", - "name": "ft:gpt-4.1-2025-04-14", - "provider": "openai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 7.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-4.1-mini-2025-04-14", - "name": "ft:gpt-4.1-mini-2025-04-14", - "provider": "openai", - "input_cost_per_token": 8e-7, - "output_cost_per_token": 0.0000032, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:gpt-4.1-nano-2025-04-14", - "name": "ft:gpt-4.1-nano-2025-04-14", - "provider": "openai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 8e-7, - "cache_read_input_token_cost": 5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "ft:o4-mini-2025-04-16", - "name": "ft:o4-mini-2025-04-16", - "provider": "openai", - "input_cost_per_token": 0.000004, - "output_cost_per_token": 0.000016, - "cache_read_input_token_cost": 0.000001, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-2.0-flash", - "name": "gemini-2.0-flash", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-2.0-flash-001", - "name": "gemini-2.0-flash-001", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": 3.75e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-2.0-flash-lite", - "name": "gemini-2.0-flash-lite", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 7.5e-8, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": 1.875e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-2.0-flash-lite-001", - "name": "gemini-2.0-flash-lite-001", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 7.5e-8, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": 1.875e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-2.5-flash", - "name": "gemini-2.5-flash", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini-3.1-flash-lite-preview", - "name": "gemini-3.1-flash-lite-preview", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": 0.0000015 - }, - { - "id": "gemini-3.1-flash-lite", - "name": "gemini-3.1-flash-lite", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": 0.0000015 - }, - { - "id": "gemini-3.5-flash-lite", - "name": "gemini-3.5-flash-lite", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini-2.5-flash-lite", - "name": "gemini-2.5-flash-lite", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 1e-8, - "output_cost_per_reasoning_token": 4e-7 - }, - { - "id": "gemini-2.5-flash-lite-preview-09-2025", - "name": "gemini-2.5-flash-lite-preview-09-2025", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 1e-8, - "output_cost_per_reasoning_token": 4e-7 - }, - { - "id": "gemini-2.5-flash-preview-09-2025", - "name": "gemini-2.5-flash-preview-09-2025", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini-2.5-flash-lite-preview-06-17", - "name": "gemini-2.5-flash-lite-preview-06-17", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": 4e-7 - }, - { - "id": "gemini-2.5-pro", - "name": "gemini-2.5-pro", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-3-pro-preview", - "name": "gemini-3-pro-preview", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-3.1-pro-preview", - "name": "gemini-3.1-pro-preview", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-3.1-pro-preview-customtools", - "name": "gemini-3.1-pro-preview-customtools", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/gemini-3-pro-preview", - "name": "gemini-3-pro-preview", - "provider": "vertex_ai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/gemini-3-flash-preview", - "name": "gemini-3-flash-preview", - "provider": "vertex_ai", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": 5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/gemini-3.5-flash", - "name": "gemini-3.5-flash", - "provider": "vertex_ai", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000009, - "cache_read_input_token_cost": 1.5e-7, - "output_cost_per_reasoning_token": 0.000009 - }, - { - "id": "vertex_ai/gemini-3.6-flash", - "name": "gemini-3.6-flash", - "provider": "vertex_ai", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.0000075, - "cache_read_input_token_cost": 1.5e-7, - "output_cost_per_reasoning_token": 0.0000075 - }, - { - "id": "vertex_ai/gemini-3.1-pro-preview", - "name": "gemini-3.1-pro-preview", - "provider": "vertex_ai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/gemini-3.1-pro-preview-customtools", - "name": "gemini-3.1-pro-preview-customtools", - "provider": "vertex_ai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-2.5-pro-preview-tts", - "name": "gemini-2.5-pro-preview-tts", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-robotics-er-1.5-preview", - "name": "gemini-robotics-er-1.5-preview", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 0, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini/gemini-robotics-er-1.5-preview", - "name": "gemini-robotics-er-1.5-preview", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 0, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini/gemini-robotics-er-2-preview", - "name": "gemini-robotics-er-2-preview", - "provider": "gemini", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": 0.00001 - }, - { - "id": "gemini/gemini-robotics-er-1.6-preview", - "name": "gemini-robotics-er-1.6-preview", - "provider": "gemini", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": 0.000005 - }, - { - "id": "gemini-2.5-computer-use-preview-10-2025", - "name": "gemini-2.5-computer-use-preview-10-2025", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-2.0-flash", - "name": "gemini-2.0-flash", - "provider": "gemini", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-2.0-flash-001", - "name": "gemini-2.0-flash-001", - "provider": "gemini", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-2.0-flash-lite", - "name": "gemini-2.0-flash-lite", - "provider": "gemini", - "input_cost_per_token": 7.5e-8, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": 1.875e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-2.5-flash", - "name": "gemini-2.5-flash", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini/gemini-2.5-flash-lite", - "name": "gemini-2.5-flash-lite", - "provider": "gemini", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 1e-8, - "output_cost_per_reasoning_token": 4e-7 - }, - { - "id": "gemini/gemini-2.5-flash-lite-preview-09-2025", - "name": "gemini-2.5-flash-lite-preview-09-2025", - "provider": "gemini", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 1e-8, - "output_cost_per_reasoning_token": 4e-7 - }, - { - "id": "gemini/gemini-2.5-flash-preview-09-2025", - "name": "gemini-2.5-flash-preview-09-2025", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini/gemini-flash-latest", - "name": "gemini-flash-latest", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini/gemini-flash-lite-latest", - "name": "gemini-flash-lite-latest", - "provider": "gemini", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": 4e-7 - }, - { - "id": "gemini/gemini-2.5-flash-lite-preview-06-17", - "name": "gemini-2.5-flash-lite-preview-06-17", - "provider": "gemini", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": 4e-7 - }, - { - "id": "gemini/gemini-2.5-pro", - "name": "gemini-2.5-pro", - "provider": "gemini", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-2.5-computer-use-preview-10-2025", - "name": "gemini-2.5-computer-use-preview-10-2025", - "provider": "gemini", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-3-pro-preview", - "name": "gemini-3-pro-preview", - "provider": "gemini", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-3.1-flash-lite-preview", - "name": "gemini-3.1-flash-lite-preview", - "provider": "gemini", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": 0.0000015 - }, - { - "id": "gemini/gemini-3.1-flash-lite", - "name": "gemini-3.1-flash-lite", - "provider": "gemini", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": 0.0000015 - }, - { - "id": "gemini/gemini-3.5-flash-lite", - "name": "gemini-3.5-flash-lite", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini/gemini-3-flash-preview", - "name": "gemini-3-flash-preview", - "provider": "gemini", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": 5e-8, - "output_cost_per_reasoning_token": 0.000003 - }, - { - "id": "gemini/gemini-3.5-flash", - "name": "gemini-3.5-flash", - "provider": "gemini", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000009, - "cache_read_input_token_cost": 1.5e-7, - "output_cost_per_reasoning_token": 0.000009 - }, - { - "id": "gemini/gemini-3.6-flash", - "name": "gemini-3.6-flash", - "provider": "gemini", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.0000075, - "cache_read_input_token_cost": 1.5e-7, - "output_cost_per_reasoning_token": 0.0000075 - }, - { - "id": "gemini/gemini-omni-flash-preview", - "name": "gemini-omni-flash-preview", - "provider": "gemini", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000009, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": 0.000009 - }, - { - "id": "gemini/gemini-3.1-pro-preview", - "name": "gemini-3.1-pro-preview", - "provider": "gemini", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-3.1-pro-preview-customtools", - "name": "gemini-3.1-pro-preview-customtools", - "provider": "gemini", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-3-flash-preview", - "name": "gemini-3-flash-preview", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": 5e-8, - "output_cost_per_reasoning_token": 0.000003 - }, - { - "id": "gemini-omni-flash-preview", - "name": "gemini-omni-flash-preview", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000009, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": 0.000009 - }, - { - "id": "gemini-3.5-flash", - "name": "gemini-3.5-flash", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000009, - "cache_read_input_token_cost": 1.5e-7, - "output_cost_per_reasoning_token": 0.000009 - }, - { - "id": "gemini-3.6-flash", - "name": "gemini-3.6-flash", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.0000075, - "cache_read_input_token_cost": 1.5e-7, - "output_cost_per_reasoning_token": 0.0000075 - }, - { - "id": "gemini/gemini-2.5-pro-preview-tts", - "name": "gemini-2.5-pro-preview-tts", - "provider": "gemini", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-exp-1114", - "name": "gemini-exp-1114", - "provider": "gemini", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-exp-1206", - "name": "gemini-exp-1206", - "provider": "gemini", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-gemma-2-27b-it", - "name": "gemini-gemma-2-27b-it", - "provider": "gemini", - "input_cost_per_token": 3.5e-7, - "output_cost_per_token": 0.00000105, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-gemma-2-9b-it", - "name": "gemini-gemma-2-9b-it", - "provider": "gemini", - "input_cost_per_token": 3.5e-7, - "output_cost_per_token": 0.00000105, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemma-3-27b-it", - "name": "gemma-3-27b-it", - "provider": "gemini", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/learnlm-1.5-pro-experimental", - "name": "learnlm-1.5-pro-experimental", - "provider": "gemini", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/lyria-3-clip-preview", - "name": "lyria-3-clip-preview", - "provider": "gemini", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/lyria-3-pro-preview", - "name": "lyria-3-pro-preview", - "provider": "gemini", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "google.gemma-3-12b-it", - "name": "google.gemma-3-12b-it", - "provider": "bedrock_converse", - "input_cost_per_token": 9e-8, - "output_cost_per_token": 2.9e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "google.gemma-3-27b-it", - "name": "google.gemma-3-27b-it", - "provider": "bedrock_converse", - "input_cost_per_token": 2.3e-7, - "output_cost_per_token": 3.8e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "google.gemma-3-4b-it", - "name": "google.gemma-3-4b-it", - "provider": "bedrock_converse", - "input_cost_per_token": 4e-8, - "output_cost_per_token": 8e-8, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", - "name": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-sonnet-4-20250514-v1:0", - "name": "global.anthropic.claude-sonnet-4-20250514-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-haiku-4-5-20251001-v1:0", - "name": "global.anthropic.claude-haiku-4-5-20251001-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.amazon.nova-2-lite-v1:0", - "name": "global.amazon.nova-2-lite-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-3.5-turbo", - "name": "gpt-3.5-turbo", - "provider": "openai", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-3.5-turbo-0125", - "name": "gpt-3.5-turbo-0125", - "provider": "openai", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-3.5-turbo-1106", - "name": "gpt-3.5-turbo-1106", - "provider": "openai", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-3.5-turbo-16k", - "name": "gpt-3.5-turbo-16k", - "provider": "openai", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4", - "name": "gpt-4", - "provider": "openai", - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4-0125-preview", - "name": "gpt-4-0125-preview", - "provider": "openai", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4-0314", - "name": "gpt-4-0314", - "provider": "openai", - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4-0613", - "name": "gpt-4-0613", - "provider": "openai", - "input_cost_per_token": 0.00003, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4-1106-preview", - "name": "gpt-4-1106-preview", - "provider": "openai", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4-turbo", - "name": "gpt-4-turbo", - "provider": "openai", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4-turbo-2024-04-09", - "name": "gpt-4-turbo-2024-04-09", - "provider": "openai", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4-turbo-preview", - "name": "gpt-4-turbo-preview", - "provider": "openai", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4.1", - "name": "gpt-4.1", - "provider": "openai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4.1-2025-04-14", - "name": "gpt-4.1-2025-04-14", - "provider": "openai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4.1-mini", - "name": "gpt-4.1-mini", - "provider": "openai", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.0000016, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4.1-mini-2025-04-14", - "name": "gpt-4.1-mini-2025-04-14", - "provider": "openai", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.0000016, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4.1-nano", - "name": "gpt-4.1-nano", - "provider": "openai", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4.1-nano-2025-04-14", - "name": "gpt-4.1-nano-2025-04-14", - "provider": "openai", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o", - "name": "gpt-4o", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-2024-05-13", - "name": "gpt-4o-2024-05-13", - "provider": "openai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-2024-08-06", - "name": "gpt-4o-2024-08-06", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-2024-11-20", - "name": "gpt-4o-2024-11-20", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-audio-preview", - "name": "gpt-4o-audio-preview", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-audio-preview-2024-12-17", - "name": "gpt-4o-audio-preview-2024-12-17", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-audio-preview-2025-06-03", - "name": "gpt-4o-audio-preview-2025-06-03", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-audio", - "name": "gpt-audio", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-audio-1.5", - "name": "gpt-audio-1.5", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-audio-2025-08-28", - "name": "gpt-audio-2025-08-28", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-audio-mini", - "name": "gpt-audio-mini", - "provider": "openai", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-audio-mini-2025-10-06", - "name": "gpt-audio-mini-2025-10-06", - "provider": "openai", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-audio-mini-2025-12-15", - "name": "gpt-audio-mini-2025-12-15", - "provider": "openai", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-mini", - "name": "gpt-4o-mini", - "provider": "openai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-mini-2024-07-18", - "name": "gpt-4o-mini-2024-07-18", - "provider": "openai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-mini-audio-preview", - "name": "gpt-4o-mini-audio-preview", - "provider": "openai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-mini-audio-preview-2024-12-17", - "name": "gpt-4o-mini-audio-preview-2024-12-17", - "provider": "openai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-mini-search-preview", - "name": "gpt-4o-mini-search-preview", - "provider": "openai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-mini-search-preview-2025-03-11", - "name": "gpt-4o-mini-search-preview-2025-03-11", - "provider": "openai", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-search-preview", - "name": "gpt-4o-search-preview", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-4o-search-preview-2025-03-11", - "name": "gpt-4o-search-preview-2025-03-11", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.00000125, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5", - "name": "gpt-5", - "provider": "openai", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.1", - "name": "gpt-5.1", - "provider": "openai", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.1-2025-11-13", - "name": "gpt-5.1-2025-11-13", - "provider": "openai", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.1-chat-latest", - "name": "gpt-5.1-chat-latest", - "provider": "openai", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.2", - "name": "gpt-5.2", - "provider": "openai", - "input_cost_per_token": 0.00000175, - "output_cost_per_token": 0.000014, - "cache_read_input_token_cost": 1.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.2-2025-12-11", - "name": "gpt-5.2-2025-12-11", - "provider": "openai", - "input_cost_per_token": 0.00000175, - "output_cost_per_token": 0.000014, - "cache_read_input_token_cost": 1.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.2-chat-latest", - "name": "gpt-5.2-chat-latest", - "provider": "openai", - "input_cost_per_token": 0.00000175, - "output_cost_per_token": 0.000014, - "cache_read_input_token_cost": 1.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.3-chat-latest", - "name": "gpt-5.3-chat-latest", - "provider": "openai", - "input_cost_per_token": 0.00000175, - "output_cost_per_token": 0.000014, - "cache_read_input_token_cost": 1.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.6", - "name": "gpt-5.6", - "provider": "openai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.6-sol", - "name": "gpt-5.6-sol", - "provider": "openai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.6-terra", - "name": "gpt-5.6-terra", - "provider": "openai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000012, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.6-luna", - "name": "gpt-5.6-luna", - "provider": "openai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": 2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.5", - "name": "gpt-5.5", - "provider": "openai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.5-2026-04-23", - "name": "gpt-5.5-2026-04-23", - "provider": "openai", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.00003, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.4", - "name": "gpt-5.4", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 2.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.4-2026-03-05", - "name": "gpt-5.4-2026-03-05", - "provider": "openai", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 2.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.4-mini", - "name": "gpt-5.4-mini", - "provider": "openai", - "input_cost_per_token": 7.5e-7, - "output_cost_per_token": 0.0000045, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.4-mini-2026-03-17", - "name": "gpt-5.4-mini-2026-03-17", - "provider": "openai", - "input_cost_per_token": 7.5e-7, - "output_cost_per_token": 0.0000045, - "cache_read_input_token_cost": 7.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.4-nano", - "name": "gpt-5.4-nano", - "provider": "openai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5.4-nano-2026-03-17", - "name": "gpt-5.4-nano-2026-03-17", - "provider": "openai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5-2025-08-07", - "name": "gpt-5-2025-08-07", - "provider": "openai", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5-chat", - "name": "gpt-5-chat", - "provider": "openai", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5-chat-latest", - "name": "gpt-5-chat-latest", - "provider": "openai", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5-mini", - "name": "gpt-5-mini", - "provider": "openai", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5-mini-2025-08-07", - "name": "gpt-5-mini-2025-08-07", - "provider": "openai", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5-nano", - "name": "gpt-5-nano", - "provider": "openai", - "input_cost_per_token": 5e-8, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 5e-9, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5-nano-2025-08-07", - "name": "gpt-5-nano-2025-08-07", - "provider": "openai", - "input_cost_per_token": 5e-8, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 5e-9, - "output_cost_per_reasoning_token": null - }, - { - "id": "jp.anthropic.claude-sonnet-4-5-20250929-v1:0", - "name": "jp.anthropic.claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000033, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 3.3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "jp.anthropic.claude-haiku-4-5-20251001-v1:0", - "name": "jp.anthropic.claude-haiku-4-5-20251001-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000055, - "cache_read_input_token_cost": 1.1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama2-13b-chat-v1", - "name": "meta.llama2-13b-chat-v1", - "provider": "bedrock", - "input_cost_per_token": 7.5e-7, - "output_cost_per_token": 0.000001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama2-70b-chat-v1", - "name": "meta.llama2-70b-chat-v1", - "provider": "bedrock", - "input_cost_per_token": 0.00000195, - "output_cost_per_token": 0.00000256, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-1-405b-instruct-v1:0", - "name": "meta.llama3-1-405b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000532, - "output_cost_per_token": 0.000016, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-1-70b-instruct-v1:0", - "name": "meta.llama3-1-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 9.9e-7, - "output_cost_per_token": 9.9e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-1-8b-instruct-v1:0", - "name": "meta.llama3-1-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 2.2e-7, - "output_cost_per_token": 2.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-2-11b-instruct-v1:0", - "name": "meta.llama3-2-11b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3.5e-7, - "output_cost_per_token": 3.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-2-1b-instruct-v1:0", - "name": "meta.llama3-2-1b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 1e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-2-3b-instruct-v1:0", - "name": "meta.llama3-2-3b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 1.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-2-90b-instruct-v1:0", - "name": "meta.llama3-2-90b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-3-70b-instruct-v1:0", - "name": "meta.llama3-3-70b-instruct-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 7.2e-7, - "output_cost_per_token": 7.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-70b-instruct-v1:0", - "name": "meta.llama3-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000265, - "output_cost_per_token": 0.0000035, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama3-8b-instruct-v1:0", - "name": "meta.llama3-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama4-maverick-17b-instruct-v1:0", - "name": "meta.llama4-maverick-17b-instruct-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 2.4e-7, - "output_cost_per_token": 9.7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "meta.llama4-scout-17b-instruct-v1:0", - "name": "meta.llama4-scout-17b-instruct-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 1.7e-7, - "output_cost_per_token": 6.6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "minimax.minimax-m2", - "name": "minimax.minimax-m2", - "provider": "bedrock_converse", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "minimax.minimax-m2.1", - "name": "minimax.minimax-m2.1", - "provider": "bedrock_converse", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "minimax.minimax-m2.5", - "name": "minimax.minimax-m2.5", - "provider": "bedrock_converse", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.devstral-2-123b", - "name": "mistral.devstral-2-123b", - "provider": "bedrock_converse", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.magistral-small-2509", - "name": "mistral.magistral-small-2509", - "provider": "bedrock_converse", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.ministral-3-14b-instruct", - "name": "mistral.ministral-3-14b-instruct", - "provider": "bedrock_converse", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.ministral-3-3b-instruct", - "name": "mistral.ministral-3-3b-instruct", - "provider": "bedrock_converse", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 1e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.ministral-3-8b-instruct", - "name": "mistral.ministral-3-8b-instruct", - "provider": "bedrock_converse", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 1.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.mistral-7b-instruct-v0:2", - "name": "mistral.mistral-7b-instruct-v0:2", - "provider": "bedrock", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.mistral-large-2402-v1:0", - "name": "mistral.mistral-large-2402-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.mistral-large-2407-v1:0", - "name": "mistral.mistral-large-2407-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000009, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.mistral-large-3-675b-instruct", - "name": "mistral.mistral-large-3-675b-instruct", - "provider": "bedrock_converse", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.mistral-small-2402-v1:0", - "name": "mistral.mistral-small-2402-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.mixtral-8x7b-instruct-v0:1", - "name": "mistral.mixtral-8x7b-instruct-v0:1", - "provider": "bedrock", - "input_cost_per_token": 4.5e-7, - "output_cost_per_token": 7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.voxtral-mini-3b-2507", - "name": "mistral.voxtral-mini-3b-2507", - "provider": "bedrock_converse", - "input_cost_per_token": 4e-8, - "output_cost_per_token": 4e-8, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "mistral.voxtral-small-24b-2507", - "name": "mistral.voxtral-small-24b-2507", - "provider": "bedrock_converse", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "moonshot.kimi-k2-thinking", - "name": "moonshot.kimi-k2-thinking", - "provider": "bedrock_converse", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "moonshotai.kimi-k2.5", - "name": "moonshotai.kimi-k2.5", - "provider": "bedrock_converse", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "nvidia.nemotron-nano-12b-v2", - "name": "nvidia.nemotron-nano-12b-v2", - "provider": "bedrock_converse", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "nvidia.nemotron-nano-9b-v2", - "name": "nvidia.nemotron-nano-9b-v2", - "provider": "bedrock_converse", - "input_cost_per_token": 6e-8, - "output_cost_per_token": 2.3e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "nvidia.nemotron-nano-3-30b", - "name": "nvidia.nemotron-nano-3-30b", - "provider": "bedrock_converse", - "input_cost_per_token": 6e-8, - "output_cost_per_token": 2.4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "nvidia.nemotron-super-3-120b", - "name": "nvidia.nemotron-super-3-120b", - "provider": "bedrock_converse", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "o1", - "name": "o1", - "provider": "openai", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": 0.0000075, - "output_cost_per_reasoning_token": null - }, - { - "id": "o1-2024-12-17", - "name": "o1-2024-12-17", - "provider": "openai", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.00006, - "cache_read_input_token_cost": 0.0000075, - "output_cost_per_reasoning_token": null - }, - { - "id": "o3", - "name": "o3", - "provider": "openai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "o3-2025-04-16", - "name": "o3-2025-04-16", - "provider": "openai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "o3-mini", - "name": "o3-mini", - "provider": "openai", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000044, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "o3-mini-2025-01-31", - "name": "o3-mini-2025-01-31", - "provider": "openai", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000044, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "o4-mini", - "name": "o4-mini", - "provider": "openai", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000044, - "cache_read_input_token_cost": 2.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "o4-mini-2025-04-16", - "name": "o4-mini-2025-04-16", - "provider": "openai", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000044, - "cache_read_input_token_cost": 2.75e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "openai.gpt-oss-120b-1:0", - "name": "openai.gpt-oss-120b-1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "openai.gpt-oss-20b-1:0", - "name": "openai.gpt-oss-20b-1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 7e-8, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "openai.gpt-oss-safeguard-120b", - "name": "openai.gpt-oss-safeguard-120b", - "provider": "bedrock_converse", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "openai.gpt-oss-safeguard-20b", - "name": "openai.gpt-oss-safeguard-20b", - "provider": "bedrock_converse", - "input_cost_per_token": 7e-8, - "output_cost_per_token": 2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "qwen.qwen3-coder-480b-a35b-v1:0", - "name": "qwen.qwen3-coder-480b-a35b-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 2.2e-7, - "output_cost_per_token": 0.0000018, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "qwen.qwen3-235b-a22b-2507-v1:0", - "name": "qwen.qwen3-235b-a22b-2507-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 2.2e-7, - "output_cost_per_token": 8.8e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "qwen.qwen3-coder-30b-a3b-v1:0", - "name": "qwen.qwen3-coder-30b-a3b-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "qwen.qwen3-32b-v1:0", - "name": "qwen.qwen3-32b-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "qwen.qwen3-next-80b-a3b", - "name": "qwen.qwen3-next-80b-a3b", - "provider": "bedrock_converse", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "qwen.qwen3-vl-235b-a22b", - "name": "qwen.qwen3-vl-235b-a22b", - "provider": "bedrock_converse", - "input_cost_per_token": 5.3e-7, - "output_cost_per_token": 0.00000266, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "qwen.qwen3-coder-next", - "name": "qwen.qwen3-coder-next", - "provider": "bedrock_converse", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.amazon.nova-lite-v1:0", - "name": "us.amazon.nova-lite-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 6e-8, - "output_cost_per_token": 2.4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.amazon.nova-micro-v1:0", - "name": "us.amazon.nova-micro-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 3.5e-8, - "output_cost_per_token": 1.4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.amazon.nova-premier-v1:0", - "name": "us.amazon.nova-premier-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000025, - "output_cost_per_token": 0.0000125, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.amazon.nova-pro-v1:0", - "name": "us.amazon.nova-pro-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 8e-7, - "output_cost_per_token": 0.0000032, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-3-5-haiku-20241022-v1:0", - "name": "us.anthropic.claude-3-5-haiku-20241022-v1:0", - "provider": "bedrock", - "input_cost_per_token": 8e-7, - "output_cost_per_token": 0.000004, - "cache_read_input_token_cost": 8e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-haiku-4-5-20251001-v1:0", - "name": "us.anthropic.claude-haiku-4-5-20251001-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000055, - "cache_read_input_token_cost": 1.1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-3-5-sonnet-20240620-v1:0", - "name": "us.anthropic.claude-3-5-sonnet-20240620-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", - "name": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", - "name": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-3-haiku-20240307-v1:0", - "name": "us.anthropic.claude-3-haiku-20240307-v1:0", - "provider": "bedrock", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-3-opus-20240229-v1:0", - "name": "us.anthropic.claude-3-opus-20240229-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-3-sonnet-20240229-v1:0", - "name": "us.anthropic.claude-3-sonnet-20240229-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-opus-4-1-20250805-v1:0", - "name": "us.anthropic.claude-opus-4-1-20250805-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "name": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000033, - "output_cost_per_token": 0.0000165, - "cache_read_input_token_cost": 3.3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0", - "name": "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000036, - "output_cost_per_token": 0.000018, - "cache_read_input_token_cost": 3.6e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "au.anthropic.claude-haiku-4-5-20251001-v1:0", - "name": "au.anthropic.claude-haiku-4-5-20251001-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000011, - "output_cost_per_token": 0.0000055, - "cache_read_input_token_cost": 1.1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-opus-4-20250514-v1:0", - "name": "us.anthropic.claude-opus-4-20250514-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-opus-4-5-20251101-v1:0", - "name": "us.anthropic.claude-opus-4-5-20251101-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.0000055, - "output_cost_per_token": 0.0000275, - "cache_read_input_token_cost": 5.5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "global.anthropic.claude-opus-4-5-20251101-v1:0", - "name": "global.anthropic.claude-opus-4-5-20251101-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.anthropic.claude-opus-4-5-20251101-v1:0", - "name": "eu.anthropic.claude-opus-4-5-20251101-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.anthropic.claude-sonnet-4-20250514-v1:0", - "name": "us.anthropic.claude-sonnet-4-20250514-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.deepseek.r1-v1:0", - "name": "us.deepseek.r1-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.deepseek.v3.2", - "name": "us.deepseek.v3.2", - "provider": "bedrock_converse", - "input_cost_per_token": 6.2e-7, - "output_cost_per_token": 0.00000185, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "eu.deepseek.v3.2", - "name": "eu.deepseek.v3.2", - "provider": "bedrock_converse", - "input_cost_per_token": 7.4e-7, - "output_cost_per_token": 0.00000222, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama3-1-405b-instruct-v1:0", - "name": "us.meta.llama3-1-405b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.00000532, - "output_cost_per_token": 0.000016, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama3-1-70b-instruct-v1:0", - "name": "us.meta.llama3-1-70b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 9.9e-7, - "output_cost_per_token": 9.9e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama3-1-8b-instruct-v1:0", - "name": "us.meta.llama3-1-8b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 2.2e-7, - "output_cost_per_token": 2.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama3-2-11b-instruct-v1:0", - "name": "us.meta.llama3-2-11b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 3.5e-7, - "output_cost_per_token": 3.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama3-2-1b-instruct-v1:0", - "name": "us.meta.llama3-2-1b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 1e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama3-2-3b-instruct-v1:0", - "name": "us.meta.llama3-2-3b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 1.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama3-2-90b-instruct-v1:0", - "name": "us.meta.llama3-2-90b-instruct-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama3-3-70b-instruct-v1:0", - "name": "us.meta.llama3-3-70b-instruct-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 7.2e-7, - "output_cost_per_token": 7.2e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama4-maverick-17b-instruct-v1:0", - "name": "us.meta.llama4-maverick-17b-instruct-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 2.4e-7, - "output_cost_per_token": 9.7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.meta.llama4-scout-17b-instruct-v1:0", - "name": "us.meta.llama4-scout-17b-instruct-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 1.7e-7, - "output_cost_per_token": 6.6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "us.mistral.pixtral-large-2502-v1:0", - "name": "us.mistral.pixtral-large-2502-v1:0", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-5-haiku", - "name": "claude-3-5-haiku", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-5-haiku@20241022", - "name": "claude-3-5-haiku@20241022", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-haiku-4-5", - "name": "claude-haiku-4-5", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-haiku-4-5@20251001", - "name": "claude-haiku-4-5@20251001", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-5-sonnet", - "name": "claude-3-5-sonnet", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-5-sonnet@20240620", - "name": "claude-3-5-sonnet@20240620", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-7-sonnet@20250219", - "name": "claude-3-7-sonnet@20250219", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-haiku", - "name": "claude-3-haiku", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-haiku@20240307", - "name": "claude-3-haiku@20240307", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.00000125, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-opus", - "name": "claude-3-opus", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-opus@20240229", - "name": "claude-3-opus@20240229", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-sonnet", - "name": "claude-3-sonnet", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-3-sonnet@20240229", - "name": "claude-3-sonnet@20240229", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4", - "name": "claude-opus-4", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-1", - "name": "claude-opus-4-1", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-1@20250805", - "name": "claude-opus-4-1@20250805", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-5", - "name": "claude-opus-4-5", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-5@20251101", - "name": "claude-opus-4-5@20251101", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-6", - "name": "claude-opus-4-6", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-6@default", - "name": "claude-opus-4-6@default", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-7", - "name": "claude-opus-4-7", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-7@default", - "name": "claude-opus-4-7@default", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-fable-5", - "name": "claude-fable-5", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00005, - "cache_read_input_token_cost": 0.000001, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-fable-5@default", - "name": "claude-fable-5@default", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.00005, - "cache_read_input_token_cost": 0.000001, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-5", - "name": "claude-opus-5", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-5@default", - "name": "claude-opus-5@default", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-8", - "name": "claude-opus-4-8", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4-8@default", - "name": "claude-opus-4-8@default", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 5e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-sonnet-4-5", - "name": "claude-sonnet-4-5", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-sonnet-5", - "name": "claude-sonnet-5", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-sonnet-4-6", - "name": "claude-sonnet-4-6", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-sonnet-4-5@20250929", - "name": "claude-sonnet-4-5@20250929", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-opus-4@20250514", - "name": "claude-opus-4@20250514", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000075, - "cache_read_input_token_cost": 0.0000015, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-sonnet-4", - "name": "claude-sonnet-4", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-sonnet-4@20250514", - "name": "claude-sonnet-4@20250514", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistralai/codestral-2@001", - "name": "mistralai/codestral-2@001", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 9e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/codestral-2", - "name": "codestral-2", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 9e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/codestral-2@001", - "name": "codestral-2@001", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 9e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistralai/codestral-2", - "name": "mistralai/codestral-2", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 9e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/codestral-2501", - "name": "codestral-2501", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/codestral@2405", - "name": "codestral@2405", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/codestral@latest", - "name": "codestral@latest", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/deepseek-ai/deepseek-v3.1-maas", - "name": "deepseek-ai/deepseek-v3.1-maas", - "provider": "vertex_ai-deepseek_models", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/deepseek-ai/deepseek-v3.2-maas", - "name": "deepseek-ai/deepseek-v3.2-maas", - "provider": "vertex_ai-deepseek_models", - "input_cost_per_token": 5.6e-7, - "output_cost_per_token": 0.00000168, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/deepseek-ai/deepseek-r1-0528-maas", - "name": "deepseek-ai/deepseek-r1-0528-maas", - "provider": "vertex_ai-deepseek_models", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/gemini-3.1-flash-lite-preview", - "name": "gemini-3.1-flash-lite-preview", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": 0.0000015 - }, - { - "id": "vertex_ai/gemini-3.1-flash-lite", - "name": "gemini-3.1-flash-lite", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.0000015, - "cache_read_input_token_cost": 2.5e-8, - "output_cost_per_reasoning_token": 0.0000015 - }, - { - "id": "vertex_ai/gemini-3.5-flash-lite", - "name": "gemini-3.5-flash-lite", - "provider": "vertex_ai-language-models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "vertex_ai/jamba-1.5", - "name": "jamba-1.5", - "provider": "vertex_ai-ai21_models", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/jamba-1.5-large", - "name": "jamba-1.5-large", - "provider": "vertex_ai-ai21_models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/jamba-1.5-large@001", - "name": "jamba-1.5-large@001", - "provider": "vertex_ai-ai21_models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/jamba-1.5-mini", - "name": "jamba-1.5-mini", - "provider": "vertex_ai-ai21_models", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/jamba-1.5-mini@001", - "name": "jamba-1.5-mini@001", - "provider": "vertex_ai-ai21_models", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama-3.1-405b-instruct-maas", - "name": "meta/llama-3.1-405b-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000016, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama-3.1-70b-instruct-maas", - "name": "meta/llama-3.1-70b-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama-3.1-8b-instruct-maas", - "name": "meta/llama-3.1-8b-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama-3.2-90b-vision-instruct-maas", - "name": "meta/llama-3.2-90b-vision-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas", - "name": "meta/llama-4-maverick-17b-128e-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 3.5e-7, - "output_cost_per_token": 0.00000115, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas", - "name": "meta/llama-4-maverick-17b-16e-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 3.5e-7, - "output_cost_per_token": 0.00000115, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas", - "name": "meta/llama-4-scout-17b-128e-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas", - "name": "meta/llama-4-scout-17b-16e-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 7e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama3-405b-instruct-maas", - "name": "meta/llama3-405b-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama3-70b-instruct-maas", - "name": "meta/llama3-70b-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/meta/llama3-8b-instruct-maas", - "name": "meta/llama3-8b-instruct-maas", - "provider": "vertex_ai-llama_models", - "input_cost_per_token": 0, - "output_cost_per_token": 0, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/minimaxai/minimax-m2-maas", - "name": "minimaxai/minimax-m2-maas", - "provider": "vertex_ai-minimax_models", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/moonshotai/kimi-k2-thinking-maas", - "name": "moonshotai/kimi-k2-thinking-maas", - "provider": "vertex_ai-moonshot_models", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/zai-org/glm-4.7-maas", - "name": "zai-org/glm-4.7-maas", - "provider": "vertex_ai-zai_models", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000022, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/zai-org/glm-5-maas", - "name": "zai-org/glm-5-maas", - "provider": "vertex_ai-zai_models", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.0000032, - "cache_read_input_token_cost": 1e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-medium-3", - "name": "mistral-medium-3", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-medium-3@001", - "name": "mistral-medium-3@001", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistralai/mistral-medium-3", - "name": "mistralai/mistral-medium-3", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistralai/mistral-medium-3@001", - "name": "mistralai/mistral-medium-3@001", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 4e-7, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-large-2411", - "name": "mistral-large-2411", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-large@2407", - "name": "mistral-large@2407", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-large@2411-001", - "name": "mistral-large@2411-001", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-large@latest", - "name": "mistral-large@latest", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-nemo@2407", - "name": "mistral-nemo@2407", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-nemo@latest", - "name": "mistral-nemo@latest", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 1.5e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-small-2503", - "name": "mistral-small-2503", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/mistral-small-2503@001", - "name": "mistral-small-2503@001", - "provider": "vertex_ai-mistral_models", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000003, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/google/gemma-4-26b-a4b-it-maas", - "name": "google/gemma-4-26b-a4b-it-maas", - "provider": "vertex_ai-openai_models", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/openai/gpt-oss-120b-maas", - "name": "openai/gpt-oss-120b-maas", - "provider": "vertex_ai-openai_models", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/openai/gpt-oss-20b-maas", - "name": "openai/gpt-oss-20b-maas", - "provider": "vertex_ai-openai_models", - "input_cost_per_token": 7.5e-8, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/xai/grok-4.1-fast-non-reasoning", - "name": "xai/grok-4.1-fast-non-reasoning", - "provider": "vertex_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 5e-7, - "cache_read_input_token_cost": 5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/xai/grok-4.1-fast-reasoning", - "name": "xai/grok-4.1-fast-reasoning", - "provider": "vertex_ai", - "input_cost_per_token": 2e-7, - "output_cost_per_token": 5e-7, - "cache_read_input_token_cost": 5e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/xai/grok-4.20-non-reasoning", - "name": "xai/grok-4.20-non-reasoning", - "provider": "vertex_ai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/xai/grok-4.20-reasoning", - "name": "xai/grok-4.20-reasoning", - "provider": "vertex_ai", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas", - "name": "qwen/qwen3-235b-a22b-instruct-2507-maas", - "provider": "vertex_ai-qwen_models", - "input_cost_per_token": 2.5e-7, - "output_cost_per_token": 0.000001, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", - "name": "qwen/qwen3-coder-480b-a35b-instruct-maas", - "provider": "vertex_ai-qwen_models", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000004, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas", - "name": "qwen/qwen3-next-80b-a3b-instruct-maas", - "provider": "vertex_ai-qwen_models", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas", - "name": "qwen/qwen3-next-80b-a3b-thinking-maas", - "provider": "vertex_ai-qwen_models", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 0.0000012, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "zai.glm-4.7", - "name": "zai.glm-4.7", - "provider": "bedrock_converse", - "input_cost_per_token": 6e-7, - "output_cost_per_token": 0.0000022, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "zai.glm-5", - "name": "zai.glm-5", - "provider": "bedrock_converse", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.0000032, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "zai.glm-4.7-flash", - "name": "zai.glm-4.7-flash", - "provider": "bedrock_converse", - "input_cost_per_token": 7e-8, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5-search-api", - "name": "gpt-5-search-api", - "provider": "openai", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gpt-5-search-api-2025-10-14", - "name": "gpt-5-search-api-2025-10-14", - "provider": "openai", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-2.0-flash-lite-001", - "name": "gemini-2.0-flash-lite-001", - "provider": "gemini", - "input_cost_per_token": 7.5e-8, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": 1.875e-8, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-2.5-flash-native-audio-latest", - "name": "gemini-2.5-flash-native-audio-latest", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-2.5-flash-native-audio-preview-09-2025", - "name": "gemini-2.5-flash-native-audio-preview-09-2025", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-2.5-flash-native-audio-preview-12-2025", - "name": "gemini-2.5-flash-native-audio-preview-12-2025", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-3.1-flash-live-preview", - "name": "gemini-3.1-flash-live-preview", - "provider": "gemini", - "input_cost_per_token": 7.5e-7, - "output_cost_per_token": 0.0000045, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-2.5-flash-native-audio-latest", - "name": "gemini-2.5-flash-native-audio-latest", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-2.5-flash-native-audio-preview-09-2025", - "name": "gemini-2.5-flash-native-audio-preview-09-2025", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-2.5-flash-native-audio-preview-12-2025", - "name": "gemini-2.5-flash-native-audio-preview-12-2025", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-3.1-flash-live-preview", - "name": "gemini-3.1-flash-live-preview", - "provider": "gemini", - "input_cost_per_token": 7.5e-7, - "output_cost_per_token": 0.0000045, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-flash-latest", - "name": "gemini-flash-latest", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "gemini-flash-lite-latest", - "name": "gemini-flash-lite-latest", - "provider": "gemini", - "input_cost_per_token": 1e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": 1e-8, - "output_cost_per_reasoning_token": 4e-7 - }, - { - "id": "gemini-pro-latest", - "name": "gemini-pro-latest", - "provider": "gemini", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini/gemini-pro-latest", - "name": "gemini-pro-latest", - "provider": "gemini", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 1.25e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "gemini-exp-1206", - "name": "gemini-exp-1206", - "provider": "gemini", - "input_cost_per_token": 3e-7, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 3e-8, - "output_cost_per_reasoning_token": 0.0000025 - }, - { - "id": "vertex_ai/claude-sonnet-5@default", - "name": "claude-sonnet-5@default", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "vertex_ai/claude-sonnet-4-6@default", - "name": "claude-sonnet-4-6@default", - "provider": "vertex_ai-anthropic_models", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 3e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock_mantle/openai.gpt-oss-120b", - "name": "openai.gpt-oss-120b", - "provider": "bedrock_mantle", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock_mantle/openai.gpt-oss-20b", - "name": "openai.gpt-oss-20b", - "provider": "bedrock_mantle", - "input_cost_per_token": 7.5e-8, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock_mantle/openai.gpt-oss-safeguard-120b", - "name": "openai.gpt-oss-safeguard-120b", - "provider": "bedrock_mantle", - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock_mantle/openai.gpt-oss-safeguard-20b", - "name": "openai.gpt-oss-safeguard-20b", - "provider": "bedrock_mantle", - "input_cost_per_token": 7.5e-8, - "output_cost_per_token": 3e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock_mantle/google.gemma-4-31b", - "name": "google.gemma-4-31b", - "provider": "bedrock_mantle", - "input_cost_per_token": 1.4e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock_mantle/google.gemma-4-26b-a4b", - "name": "google.gemma-4-26b-a4b", - "provider": "bedrock_mantle", - "input_cost_per_token": 1.3e-7, - "output_cost_per_token": 4e-7, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock_mantle/google.gemma-4-e2b", - "name": "google.gemma-4-e2b", - "provider": "bedrock_mantle", - "input_cost_per_token": 4e-8, - "output_cost_per_token": 8e-8, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock_mantle/xai.grok-4.3", - "name": "xai.grok-4.3", - "provider": "bedrock_mantle", - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.0000025, - "cache_read_input_token_cost": 2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-east-1/zai.glm-5", - "name": "us-east-1/zai.glm-5", - "provider": "bedrock", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.0000032, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-west-2/zai.glm-5", - "name": "us-west-2/zai.glm-5", - "provider": "bedrock", - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.0000032, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", - "name": "us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000012, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": 1.2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", - "name": "us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", - "provider": "bedrock", - "input_cost_per_token": 0.0000012, - "output_cost_per_token": 0.000006, - "cache_read_input_token_cost": 1.2e-7, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-3.5-turbo-instruct-0914", - "name": "gpt-3.5-turbo-instruct-0914", - "provider": "azure_text", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-35-turbo-instruct", - "name": "gpt-35-turbo-instruct", - "provider": "azure_text", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "azure/gpt-35-turbo-instruct-0914", - "name": "gpt-35-turbo-instruct-0914", - "provider": "azure_text", - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "text-unicorn", - "name": "text-unicorn", - "provider": "vertex_ai-text-models", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.000028, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - }, - { - "id": "text-unicorn@001", - "name": "text-unicorn@001", - "provider": "vertex_ai-text-models", - "input_cost_per_token": 0.00001, - "output_cost_per_token": 0.000028, - "cache_read_input_token_cost": null, - "output_cost_per_reasoning_token": null - } - ], - "success": true, - "error": false, - "message": "Model catalog retrieved successfully", - "status": 200 -} \ No newline at end of file diff --git a/server/lib/aiModelCatalog.ts b/server/lib/aiModelCatalog.ts index 9986366fb..a6eab149f 100644 --- a/server/lib/aiModelCatalog.ts +++ b/server/lib/aiModelCatalog.ts @@ -47,27 +47,21 @@ export type AiModelCatalogEntry = { provider: CatalogProvider; model: string; pricing: { - input: number | null; - output: number | null; - cacheRead: number | null; - reasoningOutput: number | null; + in: number | null; + out: number | null; + cache: number | null; + reasoning: number | null; }; }; type RawCatalogEntry = { - id?: string; - name?: string; - model?: string; + model: string; provider: string; - input_cost_per_token?: number | null; - output_cost_per_token?: number | null; - cache_read_input_token_cost?: number | null; - output_cost_per_reasoning_token?: number | null; pricing?: { - input?: number | null; - output?: number | null; - cacheRead?: number | null; - reasoningOutput?: number | null; + in?: number | null; + out?: number | null; + cache?: number | null; + reasoning?: number | null; }; }; @@ -93,25 +87,18 @@ function normalizeEntry(raw: RawCatalogEntry): AiModelCatalogEntry | null { return null; } - const model = raw.model ?? raw.name ?? raw.id; - if (!model) { + if (!raw.model) { return null; } return { provider, - model, + model: raw.model, pricing: { - input: raw.pricing?.input ?? raw.input_cost_per_token ?? null, - output: raw.pricing?.output ?? raw.output_cost_per_token ?? null, - cacheRead: - raw.pricing?.cacheRead ?? - raw.cache_read_input_token_cost ?? - null, - reasoningOutput: - raw.pricing?.reasoningOutput ?? - raw.output_cost_per_reasoning_token ?? - null + in: raw.pricing?.in ?? null, + out: raw.pricing?.out ?? null, + cache: raw.pricing?.cache ?? null, + reasoning: raw.pricing?.reasoning ?? null } }; } diff --git a/server/lib/aiModelPricing.ts b/server/lib/aiModelPricing.ts index 569bfa42e..cb84d6d47 100644 --- a/server/lib/aiModelPricing.ts +++ b/server/lib/aiModelPricing.ts @@ -33,10 +33,10 @@ function toPricing( approximate: boolean ): AiModelPricing { return { - inputCostPerToken: entry.pricing.input, - outputCostPerToken: entry.pricing.output, - cacheReadInputTokenCost: entry.pricing.cacheRead, - outputCostPerReasoningToken: entry.pricing.reasoningOutput, + inputCostPerToken: entry.pricing.in, + outputCostPerToken: entry.pricing.out, + cacheReadInputTokenCost: entry.pricing.cache, + outputCostPerReasoningToken: entry.pricing.reasoning, approximate }; } From 49020fa6ea8601beb8e0a12a25c89b09ea7c002d Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 12 Aug 2026 10:38:08 -0400 Subject: [PATCH 133/296] add virtual api key validation in verifySession --- server/auth/verifyVirtualApiKey.ts | 316 ++++++++++++++++++ server/lib/aiCapabilities.ts | 14 +- server/lib/aiGatewayTrust.ts | 38 +++ server/lib/aiProviderDefaults.ts | 132 ++------ server/lib/traefik/getTraefikConfig.ts | 10 +- server/lib/virtualApiKey.ts | 8 + .../private/lib/traefik/getTraefikConfig.ts | 50 +-- server/routers/aiGateway/pipeline.ts | 65 +++- server/routers/aiGateway/targetRouting.ts | 4 +- server/routers/badger/logRequestAudit.ts | 5 +- server/routers/badger/verifySession.ts | 142 ++++++-- .../[providerId]/authentication/page.tsx | 2 +- .../[providerId]/general/page.tsx | 2 +- .../[providerId]/network/page.tsx | 2 +- .../settings/ai-providers/create/page.tsx | 2 +- src/components/AiProviderAuthTypeSelect.tsx | 2 +- .../AiProviderCapabilitiesSelect.tsx | 2 +- src/components/AiProviderTypeSelect.tsx | 2 +- src/components/CreateVirtualApiKeyForm.tsx | 8 +- src/components/EditVirtualApiKeyForm.tsx | 6 +- src/components/UserVirtualApiKeys.tsx | 16 +- src/components/ViewVirtualApiKeySecret.tsx | 8 +- src/components/VirtualApiKeysTable.tsx | 10 +- .../LauncherInferenceApiKeysSection.tsx | 14 +- src/components/user-selector.tsx | 2 +- src/lib/aiCapabilities.ts | 12 + src/lib/aiProviderDefaults.ts | 107 ++++++ src/lib/aiProviderFormSchema.ts | 5 +- src/lib/virtualApiKeyFormat.ts | 63 ++++ 29 files changed, 852 insertions(+), 197 deletions(-) create mode 100644 server/auth/verifyVirtualApiKey.ts create mode 100644 server/lib/aiGatewayTrust.ts create mode 100644 src/lib/aiCapabilities.ts create mode 100644 src/lib/aiProviderDefaults.ts create mode 100644 src/lib/virtualApiKeyFormat.ts diff --git a/server/auth/verifyVirtualApiKey.ts b/server/auth/verifyVirtualApiKey.ts new file mode 100644 index 000000000..354a27e74 --- /dev/null +++ b/server/auth/verifyVirtualApiKey.ts @@ -0,0 +1,316 @@ +import { canUserAccessResource } from "@server/auth/canUserAccessResource"; +import { + db, + users, + virtualApiKeyResources, + virtualApiKeys, + type VirtualApiKey +} from "@server/db"; +import config from "@server/lib/config"; +import { + decryptVirtualApiKeyToken, + VIRTUAL_API_KEY_PREFIX, + looksLikeVirtualApiKeyCredential +} from "@server/lib/virtualApiKey"; +import { getUserOrgRoles } from "@server/lib/userOrgRoles"; +import { and, eq } from "drizzle-orm"; +import { isWithinExpirationDate } from "oslo"; + +export type VirtualApiKeyCredential = { + virtualApiKeyId: string; + secret: string; +}; + +export type VirtualApiKeyUserData = { + userId: string; + username: string; + email: string | null; + name: string | null; + role: string | null; +}; + +function getHeader( + headers: Record | undefined, + name: string +): string | undefined { + if (!headers) { + return undefined; + } + if (headers[name] !== undefined) { + return headers[name]; + } + const lower = name.toLowerCase(); + for (const [key, value] of Object.entries(headers)) { + if (key.toLowerCase() === lower) { + return value; + } + } + return undefined; +} + +function parseVkCredential( + raw: string | undefined +): VirtualApiKeyCredential | null { + if (!raw || !looksLikeVirtualApiKeyCredential(raw)) { + return null; + } + const withoutPrefix = raw.trim().slice(VIRTUAL_API_KEY_PREFIX.length); + const dot = withoutPrefix.indexOf("."); + return { + virtualApiKeyId: withoutPrefix.slice(0, dot), + secret: withoutPrefix.slice(dot + 1) + }; +} + +/** + * Extract a virtual API key credential from provider-style auth headers. + * Checks Authorization Bearer / Splunk, x-api-key, x-goog-api-key, and + * cf-aig-authorization. First matching vk-{id}.{secret} wins. + */ +export function extractVirtualApiKeyCredential( + headers: Record | undefined +): VirtualApiKeyCredential | null { + if (!headers) { + return null; + } + + const authorization = getHeader(headers, "authorization"); + if (authorization) { + const bearerMatch = authorization.match(/^Bearer\s+(.+)$/i); + if (bearerMatch) { + const credential = parseVkCredential(bearerMatch[1]); + if (credential) { + return credential; + } + } + const splunkMatch = authorization.match(/^Splunk\s+(.+)$/i); + if (splunkMatch) { + const credential = parseVkCredential(splunkMatch[1]); + if (credential) { + return credential; + } + } + } + + const cfAig = getHeader(headers, "cf-aig-authorization"); + if (cfAig) { + const bearerMatch = cfAig.match(/^Bearer\s+(.+)$/i); + const credential = parseVkCredential( + bearerMatch ? bearerMatch[1] : cfAig + ); + if (credential) { + return credential; + } + } + + for (const name of ["x-api-key", "x-goog-api-key"] as const) { + const credential = parseVkCredential(getHeader(headers, name)); + if (credential) { + return credential; + } + } + + return null; +} + +async function buildUserData( + userId: string, + orgId: string +): Promise { + const [user] = await db + .select() + .from(users) + .where(eq(users.userId, userId)) + .limit(1); + + if (!user) { + return undefined; + } + + if ( + config.getRawConfig().flags?.require_email_verification && + !user.emailVerified + ) { + return undefined; + } + + const userOrgRoles = await getUserOrgRoles(user.userId, orgId); + if (userOrgRoles.length === 0) { + return undefined; + } + + return { + userId: user.userId, + username: user.username, + email: user.email, + name: user.name, + role: userOrgRoles.map((r) => r.roleName).join(", ") || null + }; +} + +async function userHasResourceAccess( + userId: string, + resourceId: number, + orgId: string +): Promise<{ allowed: boolean; userData?: VirtualApiKeyUserData }> { + const [user] = await db + .select() + .from(users) + .where(eq(users.userId, userId)) + .limit(1); + + if (!user) { + return { allowed: false }; + } + + if ( + config.getRawConfig().flags?.require_email_verification && + !user.emailVerified + ) { + return { allowed: false }; + } + + const userOrgRoles = await getUserOrgRoles(user.userId, orgId); + if (userOrgRoles.length === 0) { + return { allowed: false }; + } + + const allowed = await canUserAccessResource({ + userId, + resourceId, + roleIds: userOrgRoles.map((r) => r.roleId) + }); + + if (!allowed) { + return { allowed: false }; + } + + return { + allowed: true, + userData: { + userId: user.userId, + username: user.username, + email: user.email, + name: user.name, + role: userOrgRoles.map((r) => r.roleName).join(", ") || null + } + }; +} + +async function manualKeyHasResourceAccess( + key: VirtualApiKey, + resourceId: number +): Promise { + if (key.allResources) { + return true; + } + + const [row] = await db + .select({ resourceId: virtualApiKeyResources.resourceId }) + .from(virtualApiKeyResources) + .where( + and( + eq(virtualApiKeyResources.virtualApiKeyId, key.virtualApiKeyId), + eq(virtualApiKeyResources.resourceId, resourceId) + ) + ) + .limit(1); + + return Boolean(row); +} + +async function touchLastUsedAt(virtualApiKeyId: string): Promise { + try { + await db + .update(virtualApiKeys) + .set({ lastUsedAt: Date.now() }) + .where(eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId)); + } catch { + // Best-effort; do not fail auth on audit timestamp updates. + } +} + +export async function verifyVirtualApiKey({ + credential, + resourceId, + orgId +}: { + credential: VirtualApiKeyCredential; + resourceId: number; + orgId: string; +}): Promise<{ + valid: boolean; + error?: string; + key?: VirtualApiKey; + userData?: VirtualApiKeyUserData; +}> { + const [key] = await db + .select() + .from(virtualApiKeys) + .where(eq(virtualApiKeys.virtualApiKeyId, credential.virtualApiKeyId)) + .limit(1); + + if (!key) { + return { valid: false, error: "Virtual API key not found" }; + } + + if (key.orgId !== orgId) { + return { valid: false, error: "Virtual API key org mismatch" }; + } + + let plaintext: string; + try { + plaintext = decryptVirtualApiKeyToken(key.token); + } catch { + return { valid: false, error: "Virtual API key secret is invalid" }; + } + + if (plaintext !== credential.secret) { + return { valid: false, error: "Invalid virtual API key secret" }; + } + + if (key.expiresAt && !isWithinExpirationDate(new Date(key.expiresAt))) { + return { valid: false, error: "Virtual API key has expired" }; + } + + if (key.kind === "manual") { + const scoped = await manualKeyHasResourceAccess(key, resourceId); + if (!scoped) { + return { + valid: false, + error: "Virtual API key is not scoped to this resource" + }; + } + + let userData: VirtualApiKeyUserData | undefined; + if (key.userId) { + userData = await buildUserData(key.userId, orgId); + } + + await touchLastUsedAt(key.virtualApiKeyId); + return { valid: true, key, userData }; + } + + if (key.kind === "user") { + if (!key.userId) { + return { valid: false, error: "User virtual API key has no user" }; + } + + const access = await userHasResourceAccess( + key.userId, + resourceId, + orgId + ); + if (!access.allowed || !access.userData) { + return { + valid: false, + error: "User is not allowed to access this resource" + }; + } + + await touchLastUsedAt(key.virtualApiKeyId); + return { valid: true, key, userData: access.userData }; + } + + return { valid: false, error: "Unknown virtual API key kind" }; +} diff --git a/server/lib/aiCapabilities.ts b/server/lib/aiCapabilities.ts index 89fbd3d75..268acd36e 100644 --- a/server/lib/aiCapabilities.ts +++ b/server/lib/aiCapabilities.ts @@ -1,17 +1,7 @@ import type { Request } from "express"; +import { AI_CAPABILITIES, type AiCapability } from "@app/lib/aiCapabilities"; -export const AI_CAPABILITIES = [ - "openai_chat", - "openai_responses", - "anthropic_messages", - "gemini_generate_content", - "bedrock_model_invoke", - "google_generate_content", - "google_raw_predict", - "bedrock_converse" -] as const; - -export type AiCapability = (typeof AI_CAPABILITIES)[number]; +export { AI_CAPABILITIES, type AiCapability }; export type AiCapabilityRoute = { method: "POST"; diff --git a/server/lib/aiGatewayTrust.ts b/server/lib/aiGatewayTrust.ts new file mode 100644 index 000000000..2eb3c104d --- /dev/null +++ b/server/lib/aiGatewayTrust.ts @@ -0,0 +1,38 @@ +import { createHash } from "crypto"; +import config from "@server/lib/config"; + +export const AI_GATEWAY_TRUST_HEADER = "X-Pangolin-Ai-Gateway-Auth"; + +/** + * Derive a Traefik-injected trust token from the server secret. + * Traefik overwrites this header on inference routes so the AI gateway can + * trust Badger-injected Remote-* identity without re-validating credentials. + */ +export function deriveAiGatewayTrustToken(secret: string): string { + return createHash("sha256") + .update(`ai-gateway-trust:${secret}`) + .digest("hex"); +} + +export function getAiGatewayTrustToken(): string { + const secret = config.getRawConfig().server.secret; + if (!secret) { + throw new Error("Server secret is required for AI gateway trust token"); + } + return deriveAiGatewayTrustToken(secret); +} + +export function isAiGatewayTrustHeaderValid( + headers: Record | undefined, + expectedToken?: string +): boolean { + if (!headers) { + return false; + } + const expected = expectedToken ?? getAiGatewayTrustToken(); + const raw = + headers[AI_GATEWAY_TRUST_HEADER] ?? + headers[AI_GATEWAY_TRUST_HEADER.toLowerCase()]; + const value = Array.isArray(raw) ? raw[0] : raw; + return typeof value === "string" && value === expected; +} diff --git a/server/lib/aiProviderDefaults.ts b/server/lib/aiProviderDefaults.ts index a8b3a0ca0..217709f41 100644 --- a/server/lib/aiProviderDefaults.ts +++ b/server/lib/aiProviderDefaults.ts @@ -3,82 +3,29 @@ import { parseCapabilities, type AiCapability } from "@server/lib/aiCapabilities"; +import { stripVirtualApiKeyAuthHeaders } from "@app/lib/virtualApiKeyFormat"; +import { + AI_PROVIDER_AUTH_TYPES, + AI_PROVIDER_DEFAULTS, + authTypeRequiresApiKey, + defaultsForProviderType, + providerRequiresUpstreamUrl, + type AiBudgetUnit, + type AiProviderAuthType, + type AiProviderRoutingMode, + type AiProviderType +} from "@app/lib/aiProviderDefaults"; -export type AiProviderType = - | "openai" - | "anthropic" - | "googleGemini" - | "vertexAi" - | "bedrock" - | "microsoftFoundry" - | "openRouter" - | "vercelAiGateway" - | "custom"; - -export const AI_PROVIDER_AUTH_TYPES = [ - "bearer", - "x-api-key", - "x-goog-api-key", - "hec", - "cf-aig-authorization", - "none", - "passthrough" -] as const; - -export type AiProviderAuthType = (typeof AI_PROVIDER_AUTH_TYPES)[number]; -export type AiBudgetUnit = "usd" | "tokens"; -export type AiProviderRoutingMode = "url" | "target"; - -type AiProviderDefaults = { - upstreamUrl: string | null; - authType: AiProviderAuthType; - capabilities: readonly AiCapability[]; -}; - -export const AI_PROVIDER_DEFAULTS: Record< - Exclude, - AiProviderDefaults -> = { - openai: { - upstreamUrl: "https://api.openai.com/v1", - authType: "bearer", - capabilities: ["openai_chat", "openai_responses"] - }, - anthropic: { - upstreamUrl: "https://api.anthropic.com", - authType: "x-api-key", - capabilities: ["anthropic_messages"] - }, - googleGemini: { - upstreamUrl: "https://generativelanguage.googleapis.com", - authType: "x-goog-api-key", - capabilities: ["gemini_generate_content"] - }, - vertexAi: { - upstreamUrl: null, - authType: "bearer", - capabilities: ["google_generate_content", "google_raw_predict"] - }, - bedrock: { - upstreamUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", - authType: "bearer", - capabilities: ["bedrock_converse"] - }, - microsoftFoundry: { - upstreamUrl: null, - authType: "bearer", - capabilities: ["openai_chat", "openai_responses", "anthropic_messages"] - }, - openRouter: { - upstreamUrl: "https://openrouter.ai/api/v1", - authType: "bearer", - capabilities: ["openai_chat"] - }, - vercelAiGateway: { - upstreamUrl: "https://ai-gateway.vercel.sh/v1", - authType: "bearer", - capabilities: ["openai_chat", "openai_responses"] - } +export { + AI_PROVIDER_AUTH_TYPES, + AI_PROVIDER_DEFAULTS, + authTypeRequiresApiKey, + defaultsForProviderType, + providerRequiresUpstreamUrl, + type AiBudgetUnit, + type AiProviderAuthType, + type AiProviderRoutingMode, + type AiProviderType }; const CONFLICTING_AUTH_HEADERS = [ @@ -88,23 +35,6 @@ const CONFLICTING_AUTH_HEADERS = [ "cf-aig-authorization" ] as const; -export function authTypeRequiresApiKey(authType: AiProviderAuthType): boolean { - return authType !== "none" && authType !== "passthrough"; -} - -export function providerRequiresUpstreamUrl( - type: AiProviderType, - routingMode: AiProviderRoutingMode = "url" -): boolean { - if (routingMode === "target") { - return false; - } - if (type === "custom") { - return true; - } - return AI_PROVIDER_DEFAULTS[type].upstreamUrl === null; -} - export function resolveAiProviderCreateFields(input: { type: AiProviderType; upstreamUrl?: string | null; @@ -191,15 +121,18 @@ export function applyAiProviderCustomHeaders( /** * Apply provider auth to upstream headers. - * - Injected modes: strip client auth headers, then set the provider key. - * - none: strip client auth headers, send no auth. - * - passthrough: leave client auth headers as-is. + * - Always strips Pangolin virtual API key credentials from client auth headers. + * - Injected modes: strip conflicting client auth headers, then set the provider key. + * - none: strip conflicting client auth headers, send no auth. + * - passthrough: leave remaining client auth headers as-is (after VAK strip). */ export function applyAiProviderAuthHeaders( headers: Record, authType: AiProviderAuthType, apiKey: string | null ): void { + stripVirtualApiKeyAuthHeaders(headers); + if (authType === "passthrough") { return; } @@ -251,12 +184,3 @@ export function resolveCapabilitiesForCreate(input: { } return [...AI_PROVIDER_DEFAULTS[input.type].capabilities]; } - -export function defaultsForProviderType( - type: AiProviderType -): readonly AiCapability[] { - if (type === "custom") { - return []; - } - return AI_PROVIDER_DEFAULTS[type].capabilities; -} diff --git a/server/lib/traefik/getTraefikConfig.ts b/server/lib/traefik/getTraefikConfig.ts index a6535c766..559f1f7ad 100644 --- a/server/lib/traefik/getTraefikConfig.ts +++ b/server/lib/traefik/getTraefikConfig.ts @@ -24,6 +24,10 @@ import { resources, sites, Target, targets } from "@server/db"; import createPathRewriteMiddleware from "./middleware"; import { sanitize, encodePath, validatePathRewriteConfig } from "./utils"; import regionalCache from "@server/lib/cache"; +import { + AI_GATEWAY_TRUST_HEADER, + getAiGatewayTrustToken +} from "@server/lib/aiGatewayTrust"; const redirectHttpsMiddlewareName = "redirect-to-https"; const badgerMiddlewareName = "badger"; @@ -808,7 +812,8 @@ export async function getTraefikConfig( headers: { customRequestHeaders: { ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": fullDomain + "p-host": fullDomain, + [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken() } } }; @@ -911,7 +916,8 @@ export async function getTraefikConfig( headers: { customRequestHeaders: { ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": fullDomain + "p-host": fullDomain, + [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken() } } }; diff --git a/server/lib/virtualApiKey.ts b/server/lib/virtualApiKey.ts index 66ebb4b5a..e4f7480db 100644 --- a/server/lib/virtualApiKey.ts +++ b/server/lib/virtualApiKey.ts @@ -14,6 +14,14 @@ import config from "@server/lib/config"; import { decrypt, encrypt } from "@server/lib/crypto"; import { and, eq, inArray } from "drizzle-orm"; +export { + VIRTUAL_API_KEY_PREFIX, + formatVirtualApiKeyCredential, + formatVirtualApiKeyPreview, + looksLikeVirtualApiKeyCredential, + stripVirtualApiKeyAuthHeaders +} from "@app/lib/virtualApiKeyFormat"; + export type MintedVirtualApiKeySecret = { virtualApiKeyId: string; secret: string; diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index f51dad24e..dc7f0fef0 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -59,6 +59,10 @@ import { } from "#private/lib/certificates"; import { build } from "@server/build"; import regionalCache from "#private/lib/cache"; +import { + AI_GATEWAY_TRUST_HEADER, + getAiGatewayTrustToken +} from "@server/lib/aiGatewayTrust"; const redirectHttpsMiddlewareName = "redirect-to-https"; const redirectToRootMiddlewareName = "redirect-to-root"; @@ -1634,18 +1638,19 @@ export async function getTraefikConfig( config.getRawConfig().traefik.additional_middlewares || []; const routerMiddlewares = [badgerMiddlewareName]; - if (aiGatewayOverride) { - const irHeadersMiddlewareName = `${irKey}-headers-middleware`; - config_output.http.middlewares[irHeadersMiddlewareName] = { - headers: { - customRequestHeaders: { - ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": fullDomain - } + const irHeadersMiddlewareName = `${irKey}-headers-middleware`; + config_output.http.middlewares[irHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayOverride && aiGatewayHost + ? { Host: aiGatewayHost } + : {}), + ...(aiGatewayOverride ? { "p-host": fullDomain } : {}), + [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken() } - }; - routerMiddlewares.push(irHeadersMiddlewareName); - } + } + }; + routerMiddlewares.push(irHeadersMiddlewareName); routerMiddlewares.push(...additionalMiddlewares); @@ -1733,18 +1738,19 @@ export async function getTraefikConfig( config.getRawConfig().traefik.additional_middlewares || []; const routerMiddlewares: string[] = []; - if (aiGatewayOverride) { - const srHeadersMiddlewareName = `${srKey}-headers-middleware`; - config_output.http.middlewares[srHeadersMiddlewareName] = { - headers: { - customRequestHeaders: { - ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": fullDomain - } + const srHeadersMiddlewareName = `${srKey}-headers-middleware`; + config_output.http.middlewares[srHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayOverride && aiGatewayHost + ? { Host: aiGatewayHost } + : {}), + ...(aiGatewayOverride ? { "p-host": fullDomain } : {}), + [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken() } - }; - routerMiddlewares.push(srHeadersMiddlewareName); - } + } + }; + routerMiddlewares.push(srHeadersMiddlewareName); routerMiddlewares.push(...additionalMiddlewares); diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 7648e2355..5aa69d7fc 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -39,6 +39,10 @@ import { import { getUserOrgRoles } from "@server/lib/userOrgRoles"; import { isIpInCidr } from "@server/lib/ip"; import { localCache } from "@server/lib/cache"; +import { + AI_GATEWAY_TRUST_HEADER, + isAiGatewayTrustHeaderValid +} from "@server/lib/aiGatewayTrust"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; import { @@ -217,9 +221,42 @@ async function buildRequestUser( async function resolveRequestUser( req: Request, - _resourceId: number | null, + resourceId: number | null, orgId: string | null ): Promise { + // Public inference: identity comes from Badger via Remote-* only when the + // Traefik trust header proves the request passed verify-session (VAK). + if (isAiGatewayTrustHeaderValid(req.headers as Record)) { + const userId = getRequestHeader(req, "remote-user-id"); + if (userId) { + const username = getRequestHeader(req, "remote-user") || userId; + const email = getRequestHeader(req, "remote-email"); + const name = getRequestHeader(req, "remote-name"); + const role = getRequestHeader(req, "remote-role"); + const orgRoles = orgId ? await getUserOrgRoles(userId, orgId) : []; + + return { + userId, + username, + email: email || null, + name: name || null, + role: + role || orgRoles.map((r) => r.roleName).join(", ") || null, + roleIds: orgRoles.map((r) => r.roleId) + }; + } + + // Trusted request with no associated user (manual key without userId). + if (resourceId != null) { + return null; + } + } + + // Public inference must come through Badger; do not authorize via app session. + if (resourceId != null) { + return null; + } + const sessionToken = req.cookies?.[SESSION_COOKIE_NAME]; if (sessionToken) { const { session, user } = await validateSessionToken(sessionToken); @@ -251,6 +288,14 @@ async function resolveRequestUser( return buildRequestUser(client.userId, orgId); } +function getRequestHeader(req: Request, name: string): string | undefined { + const raw = req.headers[name.toLowerCase()]; + if (Array.isArray(raw)) { + return raw[0]; + } + return raw; +} + async function resolveTarget(host: string): Promise { const [[resourceRow], [siteResourceRow]] = await Promise.all([ db @@ -696,6 +741,21 @@ export async function handleAiGatewayProxy( orgId } = target; + // Public inference must pass Badger verify-session first. Traefik + // injects the trust header only on that path; the gateway trusts it + // and does not re-verify the virtual API key. + if ( + resourceId != null && + !isAiGatewayTrustHeaderValid(req.headers as Record) + ) { + return res.status(HttpCode.UNAUTHORIZED).json({ + error: { + message: + "Request must be authenticated via the inference resource" + } + }); + } + const capableAttachments = attachments.filter((a) => providerHasCapability(a.provider.capabilities, capability) ); @@ -823,7 +883,8 @@ export async function handleAiGatewayProxy( "transfer-encoding", "upgrade", "content-length", - "accept-encoding" + "accept-encoding", + AI_GATEWAY_TRUST_HEADER.toLowerCase() ]); const headers: Record = {}; diff --git a/server/routers/aiGateway/targetRouting.ts b/server/routers/aiGateway/targetRouting.ts index 00ef39d12..9ffbf1d1b 100644 --- a/server/routers/aiGateway/targetRouting.ts +++ b/server/routers/aiGateway/targetRouting.ts @@ -33,6 +33,7 @@ import { type RequestUser } from "@server/routers/aiGateway/pipeline"; import { streamAiGatewayResponse } from "@server/routers/aiGateway/streamAiGatewayResponse"; +import { AI_GATEWAY_TRUST_HEADER } from "@server/lib/aiGatewayTrust"; // Short TTL: long enough to spare the DB on a burst of requests, short // enough that target/site changes (added, removed, exit node moved) show up @@ -62,7 +63,8 @@ const SKIP_HEADERS = new Set([ "transfer-encoding", "upgrade", "content-length", - "accept-encoding" + "accept-encoding", + AI_GATEWAY_TRUST_HEADER.toLowerCase() ]); type ResolvedProviderTarget = { diff --git a/server/routers/badger/logRequestAudit.ts b/server/routers/badger/logRequestAudit.ts index 884fb7ae4..3fb97dce5 100644 --- a/server/routers/badger/logRequestAudit.ts +++ b/server/routers/badger/logRequestAudit.ts @@ -19,6 +19,7 @@ Reasons: 106 - Valid email 107 - Valid SSO 108 - Connected Client +109 - Valid Virtual API Key 201 - Resource Not Found 202 - Resource Blocked @@ -90,7 +91,9 @@ async function flushAuditLogs() { auditLogBuffer.unshift(...logsToWrite); logger.info(`Re-queued ${logsToWrite.length} audit logs for retry`); } else { - logger.error(`Buffer full, dropped ${logsToWrite.length} audit logs`); + logger.error( + `Buffer full, dropped ${logsToWrite.length} audit logs` + ); } } finally { isFlushInProgress = false; diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index b0aed6e30..ee69ce163 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -5,6 +5,10 @@ import { } from "@server/auth/sessions/resource"; import { generateSessionToken } from "@server/auth/sessions/app"; import { verifyResourceAccessToken } from "@server/auth/verifyResourceAccessToken"; +import { + extractVirtualApiKeyCredential, + verifyVirtualApiKey +} from "@server/auth/verifyVirtualApiKey"; import { getResourceByDomain, getResourceRules, @@ -127,7 +131,8 @@ export async function verifyResourceSession( // Extract HTTP Basic Auth credentials if present const clientHeaderAuth = extractBasicAuth(headers); - const clientUserAgent = headers?.["user-agent"] || headers?.["User-Agent"]; + const clientUserAgent = + headers?.["user-agent"] || headers?.["User-Agent"]; const clientIsBrowser = isBrowserUserAgent(clientUserAgent); const clientIp = requestIp @@ -254,20 +259,28 @@ export async function verifyResourceSession( ); if (action == "ACCEPT") { - logger.debug("Resource allowed by rule"); + // Public inference still requires a virtual API key; do not + // bypass that with an allow rule. + if (mode === "inference") { + logger.debug( + "Rule ACCEPT ignored for inference; continuing to virtual API key check" + ); + } else { + logger.debug("Resource allowed by rule"); - logRequestAudit( - { - action: true, - reason: 100, // allowed by rule - resourceId: resource.resourceId, - orgId: resource.orgId, - location: ipCC - }, - parsedBody.data - ); + logRequestAudit( + { + action: true, + reason: 100, // allowed by rule + resourceId: resource.resourceId, + orgId: resource.orgId, + location: ipCC + }, + parsedBody.data + ); - return allowed(res, undefined, dontStripSession); + return allowed(res, undefined, dontStripSession); + } } else if (action == "DROP") { logger.debug("Resource denied by rule"); @@ -302,20 +315,23 @@ export async function verifyResourceSession( !emailWhitelistEnabled && !headerAuth ) { - logger.debug("Resource allowed because no auth"); + // Public inference always requires a virtual API key. + if (mode !== "inference") { + logger.debug("Resource allowed because no auth"); - logRequestAudit( - { - action: true, - reason: 101, // allowed no auth - resourceId: resource.resourceId, - orgId: resource.orgId, - location: ipCC - }, - parsedBody.data - ); + logRequestAudit( + { + action: true, + reason: 101, // allowed no auth + resourceId: resource.resourceId, + orgId: resource.orgId, + location: ipCC + }, + parsedBody.data + ); - return allowed(res, undefined, dontStripSession); + return allowed(res, undefined, dontStripSession); + } } // Only offer a browser redirect to clients that can actually follow one and log in @@ -327,6 +343,82 @@ export async function verifyResourceSession( )}?redirect=${encodeURIComponent(originalRequestURL)}` : undefined; + // Virtual API keys for public inference resources (provider-style auth headers). + // Session/SSO may authenticate users elsewhere (e.g. dashboard key pages), but + // only a valid virtual API key is allowed through to the AI gateway. + if (mode === "inference") { + const vakCredential = extractVirtualApiKeyCredential(headers); + if (vakCredential) { + const { + valid, + error, + key, + userData: vakUserData + } = await verifyVirtualApiKey({ + credential: vakCredential, + resourceId: resource.resourceId, + orgId: resource.orgId + }); + + if (error) { + logger.debug("Virtual API key invalid: " + error); + } + + if (!valid) { + if (config.getRawConfig().app.log_failed_attempts) { + logger.info( + `Virtual API key is invalid. Resource ID: ${resource.resourceId}. IP: ${clientIp}.` + ); + } + } + + if (valid && key) { + logRequestAudit( + { + action: true, + reason: 109, // valid virtual API key + resourceId: resource.resourceId, + orgId: resource.orgId, + location: ipCC, + ...(vakUserData + ? { + user: { + username: vakUserData.username, + userId: vakUserData.userId + } + } + : { + apiKey: { + name: key.name, + apiKeyId: key.virtualApiKeyId + } + }), + metadata: { + virtualApiKeyId: key.virtualApiKeyId, + virtualApiKeyKind: key.kind + } + }, + parsedBody.data + ); + + return allowed(res, vakUserData, dontStripSession); + } + } + + logRequestAudit( + { + action: false, + reason: 299, // no more auth methods / VAK required + resourceId: resource.resourceId, + orgId: resource.orgId, + location: ipCC + }, + parsedBody.data + ); + + return notAllowed(res, redirectPath); + } + // check for access token in headers if ( headers && diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx index f9e44cb09..74f5552d0 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx @@ -38,7 +38,7 @@ import { authTypeRequiresApiKey, type AiProviderAuthType, type AiProviderType -} from "@server/lib/aiProviderDefaults"; +} from "@app/lib/aiProviderDefaults"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx index 6dabdbaad..9e41ff9c3 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx @@ -30,7 +30,7 @@ import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { zodResolver } from "@hookform/resolvers/zod"; -import { AI_CAPABILITIES, type AiCapability } from "@server/lib/aiCapabilities"; +import { AI_CAPABILITIES, type AiCapability } from "@app/lib/aiCapabilities"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx index 1aa25e9e5..e685bc560 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx @@ -49,7 +49,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import type { AiProviderAuthType, AiProviderType -} from "@server/lib/aiProviderDefaults"; +} from "@app/lib/aiProviderDefaults"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import { useQuery } from "@tanstack/react-query"; import type { AxiosResponse } from "axios"; diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 999782bc3..f5e407cc1 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -50,7 +50,7 @@ import { type AiProviderFormValues } from "@app/lib/aiProviderFormSchema"; import { zodResolver } from "@hookform/resolvers/zod"; -import { authTypeRequiresApiKey } from "@server/lib/aiProviderDefaults"; +import { authTypeRequiresApiKey } from "@app/lib/aiProviderDefaults"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; diff --git a/src/components/AiProviderAuthTypeSelect.tsx b/src/components/AiProviderAuthTypeSelect.tsx index a040c6fa1..460ec6a47 100644 --- a/src/components/AiProviderAuthTypeSelect.tsx +++ b/src/components/AiProviderAuthTypeSelect.tsx @@ -18,7 +18,7 @@ import { cn } from "@app/lib/cn"; import { AI_PROVIDER_AUTH_TYPES, type AiProviderAuthType -} from "@server/lib/aiProviderDefaults"; +} from "@app/lib/aiProviderDefaults"; import { CheckIcon, ChevronsUpDown } from "lucide-react"; import { useTranslations } from "next-intl"; import { useMemo, useState } from "react"; diff --git a/src/components/AiProviderCapabilitiesSelect.tsx b/src/components/AiProviderCapabilitiesSelect.tsx index 086b6f03e..ecb636472 100644 --- a/src/components/AiProviderCapabilitiesSelect.tsx +++ b/src/components/AiProviderCapabilitiesSelect.tsx @@ -1,7 +1,7 @@ "use client"; import { MultiSelectTagInput } from "@app/components/multi-select/multi-select-tag-input"; -import { AI_CAPABILITIES, type AiCapability } from "@server/lib/aiCapabilities"; +import { AI_CAPABILITIES, type AiCapability } from "@app/lib/aiCapabilities"; import { useTranslations } from "next-intl"; import { useMemo, useState } from "react"; diff --git a/src/components/AiProviderTypeSelect.tsx b/src/components/AiProviderTypeSelect.tsx index ed0565f53..b817d4169 100644 --- a/src/components/AiProviderTypeSelect.tsx +++ b/src/components/AiProviderTypeSelect.tsx @@ -16,7 +16,7 @@ import { } from "@app/components/ui/popover"; import { cn } from "@app/lib/cn"; import { aiProviderTypeValues } from "@app/lib/aiProviderFormSchema"; -import type { AiProviderType } from "@server/lib/aiProviderDefaults"; +import type { AiProviderType } from "@app/lib/aiProviderDefaults"; import { CheckIcon, ChevronsUpDown } from "lucide-react"; import { useTranslations } from "next-intl"; import { useMemo, useState } from "react"; diff --git a/src/components/CreateVirtualApiKeyForm.tsx b/src/components/CreateVirtualApiKeyForm.tsx index 5a3e88ce8..e67488b5d 100644 --- a/src/components/CreateVirtualApiKeyForm.tsx +++ b/src/components/CreateVirtualApiKeyForm.tsx @@ -42,6 +42,7 @@ import { Checkbox } from "@app/components/ui/checkbox"; import { useTranslations } from "next-intl"; import { UserSelector, type SelectedUser } from "@app/components/user-selector"; import type { CreateOrEditVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; +import { formatVirtualApiKeyCredential } from "@app/lib/virtualApiKeyFormat"; import { MultiResourcesSelector, formatMultiResourcesSelectorLabel @@ -146,7 +147,12 @@ export default function CreateVirtualApiKeyForm({ if (res?.data.data.virtualApiKey) { const key = res.data.data.virtualApiKey; if (key.secret) { - setCredential(`vk-${key.virtualApiKeyId}.${key.secret}`); + setCredential( + formatVirtualApiKeyCredential( + key.virtualApiKeyId, + key.secret + ) + ); } const resourceLookup = new Map( diff --git a/src/components/EditVirtualApiKeyForm.tsx b/src/components/EditVirtualApiKeyForm.tsx index 533de947c..0ef5e936b 100644 --- a/src/components/EditVirtualApiKeyForm.tsx +++ b/src/components/EditVirtualApiKeyForm.tsx @@ -40,6 +40,7 @@ import { Checkbox } from "@app/components/ui/checkbox"; import { useTranslations } from "next-intl"; import { UserSelector, type SelectedUser } from "@app/components/user-selector"; import type { CreateOrEditVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; +import { formatVirtualApiKeyCredential } from "@app/lib/virtualApiKeyFormat"; import { MultiResourcesSelector, formatMultiResourcesSelectorLabel @@ -151,7 +152,10 @@ export default function EditVirtualApiKeyForm({ const secret = res.data.data.virtualApiKey.secret; if (secret) { setCredential( - `vk-${virtualApiKey.virtualApiKeyId}.${secret}` + formatVirtualApiKeyCredential( + virtualApiKey.virtualApiKeyId, + secret + ) ); } else { toast({ diff --git a/src/components/UserVirtualApiKeys.tsx b/src/components/UserVirtualApiKeys.tsx index 5513b59e4..d90241d0b 100644 --- a/src/components/UserVirtualApiKeys.tsx +++ b/src/components/UserVirtualApiKeys.tsx @@ -27,6 +27,10 @@ import type { ListMyVirtualApiKeysResponse, VirtualApiKeyWithResources } from "@server/routers/virtualApiKey/types"; +import { + formatVirtualApiKeyCredential, + formatVirtualApiKeyPreview +} from "@app/lib/virtualApiKeyFormat"; type UserVirtualApiKeysProps = { orgId: string; @@ -34,10 +38,6 @@ type UserVirtualApiKeysProps = { initialData: ListMyVirtualApiKeysResponse; }; -function keyPreview(virtualApiKeyId: string, lastChars: string): string { - return `vk-${virtualApiKeyId}••••${lastChars}`; -} - function useRevealSecret(orgId: string, virtualApiKeyId: string) { const t = useTranslations(); const api = createApiClient(useEnvContext()); @@ -56,7 +56,9 @@ function useRevealSecret(orgId: string, virtualApiKeyId: string) { .then((res) => { const secret = res.data.data.virtualApiKey.secret; if (secret) { - setCredential(`vk-${virtualApiKeyId}.${secret}`); + setCredential( + formatVirtualApiKeyCredential(virtualApiKeyId, secret) + ); } else { toast({ variant: "destructive", @@ -95,7 +97,7 @@ function OwnedKeySecret({ lastChars: string; }) { const t = useTranslations(); - const preview = keyPreview(virtualApiKeyId, lastChars); + const preview = formatVirtualApiKeyPreview(virtualApiKeyId, lastChars); const { credential, loading, revealSecret } = useRevealSecret( orgId, virtualApiKeyId @@ -137,7 +139,7 @@ function IdentityKeyCenterpiece({ resourceGuid?: string; }) { const t = useTranslations(); - const preview = keyPreview(virtualApiKeyId, lastChars); + const preview = formatVirtualApiKeyPreview(virtualApiKeyId, lastChars); const { credential, loading, revealSecret } = useRevealSecret( orgId, virtualApiKeyId diff --git a/src/components/ViewVirtualApiKeySecret.tsx b/src/components/ViewVirtualApiKeySecret.tsx index 800fb3484..3e155053b 100644 --- a/src/components/ViewVirtualApiKeySecret.tsx +++ b/src/components/ViewVirtualApiKeySecret.tsx @@ -19,6 +19,7 @@ import { createApiClient, formatAxiosError } from "@app/lib/api"; import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import type { GetVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; +import { formatVirtualApiKeyCredential } from "@app/lib/virtualApiKeyFormat"; type ViewVirtualApiKeySecretProps = { open: boolean; @@ -56,7 +57,12 @@ export default function ViewVirtualApiKeySecret({ } const key = res.data.data.virtualApiKey; if (key.secret) { - setCredential(`vk-${key.virtualApiKeyId}.${key.secret}`); + setCredential( + formatVirtualApiKeyCredential( + key.virtualApiKeyId, + key.secret + ) + ); } else { toast({ variant: "destructive", diff --git a/src/components/VirtualApiKeysTable.tsx b/src/components/VirtualApiKeysTable.tsx index 074d82f74..1f5f8e4e8 100644 --- a/src/components/VirtualApiKeysTable.tsx +++ b/src/components/VirtualApiKeysTable.tsx @@ -46,6 +46,10 @@ import { import { cn } from "@app/lib/cn"; import { dataTableFilterPopoverContentClassName } from "@app/lib/dataTableFilterPopover"; import type { GetVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; +import { + formatVirtualApiKeyCredential, + formatVirtualApiKeyPreview +} from "@app/lib/virtualApiKeyFormat"; import { AxiosResponse } from "axios"; export type VirtualApiKeyRow = CreatedVirtualApiKey; @@ -507,7 +511,7 @@ function VirtualApiKeySecretCell({ }) { const t = useTranslations(); const api = createApiClient(useEnvContext()); - const preview = `vk-${virtualApiKeyId}••••${lastChars}`; + const preview = formatVirtualApiKeyPreview(virtualApiKeyId, lastChars); const [credential, setCredential] = useState(null); useEffect(() => { @@ -522,7 +526,9 @@ function VirtualApiKeySecretCell({ } const secret = res.data.data.virtualApiKey.secret; if (secret) { - setCredential(`vk-${virtualApiKeyId}.${secret}`); + setCredential( + formatVirtualApiKeyCredential(virtualApiKeyId, secret) + ); } }) .catch((e) => { diff --git a/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx b/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx index 8a71a07db..2434ed575 100644 --- a/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx +++ b/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx @@ -20,6 +20,10 @@ import type { GetMyVirtualApiKeyResponse, VirtualApiKeyWithResources } from "@server/routers/virtualApiKey/types"; +import { + formatVirtualApiKeyCredential, + formatVirtualApiKeyPreview +} from "@app/lib/virtualApiKeyFormat"; import { useQuery } from "@tanstack/react-query"; import type { AxiosResponse } from "axios"; import { Loader2 } from "lucide-react"; @@ -31,10 +35,6 @@ type LauncherInferenceApiKeysSectionProps = { resourceGuid: string; }; -function keyPreview(virtualApiKeyId: string, lastChars: string): string { - return `vk-${virtualApiKeyId}••••${lastChars}`; -} - function useRevealSecret(orgId: string, virtualApiKeyId: string) { const t = useTranslations(); const api = createApiClient(useEnvContext()); @@ -53,7 +53,9 @@ function useRevealSecret(orgId: string, virtualApiKeyId: string) { .then((res) => { const secret = res.data.data.virtualApiKey.secret; if (secret) { - setCredential(`vk-${virtualApiKeyId}.${secret}`); + setCredential( + formatVirtualApiKeyCredential(virtualApiKeyId, secret) + ); } else { toast({ variant: "destructive", @@ -92,7 +94,7 @@ function PanelKeySecret({ lastChars: string; }) { const t = useTranslations(); - const preview = keyPreview(virtualApiKeyId, lastChars); + const preview = formatVirtualApiKeyPreview(virtualApiKeyId, lastChars); const { credential, loading, revealSecret } = useRevealSecret( orgId, virtualApiKeyId diff --git a/src/components/user-selector.tsx b/src/components/user-selector.tsx index 58008f465..74258a165 100644 --- a/src/components/user-selector.tsx +++ b/src/components/user-selector.tsx @@ -46,7 +46,7 @@ export function UserSelector({ const [debouncedValue] = useDebounce(userSearchQuery, 150); const { data: users = [] } = useQuery( - orgQueries.users({ orgId, perPage: 10, term: debouncedValue }) + orgQueries.users({ orgId, perPage: 10, query: debouncedValue }) ); const usersShown = useMemo(() => { diff --git a/src/lib/aiCapabilities.ts b/src/lib/aiCapabilities.ts new file mode 100644 index 000000000..faa1f7af6 --- /dev/null +++ b/src/lib/aiCapabilities.ts @@ -0,0 +1,12 @@ +export const AI_CAPABILITIES = [ + "openai_chat", + "openai_responses", + "anthropic_messages", + "gemini_generate_content", + "bedrock_model_invoke", + "google_generate_content", + "google_raw_predict", + "bedrock_converse" +] as const; + +export type AiCapability = (typeof AI_CAPABILITIES)[number]; diff --git a/src/lib/aiProviderDefaults.ts b/src/lib/aiProviderDefaults.ts new file mode 100644 index 000000000..da4c43739 --- /dev/null +++ b/src/lib/aiProviderDefaults.ts @@ -0,0 +1,107 @@ +import { AI_CAPABILITIES, type AiCapability } from "@app/lib/aiCapabilities"; + +export type AiProviderType = + | "openai" + | "anthropic" + | "googleGemini" + | "vertexAi" + | "bedrock" + | "microsoftFoundry" + | "openRouter" + | "vercelAiGateway" + | "custom"; + +export const AI_PROVIDER_AUTH_TYPES = [ + "bearer", + "x-api-key", + "x-goog-api-key", + "hec", + "cf-aig-authorization", + "none", + "passthrough" +] as const; + +export type AiProviderAuthType = (typeof AI_PROVIDER_AUTH_TYPES)[number]; +export type AiBudgetUnit = "usd" | "tokens"; +export type AiProviderRoutingMode = "url" | "target"; + +type AiProviderDefaults = { + upstreamUrl: string | null; + authType: AiProviderAuthType; + capabilities: readonly AiCapability[]; +}; + +export const AI_PROVIDER_DEFAULTS: Record< + Exclude, + AiProviderDefaults +> = { + openai: { + upstreamUrl: "https://api.openai.com/v1", + authType: "bearer", + capabilities: ["openai_chat", "openai_responses"] + }, + anthropic: { + upstreamUrl: "https://api.anthropic.com", + authType: "x-api-key", + capabilities: ["anthropic_messages"] + }, + googleGemini: { + upstreamUrl: "https://generativelanguage.googleapis.com", + authType: "x-goog-api-key", + capabilities: ["gemini_generate_content"] + }, + vertexAi: { + upstreamUrl: null, + authType: "bearer", + capabilities: ["google_generate_content", "google_raw_predict"] + }, + bedrock: { + upstreamUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", + authType: "bearer", + capabilities: ["bedrock_converse"] + }, + microsoftFoundry: { + upstreamUrl: null, + authType: "bearer", + capabilities: ["openai_chat", "openai_responses", "anthropic_messages"] + }, + openRouter: { + upstreamUrl: "https://openrouter.ai/api/v1", + authType: "bearer", + capabilities: ["openai_chat"] + }, + vercelAiGateway: { + upstreamUrl: "https://ai-gateway.vercel.sh/v1", + authType: "bearer", + capabilities: ["openai_chat", "openai_responses"] + } +}; + +export function authTypeRequiresApiKey(authType: AiProviderAuthType): boolean { + return authType !== "none" && authType !== "passthrough"; +} + +export function providerRequiresUpstreamUrl( + type: AiProviderType, + routingMode: AiProviderRoutingMode = "url" +): boolean { + if (routingMode === "target") { + return false; + } + if (type === "custom") { + return true; + } + return AI_PROVIDER_DEFAULTS[type].upstreamUrl === null; +} + +export function defaultsForProviderType( + type: AiProviderType +): readonly AiCapability[] { + if (type === "custom") { + return []; + } + return AI_PROVIDER_DEFAULTS[type].capabilities; +} + +export { AI_CAPABILITIES }; +export type { AiCapability }; diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index 2f75c92ec..e033c0185 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -1,14 +1,15 @@ import { z } from "zod"; import { + AI_CAPABILITIES, AI_PROVIDER_AUTH_TYPES, AI_PROVIDER_DEFAULTS, authTypeRequiresApiKey, defaultsForProviderType, providerRequiresUpstreamUrl, + type AiCapability, type AiProviderAuthType, type AiProviderType -} from "@server/lib/aiProviderDefaults"; -import { AI_CAPABILITIES, type AiCapability } from "@server/lib/aiCapabilities"; +} from "@app/lib/aiProviderDefaults"; type TranslateFn = (key: string) => string; diff --git a/src/lib/virtualApiKeyFormat.ts b/src/lib/virtualApiKeyFormat.ts new file mode 100644 index 000000000..f09686c15 --- /dev/null +++ b/src/lib/virtualApiKeyFormat.ts @@ -0,0 +1,63 @@ +export const VIRTUAL_API_KEY_PREFIX = "vk-"; + +const VIRTUAL_API_KEY_AUTH_HEADER_NAMES = [ + "authorization", + "x-api-key", + "x-goog-api-key", + "cf-aig-authorization" +] as const; + +export function formatVirtualApiKeyCredential( + virtualApiKeyId: string, + secret: string +): string { + return `${VIRTUAL_API_KEY_PREFIX}${virtualApiKeyId}.${secret}`; +} + +export function formatVirtualApiKeyPreview( + virtualApiKeyId: string, + lastChars: string +): string { + return `${VIRTUAL_API_KEY_PREFIX}${virtualApiKeyId}••••${lastChars}`; +} + +export function looksLikeVirtualApiKeyCredential(value: string): boolean { + const trimmed = value.trim(); + if (!trimmed.startsWith(VIRTUAL_API_KEY_PREFIX)) { + return false; + } + const withoutPrefix = trimmed.slice(VIRTUAL_API_KEY_PREFIX.length); + const dot = withoutPrefix.indexOf("."); + return dot > 0 && dot < withoutPrefix.length - 1; +} + +function headerValueCarriesVirtualApiKey(raw: string): boolean { + const trimmed = raw.trim(); + const bearerMatch = trimmed.match(/^(?:Bearer|Splunk)\s+(.+)$/i); + if (bearerMatch) { + return looksLikeVirtualApiKeyCredential(bearerMatch[1]); + } + return looksLikeVirtualApiKeyCredential(trimmed); +} + +/** + * Remove client headers that carry a Pangolin virtual API key so they are + * never forwarded to upstream providers (including passthrough auth). + */ +export function stripVirtualApiKeyAuthHeaders( + headers: Record +): void { + for (const key of Object.keys(headers)) { + const lower = key.toLowerCase(); + if ( + !(VIRTUAL_API_KEY_AUTH_HEADER_NAMES as readonly string[]).includes( + lower + ) + ) { + continue; + } + if (headerValueCarriesVirtualApiKey(headers[key])) { + delete headers[key]; + } + } +} From 8a04f13dd4ef54c41fc7bc0fedc878d6488a0c0a Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 12 Aug 2026 10:37:19 -0400 Subject: [PATCH 134/296] translate keys and restructure roles and users --- messages/en-US.json | 56 +++++++++++++ .../[orgId]/settings/logs/ai-usage/page.tsx | 15 ++-- src/components/AiUsageAnalyticsData.tsx | 34 ++++---- .../ai-usage-analytics/OverviewTab.tsx | 78 ++++++++++--------- .../ai-usage-analytics/ProvidersTab.tsx | 24 +++--- .../ai-usage-analytics/ResourcesTab.tsx | 41 +++++----- .../ToggleableTrendChart.tsx | 48 +++++++----- .../ai-usage-analytics/TopEntitiesList.tsx | 18 +++-- .../ai-usage-analytics/UsersRolesTab.tsx | 66 +++++++++------- src/components/ai-usage-analytics/shared.ts | 5 +- src/components/ui/chart.tsx | 6 +- 11 files changed, 241 insertions(+), 150 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 166ca8a4b..bb1325ee9 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1936,6 +1936,62 @@ "aiResourceAliasRequired": "Alias is required for inference resources", "aiResourceDomainConfiguration": "Domain configuration", "aiResourceDomainConfigurationDescription": "Choose the domain clients will use to reach this inference resource.", + "aiUsageAnalyticsTitle": "AI Usage Analytics", + "aiUsageAnalyticsDescription": "Analyze AI gateway cost, token usage, and activity across providers, resources, roles, and users", + "aiUsageTabOverview": "Overview", + "aiUsageTabProviders": "Providers", + "aiUsageTabResources": "Resources", + "aiUsageTabUsersRoles": "Users & Roles", + "aiUsageFilterProvider": "Provider", + "aiUsageFilterModel": "Model", + "aiUsageFilterResource": "Resource", + "aiUsageFilterRole": "Role", + "aiUsageFilterUser": "User", + "aiUsageFilterAllProviders": "All Providers", + "aiUsageFilterAllModels": "All Models", + "aiUsageFilterAllResources": "All Resources", + "aiUsageFilterAllRoles": "All Roles", + "aiUsageFilterAllUsers": "All Users", + "aiUsageResetFilters": "Reset Filters", + "aiUsageRefresh": "Refresh", + "aiUsageTokenTypePrompt": "Prompt", + "aiUsageTokenTypeCacheRead": "Cache read", + "aiUsageTokenTypeCacheWrite": "Cache write", + "aiUsageTokenTypeCompletion": "Completion", + "aiUsageTokenTypeReasoning": "Reasoning", + "aiUsageRequests": "Requests", + "aiUsageCost": "Cost", + "aiUsageTokens": "Tokens", + "aiUsageOther": "Other", + "aiUsageTotalRequests": "Total Requests", + "aiUsageTotalTokens": "Total Tokens", + "aiUsageTotalCost": "Total Cost", + "aiUsageEstimated": "Estimated", + "aiUsageRequestVolume": "Request Volume", + "aiUsageTokenUsage": "Token Usage", + "aiUsageModelCost": "Model Cost", + "aiUsageModelTokens": "Model Tokens", + "aiUsageTopModels": "Top Models", + "aiUsageTopProviders": "Top Providers", + "aiUsageProviderCost": "Provider Cost", + "aiUsageProviderTokenUsage": "Provider Token Usage", + "aiUsageTopResources": "Top Resources", + "aiUsageResourceCost": "Resource Cost", + "aiUsageResourceTokenUsage": "Resource Token Usage", + "aiUsageResourceTypePublic": "Resource", + "aiUsageResourceTypeSite": "Site resource", + "aiUsageNoResource": "No resource", + "aiUsageRolesTab": "Roles", + "aiUsageUsersTab": "Users", + "aiUsageTopRoles": "Top Roles", + "aiUsageRoleCost": "Role Cost", + "aiUsageRoleTokenUsage": "Role Token Usage", + "aiUsageTopUsers": "Top Users", + "aiUsageUserCost": "User Cost", + "aiUsageUserTokenUsage": "User Token Usage", + "aiUsageUnknownUser": "Unknown user", + "aiUsageLoading": "Loading...", + "aiUsageNoData": "No data", "resourceBudgetSettings": "Budget", "resourceBudgetSettingsDescription": "Configure how this inference resource restricts usage based on spending or token limits", "sidebarApiKeys": "API Keys", diff --git a/src/app/[orgId]/settings/logs/ai-usage/page.tsx b/src/app/[orgId]/settings/logs/ai-usage/page.tsx index f045aead0..72e9cea71 100644 --- a/src/app/[orgId]/settings/logs/ai-usage/page.tsx +++ b/src/app/[orgId]/settings/logs/ai-usage/page.tsx @@ -1,10 +1,14 @@ import { AiUsageAnalyticsData } from "@app/components/AiUsageAnalyticsData"; import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; import type { Metadata } from "next"; +import { getTranslations } from "next-intl/server"; -export const metadata: Metadata = { - title: "AI Usage Analytics" -}; +export async function generateMetadata(): Promise { + const t = await getTranslations(); + return { + title: t("aiUsageAnalyticsTitle") + }; +} export interface AiUsageAnalyticsPageProps { params: Promise<{ orgId: string }>; @@ -14,12 +18,13 @@ export default async function AiUsageAnalyticsPage( props: AiUsageAnalyticsPageProps ) { const orgId = (await props.params).orgId; + const t = await getTranslations(); return ( <>
diff --git a/src/components/AiUsageAnalyticsData.tsx b/src/components/AiUsageAnalyticsData.tsx index 879aff1bc..748fc7cdf 100644 --- a/src/components/AiUsageAnalyticsData.tsx +++ b/src/components/AiUsageAnalyticsData.tsx @@ -8,6 +8,7 @@ import { } from "@app/lib/queries"; import { useIsFetching, useQuery, useQueryClient } from "@tanstack/react-query"; import { RefreshCw, XIcon } from "lucide-react"; +import { useTranslations } from "next-intl"; import { usePathname, useRouter, useSearchParams } from "next/navigation"; import { DateRangePicker, type DateTimeValue } from "./DateTimePicker"; import { Button } from "./ui/button"; @@ -35,6 +36,7 @@ export type AiUsageAnalyticsDataProps = { const AI_USAGE_ANALYTICS_QUERY_PREFIX = ["AI_USAGE_ANALYTICS"]; export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { + const t = useTranslations(); const searchParams = useSearchParams(); const path = usePathname(); const router = useRouter(); @@ -139,10 +141,10 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { })); const tabs: TabItem[] = [ - { title: "Overview", href: "#" }, - { title: "Provider Usage", href: "#" }, - { title: "Resources", href: "#" }, - { title: "Users & Roles", href: "#" } + { title: t("aiUsageTabOverview"), href: "#" }, + { title: t("aiUsageTabProviders"), href: "#" }, + { title: t("aiUsageTabResources"), href: "#" }, + { title: t("aiUsageTabUsersRoles"), href: "#" } ]; return ( @@ -172,42 +174,42 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) {
setFilter("providerId", v)} /> setFilter("model", v)} /> setFilter("resourceId", v)} /> setFilter("roleId", v)} /> setFilter("userId", v)} /> @@ -218,7 +220,7 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { className="gap-2" > - Reset Filters + {t("aiUsageResetFilters")} )}
@@ -243,7 +245,7 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { isFetching && "animate-spin" )} /> - Refresh + {t("aiUsageRefresh")}
diff --git a/src/components/ai-usage-analytics/OverviewTab.tsx b/src/components/ai-usage-analytics/OverviewTab.tsx index 1bf4dab1d..c539f11d9 100644 --- a/src/components/ai-usage-analytics/OverviewTab.tsx +++ b/src/components/ai-usage-analytics/OverviewTab.tsx @@ -3,6 +3,7 @@ import { useQuery } from "@tanstack/react-query"; import { aiUsageAnalyticsQueries } from "@app/lib/queries"; import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { useTranslations } from "next-intl"; import { Card, CardContent, CardHeader } from "@app/components/ui/card"; import { InfoSection, @@ -24,15 +25,8 @@ type OverviewTabProps = { filters: AiUsageAnalyticsFilters; }; -const TOKEN_TYPE_LABELS: Record = { - promptTokens: "Prompt", - cacheReadTokens: "Cache read", - cacheWriteTokens: "Cache write", - completionTokens: "Completion", - reasoningTokens: "Reasoning" -}; - export function OverviewTab(props: OverviewTabProps) { + const t = useTranslations(); const { data, isLoading } = useQuery( aiUsageAnalyticsQueries.overview({ orgId: props.orgId, @@ -40,8 +34,16 @@ export function OverviewTab(props: OverviewTabProps) { }) ); + const TOKEN_TYPE_LABELS: Record = { + promptTokens: t("aiUsageTokenTypePrompt"), + cacheReadTokens: t("aiUsageTokenTypeCacheRead"), + cacheWriteTokens: t("aiUsageTokenTypeCacheWrite"), + completionTokens: t("aiUsageTokenTypeCompletion"), + reasoningTokens: t("aiUsageTokenTypeReasoning") + }; + const requestsSeries = [ - { key: "requests", label: "Requests", color: SERIES_COLORS[0] } + { key: "requests", label: t("aiUsageRequests"), color: SERIES_COLORS[0] } ]; const tokensSeries = Object.keys(TOKEN_TYPE_LABELS).map((key, i) => ({ key, @@ -49,16 +51,18 @@ export function OverviewTab(props: OverviewTabProps) { color: SERIES_COLORS[i % SERIES_COLORS.length] })); const costSeries = [ - { key: "cost", label: "Cost", color: SERIES_COLORS[0] } + { key: "cost", label: t("aiUsageCost"), color: SERIES_COLORS[0] } ]; const modelCostSeries = buildSeriesFromData( data?.modelCostPerDay ?? [], - (key) => key + (key) => key, + t("aiUsageOther") ); const modelTokensSeries = buildSeriesFromData( data?.modelTokensPerDay ?? [], - (key) => key + (key) => key, + t("aiUsageOther") ); const topModels: TopEntity[] = (data?.topModels ?? []).map((m) => ({ @@ -75,7 +79,9 @@ export function OverviewTab(props: OverviewTabProps) { - Total Requests + + {t("aiUsageTotalRequests")} + {data ? compactNumberFormatter.format( @@ -85,7 +91,9 @@ export function OverviewTab(props: OverviewTabProps) { - Total Tokens + + {t("aiUsageTotalTokens")} + {data ? compactNumberFormatter.format( @@ -95,13 +103,17 @@ export function OverviewTab(props: OverviewTabProps) { - Total Cost + + {t("aiUsageTotalCost")} + {data ? formatCost(data.totalCost) : "--"} - Estimated + + {t("aiUsageEstimated")} + {data ? `${Math.round(data.estimatedPercent)}%` @@ -114,11 +126,9 @@ export function OverviewTab(props: OverviewTabProps) {
- -

Request Volume

- - + - -

Token Usage

- - + - -

Cost

- - + - -

Model Cost

- - + - -

Model Tokens

- - + -

Top Models

+

{t("aiUsageTopModels")}

 diff --git a/src/components/ai-usage-analytics/ProvidersTab.tsx b/src/components/ai-usage-analytics/ProvidersTab.tsx index 70756d558..e3b7a5d29 100644 --- a/src/components/ai-usage-analytics/ProvidersTab.tsx +++ b/src/components/ai-usage-analytics/ProvidersTab.tsx @@ -3,6 +3,7 @@ import { useQuery } from "@tanstack/react-query"; import { aiUsageAnalyticsQueries } from "@app/lib/queries"; import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { useTranslations } from "next-intl"; import { Card, CardContent, CardHeader } from "@app/components/ui/card"; import { ToggleableTrendChart } from "./ToggleableTrendChart"; import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; @@ -14,6 +15,7 @@ type ProvidersTabProps = { }; export function ProvidersTab(props: ProvidersTabProps) { + const t = useTranslations(); const { data, isLoading } = useQuery( aiUsageAnalyticsQueries.providers({ orgId: props.orgId, @@ -29,11 +31,13 @@ export function ProvidersTab(props: ProvidersTabProps) { const costSeries = buildSeriesFromData( data?.providerCostPerDay ?? [], - labelFor + labelFor, + t("aiUsageOther") ); const tokensSeries = buildSeriesFromData( data?.providerTokensPerDay ?? [], - labelFor + labelFor, + t("aiUsageOther") ); const topProviders: TopEntity[] = (data?.topProviders ?? []).map((p) => ({ @@ -48,24 +52,22 @@ export function ProvidersTab(props: ProvidersTabProps) {
-

Top Providers

+

{t("aiUsageTopProviders")}
- -

Provider Cost

- - + - -

Provider Token Usage

- - + - key === "none" ? "No resource" : (nameByKey.get(key) ?? key); + key === "none" ? t("aiUsageNoResource") : (nameByKey.get(key) ?? key); const costSeries = buildSeriesFromData( data?.resourceCostPerDay ?? [], - labelFor + labelFor, + t("aiUsageOther") ); const tokensSeries = buildSeriesFromData( data?.resourceTokensPerDay ?? [], - labelFor + labelFor, + t("aiUsageOther") ); const topResources: TopEntity[] = (data?.topResources ?? []).map((r) => ({ key: r.key, - label: r.name ?? (r.key === "none" ? "No resource" : r.key), + label: r.name ?? (r.key === "none" ? t("aiUsageNoResource") : r.key), sublabel: resourceTypeLabel(r.type), requests: r.requests, totalTokens: r.totalTokens, @@ -56,24 +61,22 @@ export function ResourcesTab(props: ResourcesTabProps) {
-

Top Resources

+

{t("aiUsageTopResources")}
- -

Resource Cost

- - + - -

Resource Token Usage

- - + ("bar"); const valueFormatter = props.valueFormatter ?? compactFormatter.format; @@ -61,25 +64,28 @@ export function ToggleableTrendChart(props: ToggleableTrendChartProps) { return (
-
- - +
+

{props.title}

+
+ + +
{!hasData ? ( @@ -87,10 +93,10 @@ export function ToggleableTrendChart(props: ToggleableTrendChartProps) { {props.isLoading ? ( <> - Loading... + {t("aiUsageLoading")} ) : ( - "No data" + t("aiUsageNoData") )}
) : ( diff --git a/src/components/ai-usage-analytics/TopEntitiesList.tsx b/src/components/ai-usage-analytics/TopEntitiesList.tsx index 9f712363f..6a445db6c 100644 --- a/src/components/ai-usage-analytics/TopEntitiesList.tsx +++ b/src/components/ai-usage-analytics/TopEntitiesList.tsx @@ -1,6 +1,7 @@ "use client"; import { LoaderIcon } from "lucide-react"; +import { useTranslations } from "next-intl"; import { compactNumberFormatter, formatCost } from "./shared"; export type TopEntity = { @@ -20,6 +21,7 @@ type TopEntitiesListProps = { }; export function TopEntitiesList(props: TopEntitiesListProps) { + const t = useTranslations(); const totalCost = props.entities.reduce( (sum, e) => sum + (e.costUsd ?? 0), 0 @@ -30,9 +32,15 @@ export function TopEntitiesList(props: TopEntitiesListProps) { {props.entities.length > 0 && (
{props.nameColumnLabel}
-
Requests
-
Tokens
-
Cost
+
+ {t("aiUsageRequests")} +
+
+ {t("aiUsageTokens")} +
+
+ {t("aiUsageCost")} +
%
)} @@ -42,10 +50,10 @@ export function TopEntitiesList(props: TopEntitiesListProps) { {props.isLoading ? ( <> - Loading... + {t("aiUsageLoading")} ) : ( - (props.emptyLabel ?? "No data") + (props.emptyLabel ?? t("aiUsageNoData")) )}
)} diff --git a/src/components/ai-usage-analytics/UsersRolesTab.tsx b/src/components/ai-usage-analytics/UsersRolesTab.tsx index 3d8d411f1..6e49ce9fb 100644 --- a/src/components/ai-usage-analytics/UsersRolesTab.tsx +++ b/src/components/ai-usage-analytics/UsersRolesTab.tsx @@ -3,7 +3,9 @@ import { useQuery } from "@tanstack/react-query"; import { aiUsageAnalyticsQueries } from "@app/lib/queries"; import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { useTranslations } from "next-intl"; import { Card, CardContent, CardHeader } from "@app/components/ui/card"; +import { HorizontalTabs, type TabItem } from "@app/components/HorizontalTabs"; import { ToggleableTrendChart } from "./ToggleableTrendChart"; import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; import { buildSeriesFromData, formatCost } from "./shared"; @@ -16,6 +18,7 @@ type UsersRolesTabProps = { const UNKNOWN_USER_KEY = "unknown"; export function UsersRolesTab(props: UsersRolesTabProps) { + const t = useTranslations(); const { data, isLoading } = useQuery( aiUsageAnalyticsQueries.usersRoles({ orgId: props.orgId, @@ -38,24 +41,28 @@ export function UsersRolesTab(props: UsersRolesTabProps) { } const userLabelFor = (key: string) => key === UNKNOWN_USER_KEY - ? "Unknown user" + ? t("aiUsageUnknownUser") : (userEmailByKey.get(key) ?? key); const roleCostSeries = buildSeriesFromData( data?.roleCostPerDay ?? [], - roleLabelFor + roleLabelFor, + t("aiUsageOther") ); const roleTokensSeries = buildSeriesFromData( data?.roleTokensPerDay ?? [], - roleLabelFor + roleLabelFor, + t("aiUsageOther") ); const userCostSeries = buildSeriesFromData( data?.userCostPerDay ?? [], - userLabelFor + userLabelFor, + t("aiUsageOther") ); const userTokensSeries = buildSeriesFromData( data?.userTokensPerDay ?? [], - userLabelFor + userLabelFor, + t("aiUsageOther") ); const topRoles: TopEntity[] = (data?.topRoles ?? []).map((r) => ({ @@ -68,35 +75,39 @@ export function UsersRolesTab(props: UsersRolesTabProps) { const topUsers: TopEntity[] = (data?.topUsers ?? []).map((u) => ({ key: u.userId ?? UNKNOWN_USER_KEY, - label: u.email ?? u.userId ?? "Unknown user", + label: u.email ?? u.userId ?? t("aiUsageUnknownUser"), requests: u.requests, totalTokens: u.totalTokens, costUsd: u.costUsd })); + const tabs: TabItem[] = [ + { title: t("aiUsageRolesTab"), href: "#" }, + { title: t("aiUsageUsersTab"), href: "#" } + ]; + return ( -
+
-

Roles

-

Top Roles

+

+ {t("aiUsageTopRoles")} +

- -

Role Cost

- - + - -

Role Token Usage

- - +
-

Users

-

Top Users

+

+ {t("aiUsageTopUsers")} +

- -

User Cost

- - + - -

User Token Usage

- - +
-
+ ); } diff --git a/src/components/ai-usage-analytics/shared.ts b/src/components/ai-usage-analytics/shared.ts index 1f551f8fa..30ad525b4 100644 --- a/src/components/ai-usage-analytics/shared.ts +++ b/src/components/ai-usage-analytics/shared.ts @@ -18,7 +18,8 @@ export const OTHER_KEY = "other"; // top-list needed. Assigns one categorical color per key, "other" last. export function buildSeriesFromData( data: Array>, - labelFor: (key: string) => string + labelFor: (key: string) => string, + otherLabel = "Other" ): TrendSeries[] { const keys = new Set(); for (const row of data) { @@ -37,7 +38,7 @@ export function buildSeriesFromData( const hasOther = data.some((row) => OTHER_KEY in row); if (hasOther) { - series.push({ key: OTHER_KEY, label: "Other", color: OTHER_COLOR }); + series.push({ key: OTHER_KEY, label: otherLabel, color: OTHER_COLOR }); } return series; diff --git a/src/components/ui/chart.tsx b/src/components/ui/chart.tsx index 83535e6cc..168393b29 100644 --- a/src/components/ui/chart.tsx +++ b/src/components/ui/chart.tsx @@ -57,7 +57,11 @@ const ChartContainer = React.forwardRef< {...props} > - + {children}
From 833b27ab4acc95c437404dfe982aa595748934ab Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 12 Aug 2026 10:40:29 -0400 Subject: [PATCH 135/296] separate the users and roles tabs --- messages/en-US.json | 1 - src/components/AiUsageAnalyticsData.tsx | 9 +- .../ai-usage-analytics/RolesTab.tsx | 91 +++++++++ .../ai-usage-analytics/UsersRolesTab.tsx | 172 ------------------ .../ai-usage-analytics/UsersTab.tsx | 97 ++++++++++ 5 files changed, 194 insertions(+), 176 deletions(-) create mode 100644 src/components/ai-usage-analytics/RolesTab.tsx delete mode 100644 src/components/ai-usage-analytics/UsersRolesTab.tsx create mode 100644 src/components/ai-usage-analytics/UsersTab.tsx diff --git a/messages/en-US.json b/messages/en-US.json index bb1325ee9..5da27a834 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1941,7 +1941,6 @@ "aiUsageTabOverview": "Overview", "aiUsageTabProviders": "Providers", "aiUsageTabResources": "Resources", - "aiUsageTabUsersRoles": "Users & Roles", "aiUsageFilterProvider": "Provider", "aiUsageFilterModel": "Model", "aiUsageFilterResource": "Resource", diff --git a/src/components/AiUsageAnalyticsData.tsx b/src/components/AiUsageAnalyticsData.tsx index 748fc7cdf..c19ca2346 100644 --- a/src/components/AiUsageAnalyticsData.tsx +++ b/src/components/AiUsageAnalyticsData.tsx @@ -27,7 +27,8 @@ import { HorizontalTabs, type TabItem } from "./HorizontalTabs"; import { OverviewTab } from "./ai-usage-analytics/OverviewTab"; import { ProvidersTab } from "./ai-usage-analytics/ProvidersTab"; import { ResourcesTab } from "./ai-usage-analytics/ResourcesTab"; -import { UsersRolesTab } from "./ai-usage-analytics/UsersRolesTab"; +import { RolesTab } from "./ai-usage-analytics/RolesTab"; +import { UsersTab } from "./ai-usage-analytics/UsersTab"; export type AiUsageAnalyticsDataProps = { orgId: string; @@ -144,7 +145,8 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { { title: t("aiUsageTabOverview"), href: "#" }, { title: t("aiUsageTabProviders"), href: "#" }, { title: t("aiUsageTabResources"), href: "#" }, - { title: t("aiUsageTabUsersRoles"), href: "#" } + { title: t("aiUsageRolesTab"), href: "#" }, + { title: t("aiUsageUsersTab"), href: "#" } ]; return ( @@ -255,7 +257,8 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { - + +
); diff --git a/src/components/ai-usage-analytics/RolesTab.tsx b/src/components/ai-usage-analytics/RolesTab.tsx new file mode 100644 index 000000000..0b60cab3c --- /dev/null +++ b/src/components/ai-usage-analytics/RolesTab.tsx @@ -0,0 +1,91 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { aiUsageAnalyticsQueries } from "@app/lib/queries"; +import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { useTranslations } from "next-intl"; +import { Card, CardContent, CardHeader } from "@app/components/ui/card"; +import { ToggleableTrendChart } from "./ToggleableTrendChart"; +import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; +import { buildSeriesFromData, formatCost } from "./shared"; + +type RolesTabProps = { + orgId: string; + filters: AiUsageAnalyticsFilters; +}; + +export function RolesTab(props: RolesTabProps) { + const t = useTranslations(); + const { data, isLoading } = useQuery( + aiUsageAnalyticsQueries.usersRoles({ + orgId: props.orgId, + filters: props.filters + }) + ); + + const roleNameByKey = new Map(); + for (const r of data?.topRoles ?? []) { + roleNameByKey.set(String(r.roleId), r.name ?? `Role #${r.roleId}`); + } + const roleLabelFor = (key: string) => + roleNameByKey.get(key) ?? `Role #${key}`; + + const roleCostSeries = buildSeriesFromData( + data?.roleCostPerDay ?? [], + roleLabelFor, + t("aiUsageOther") + ); + const roleTokensSeries = buildSeriesFromData( + data?.roleTokensPerDay ?? [], + roleLabelFor, + t("aiUsageOther") + ); + + const topRoles: TopEntity[] = (data?.topRoles ?? []).map((r) => ({ + key: String(r.roleId), + label: r.name ?? `Role #${r.roleId}`, + requests: r.requests, + totalTokens: r.totalTokens, + costUsd: r.costUsd + })); + + return ( +
+ + +

{t("aiUsageTopRoles")}

+ + + + + +
+ + + formatCost(v)} + /> + + + + + + + +
+
+ ); +} diff --git a/src/components/ai-usage-analytics/UsersRolesTab.tsx b/src/components/ai-usage-analytics/UsersRolesTab.tsx deleted file mode 100644 index 6e49ce9fb..000000000 --- a/src/components/ai-usage-analytics/UsersRolesTab.tsx +++ /dev/null @@ -1,172 +0,0 @@ -"use client"; - -import { useQuery } from "@tanstack/react-query"; -import { aiUsageAnalyticsQueries } from "@app/lib/queries"; -import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; -import { useTranslations } from "next-intl"; -import { Card, CardContent, CardHeader } from "@app/components/ui/card"; -import { HorizontalTabs, type TabItem } from "@app/components/HorizontalTabs"; -import { ToggleableTrendChart } from "./ToggleableTrendChart"; -import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; -import { buildSeriesFromData, formatCost } from "./shared"; - -type UsersRolesTabProps = { - orgId: string; - filters: AiUsageAnalyticsFilters; -}; - -const UNKNOWN_USER_KEY = "unknown"; - -export function UsersRolesTab(props: UsersRolesTabProps) { - const t = useTranslations(); - const { data, isLoading } = useQuery( - aiUsageAnalyticsQueries.usersRoles({ - orgId: props.orgId, - filters: props.filters - }) - ); - - const roleNameByKey = new Map(); - for (const r of data?.topRoles ?? []) { - roleNameByKey.set(String(r.roleId), r.name ?? `Role #${r.roleId}`); - } - const roleLabelFor = (key: string) => - roleNameByKey.get(key) ?? `Role #${key}`; - - const userEmailByKey = new Map(); - for (const u of data?.topUsers ?? []) { - if (u.userId) { - userEmailByKey.set(u.userId, u.email ?? u.userId); - } - } - const userLabelFor = (key: string) => - key === UNKNOWN_USER_KEY - ? t("aiUsageUnknownUser") - : (userEmailByKey.get(key) ?? key); - - const roleCostSeries = buildSeriesFromData( - data?.roleCostPerDay ?? [], - roleLabelFor, - t("aiUsageOther") - ); - const roleTokensSeries = buildSeriesFromData( - data?.roleTokensPerDay ?? [], - roleLabelFor, - t("aiUsageOther") - ); - const userCostSeries = buildSeriesFromData( - data?.userCostPerDay ?? [], - userLabelFor, - t("aiUsageOther") - ); - const userTokensSeries = buildSeriesFromData( - data?.userTokensPerDay ?? [], - userLabelFor, - t("aiUsageOther") - ); - - const topRoles: TopEntity[] = (data?.topRoles ?? []).map((r) => ({ - key: String(r.roleId), - label: r.name ?? `Role #${r.roleId}`, - requests: r.requests, - totalTokens: r.totalTokens, - costUsd: r.costUsd - })); - - const topUsers: TopEntity[] = (data?.topUsers ?? []).map((u) => ({ - key: u.userId ?? UNKNOWN_USER_KEY, - label: u.email ?? u.userId ?? t("aiUsageUnknownUser"), - requests: u.requests, - totalTokens: u.totalTokens, - costUsd: u.costUsd - })); - - const tabs: TabItem[] = [ - { title: t("aiUsageRolesTab"), href: "#" }, - { title: t("aiUsageUsersTab"), href: "#" } - ]; - - return ( - -
- - -

- {t("aiUsageTopRoles")} -

- - - - - -
- - - formatCost(v)} - /> - - - - - - - -
-
- -
- - -

- {t("aiUsageTopUsers")} -

- - - - - -
- - - formatCost(v)} - /> - - - - - - - -
-
- - ); -} diff --git a/src/components/ai-usage-analytics/UsersTab.tsx b/src/components/ai-usage-analytics/UsersTab.tsx new file mode 100644 index 000000000..25e914e3e --- /dev/null +++ b/src/components/ai-usage-analytics/UsersTab.tsx @@ -0,0 +1,97 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { aiUsageAnalyticsQueries } from "@app/lib/queries"; +import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { useTranslations } from "next-intl"; +import { Card, CardContent, CardHeader } from "@app/components/ui/card"; +import { ToggleableTrendChart } from "./ToggleableTrendChart"; +import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; +import { buildSeriesFromData, formatCost } from "./shared"; + +type UsersTabProps = { + orgId: string; + filters: AiUsageAnalyticsFilters; +}; + +const UNKNOWN_USER_KEY = "unknown"; + +export function UsersTab(props: UsersTabProps) { + const t = useTranslations(); + const { data, isLoading } = useQuery( + aiUsageAnalyticsQueries.usersRoles({ + orgId: props.orgId, + filters: props.filters + }) + ); + + const userEmailByKey = new Map(); + for (const u of data?.topUsers ?? []) { + if (u.userId) { + userEmailByKey.set(u.userId, u.email ?? u.userId); + } + } + const userLabelFor = (key: string) => + key === UNKNOWN_USER_KEY + ? t("aiUsageUnknownUser") + : (userEmailByKey.get(key) ?? key); + + const userCostSeries = buildSeriesFromData( + data?.userCostPerDay ?? [], + userLabelFor, + t("aiUsageOther") + ); + const userTokensSeries = buildSeriesFromData( + data?.userTokensPerDay ?? [], + userLabelFor, + t("aiUsageOther") + ); + + const topUsers: TopEntity[] = (data?.topUsers ?? []).map((u) => ({ + key: u.userId ?? UNKNOWN_USER_KEY, + label: u.email ?? u.userId ?? t("aiUsageUnknownUser"), + requests: u.requests, + totalTokens: u.totalTokens, + costUsd: u.costUsd + })); + + return ( +
+ + +

{t("aiUsageTopUsers")}

+ + + + + +
+ + + formatCost(v)} + /> + + + + + + + +
+
+ ); +} From 355294a2d4c8abb364d709ae8e5929422cd482b0 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 12 Aug 2026 10:48:06 -0400 Subject: [PATCH 136/296] fix logo warning in logs --- src/components/LayoutHeader.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/LayoutHeader.tsx b/src/components/LayoutHeader.tsx index ac265ff08..47eea0893 100644 --- a/src/components/LayoutHeader.tsx +++ b/src/components/LayoutHeader.tsx @@ -42,8 +42,8 @@ export function LayoutHeader({ const pathname = usePathname(); const logoWidth = isUnlocked() - ? env.branding.logo?.navbar?.width || 98 - : 98; + ? env.branding.logo?.navbar?.width || 128 + : 128; const logoHeight = isUnlocked() ? env.branding.logo?.navbar?.height || 32 : 32; From 379b53bcca69442db529cbc3de1f2794504422f1 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 12 Aug 2026 11:15:50 -0400 Subject: [PATCH 137/296] redirect to api key page when access gateway in browser --- server/routers/badger/verifySession.ts | 12 +++++++--- .../routers/resource/getResourceAuthInfo.ts | 4 +++- .../resource/[resourceGuid]/keys/page.tsx | 4 +++- src/app/auth/resource/[resourceGuid]/page.tsx | 23 +++++++++++++++++-- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index ee69ce163..b95b5925c 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -337,10 +337,16 @@ export async function verifyResourceSession( // Only offer a browser redirect to clients that can actually follow one and log in // (an interactive browser). Non-browser clients (curl, scripts, bots, etc.) just get // an unauthorized response from Badger instead of a login redirect URL. + // Inference browsers go to the dashboard keys page (not back to the inference host) + // so a valid session cannot create a redirect loop. const redirectPath = clientIsBrowser - ? `/auth/resource/${encodeURIComponent( - resource.resourceGuid - )}?redirect=${encodeURIComponent(originalRequestURL)}` + ? mode === "inference" + ? `/${resource.orgId}/resource/${encodeURIComponent( + resource.resourceGuid + )}/keys` + : `/auth/resource/${encodeURIComponent( + resource.resourceGuid + )}?redirect=${encodeURIComponent(originalRequestURL)}` : undefined; // Virtual API keys for public inference resources (provider-style auth headers). diff --git a/server/routers/resource/getResourceAuthInfo.ts b/server/routers/resource/getResourceAuthInfo.ts index f1328833d..667870959 100644 --- a/server/routers/resource/getResourceAuthInfo.ts +++ b/server/routers/resource/getResourceAuthInfo.ts @@ -42,6 +42,7 @@ export type GetResourceAuthInfoResponse = { skipToIdpId: number | null; orgId: string; postAuthPath: string | null; + mode: string; }; export async function getResourceAuthInfo( @@ -227,7 +228,8 @@ export async function getResourceAuthInfo( whitelist: effectivePolicy?.emailWhitelistEnabled ?? false, skipToIdpId: effectivePolicy?.idpId ?? resource.skipToIdpId, orgId: resource.orgId, - postAuthPath: resource.postAuthPath ?? null + postAuthPath: resource.postAuthPath ?? null, + mode: resource.mode }, success: true, error: false, diff --git a/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx b/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx index 29e11eb92..98d4b6582 100644 --- a/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx +++ b/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx @@ -41,7 +41,9 @@ export default async function ResourceKeysPage(props: ResourceKeysPageProps) { const user = await getUser(); if (!user) { - redirect("/"); + redirect( + `/auth/login?redirect=/${orgId}/resource/${resourceGuid}/keys` + ); } const cookieHeader = await authCookieHeader(); diff --git a/src/app/auth/resource/[resourceGuid]/page.tsx b/src/app/auth/resource/[resourceGuid]/page.tsx index dafc13f8f..4376bebc2 100644 --- a/src/app/auth/resource/[resourceGuid]/page.tsx +++ b/src/app/auth/resource/[resourceGuid]/page.tsx @@ -71,6 +71,9 @@ export default async function ResourceAuthPage(props: { ); } + const isInference = authInfo.mode === "inference"; + const keysPath = `/${authInfo.orgId}/resource/${authInfo.resourceGuid}/keys`; + const hasLoginPageDomain = await isOrgSubscribed( authInfo.orgId, tierMatrix.loginPageDomain @@ -159,10 +162,18 @@ export default async function ResourceAuthPage(props: { if (user && !user.emailVerified && env.flags.emailVerificationRequired) { redirect( - `/auth/verify-email?redirect=/auth/resource/${authInfo.resourceGuid}` + `/auth/verify-email?redirect=${encodeURIComponent( + isInference + ? keysPath + : `/auth/resource/${authInfo.resourceGuid}` + )}` ); } + if (isInference && !user) { + redirect(`/auth/login?redirect=${encodeURIComponent(keysPath)}`); + } + const cookie = await authCookieHeader(); // Check org policy compliance before proceeding @@ -181,7 +192,9 @@ export default async function ResourceAuthPage(props: { // If user is not compliant with org policies, show policy requirements if (orgPolicyCheck && !orgPolicyCheck.allowed && orgPolicyCheck.policies) { - const resourceAuthPageUrl = `/auth/resource/${authInfo.resourceGuid}${redirectUrl !== authInfo.url ? `?redirect=${encodeURIComponent(redirectUrl)}` : ""}`; + const resourceAuthPageUrl = isInference + ? keysPath + : `/auth/resource/${authInfo.resourceGuid}${redirectUrl !== authInfo.url ? `?redirect=${encodeURIComponent(redirectUrl)}` : ""}`; return (
Date: Wed, 12 Aug 2026 11:18:16 -0400 Subject: [PATCH 138/296] support bypass auth rules on gateway --- server/routers/badger/verifySession.ts | 32 ++++++++++---------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index b95b5925c..0b7cbeab0 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -259,28 +259,20 @@ export async function verifyResourceSession( ); if (action == "ACCEPT") { - // Public inference still requires a virtual API key; do not - // bypass that with an allow rule. - if (mode === "inference") { - logger.debug( - "Rule ACCEPT ignored for inference; continuing to virtual API key check" - ); - } else { - logger.debug("Resource allowed by rule"); + logger.debug("Resource allowed by rule"); - logRequestAudit( - { - action: true, - reason: 100, // allowed by rule - resourceId: resource.resourceId, - orgId: resource.orgId, - location: ipCC - }, - parsedBody.data - ); + logRequestAudit( + { + action: true, + reason: 100, // allowed by rule + resourceId: resource.resourceId, + orgId: resource.orgId, + location: ipCC + }, + parsedBody.data + ); - return allowed(res, undefined, dontStripSession); - } + return allowed(res, undefined, dontStripSession); } else if (action == "DROP") { logger.debug("Resource denied by rule"); From bed5817da3e7897c6488a358de63f33fa05eafa0 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 12 Aug 2026 11:20:25 -0400 Subject: [PATCH 139/296] send, process, store, display virtual api key ai information in usage and sessions --- messages/en-US.json | 10 + server/db/pg/schema/schema.ts | 30 ++- server/db/sqlite/schema/schema.ts | 30 ++- server/lib/aiBudgetEnforcement.ts | 36 ++- server/routers/aiGateway/logAiSession.ts | 4 + server/routers/aiGateway/pipeline.ts | 68 ++++-- server/routers/aiGateway/targetRouting.ts | 2 + .../auditLogs/aiUsageAnalyticsShared.ts | 6 +- server/routers/auditLogs/index.ts | 1 + server/routers/auditLogs/queryAiSessionLog.ts | 83 ++++++- .../auditLogs/queryAiUsageFilterOptions.ts | 73 +++++- .../auditLogs/queryAiUsageVirtualApiKeys.ts | 222 ++++++++++++++++++ server/routers/auditLogs/types.ts | 8 + server/routers/badger/verifySession.ts | 21 +- server/routers/external.ts | 7 + server/routers/integration.ts | 7 + src/app/[orgId]/settings/logs/ai/page.tsx | 103 +++++++- src/components/AiUsageAnalyticsData.tsx | 32 ++- .../ai-usage-analytics/VirtualApiKeysTab.tsx | 108 +++++++++ src/lib/queries.ts | 66 +++++- 20 files changed, 849 insertions(+), 68 deletions(-) create mode 100644 server/routers/auditLogs/queryAiUsageVirtualApiKeys.ts create mode 100644 src/components/ai-usage-analytics/VirtualApiKeysTab.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 5da27a834..43625ee73 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1989,6 +1989,14 @@ "aiUsageUserCost": "User Cost", "aiUsageUserTokenUsage": "User Token Usage", "aiUsageUnknownUser": "Unknown user", + "aiUsageVirtualApiKeysTab": "Virtual API Keys", + "aiUsageFilterVirtualApiKey": "Virtual API Key", + "aiUsageFilterAllVirtualApiKeys": "All Virtual API Keys", + "aiUsageTopVirtualApiKeys": "Top Virtual API Keys", + "aiUsageVirtualApiKeyCost": "Virtual API Key Cost", + "aiUsageVirtualApiKeyTokenUsage": "Virtual API Key Token Usage", + "aiUsageUnknownVirtualApiKey": "No virtual API key", + "aiUsageUnnamedVirtualApiKey": "Unnamed key", "aiUsageLoading": "Loading...", "aiUsageNoData": "No data", "resourceBudgetSettings": "Budget", @@ -3457,6 +3465,8 @@ "provider": "Provider", "capability": "Capability", "model": "Model", + "virtualApiKey": "Virtual API Key", + "noVirtualApiKey": "No virtual API key", "stream": "Stream", "streaming": "Streaming", "nonStreaming": "Non-streaming", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index afaccc480..a5ebfa530 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1819,18 +1819,22 @@ export const aiUsageRecords = pgTable( .references(() => orgs.orgId, { onDelete: "cascade" }), providerId: integer("providerId") .notNull() - .references(() => aiProviders.providerId, { onDelete: "cascade" }), + .references(() => aiProviders.providerId, { onDelete: "set null" }), resourceId: integer("resourceId").references( () => resources.resourceId, - { onDelete: "cascade" } + { onDelete: "set null" } ), siteResourceId: integer("siteResourceId").references( () => siteResources.siteResourceId, - { onDelete: "cascade" } + { onDelete: "set null" } ), userId: varchar("userId").references(() => users.userId, { onDelete: "set null" }), + virtualApiKeyId: varchar("virtualApiKeyId").references( + () => virtualApiKeys.virtualApiKeyId, + { onDelete: "set null" } + ), // Links this usage record back to the aiSessionLog row for the same // request (aiSessionLog.sessionId), so token/cost usage can be shown // alongside the session transcript. Not a DB-level FK - aiSessionLog @@ -1870,6 +1874,11 @@ export const aiUsageRecords = pgTable( t.userId, t.createdAt ), + index("idx_ai_usage_records_org_virtual_api_key_created").on( + t.orgId, + t.virtualApiKeyId, + t.createdAt + ), index("idx_ai_usage_records_session").on(t.sessionId) ] ); @@ -1927,19 +1936,23 @@ export const aiSessionLog = pgTable( }), providerId: integer("providerId") .notNull() - .references(() => aiProviders.providerId, { onDelete: "cascade" }), + .references(() => aiProviders.providerId, { onDelete: "set null" }), capability: varchar("capability").notNull(), resourceId: integer("resourceId").references( () => resources.resourceId, - { onDelete: "cascade" } + { onDelete: "set null" } ), siteResourceId: integer("siteResourceId").references( () => siteResources.siteResourceId, - { onDelete: "cascade" } + { onDelete: "set null" } ), userId: varchar("userId").references(() => users.userId, { onDelete: "set null" }), + virtualApiKeyId: varchar("virtualApiKeyId").references( + () => virtualApiKeys.virtualApiKeyId, + { onDelete: "set null" } + ), requestedModel: varchar("requestedModel"), isStream: boolean("isStream").notNull().default(false), requestBody: text("requestBody"), @@ -1979,6 +1992,11 @@ export const aiSessionLog = pgTable( t.userId, t.createdAt ), + index("idx_ai_session_log_org_virtual_api_key_created").on( + t.orgId, + t.virtualApiKeyId, + t.createdAt + ), index("idx_ai_session_log_session").on(t.sessionId) ] ); diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index bf3be9f87..450b99fce 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1807,18 +1807,22 @@ export const aiUsageRecords = sqliteTable( .references(() => orgs.orgId, { onDelete: "cascade" }), providerId: integer("providerId") .notNull() - .references(() => aiProviders.providerId, { onDelete: "cascade" }), + .references(() => aiProviders.providerId, { onDelete: "set null" }), resourceId: integer("resourceId").references( () => resources.resourceId, - { onDelete: "cascade" } + { onDelete: "set null" } ), siteResourceId: integer("siteResourceId").references( () => siteResources.siteResourceId, - { onDelete: "cascade" } + { onDelete: "set null" } ), userId: text("userId").references(() => users.userId, { onDelete: "set null" }), + virtualApiKeyId: text("virtualApiKeyId").references( + () => virtualApiKeys.virtualApiKeyId, + { onDelete: "set null" } + ), // Links this usage record back to the aiSessionLog row for the same // request (aiSessionLog.sessionId), so token/cost usage can be shown // alongside the session transcript. Not a DB-level FK - aiSessionLog @@ -1860,6 +1864,11 @@ export const aiUsageRecords = sqliteTable( t.userId, t.createdAt ), + index("idx_ai_usage_records_org_virtual_api_key_created").on( + t.orgId, + t.virtualApiKeyId, + t.createdAt + ), index("idx_ai_usage_records_session").on(t.sessionId) ] ); @@ -1917,19 +1926,23 @@ export const aiSessionLog = sqliteTable( }), providerId: integer("providerId") .notNull() - .references(() => aiProviders.providerId, { onDelete: "cascade" }), + .references(() => aiProviders.providerId, { onDelete: "set null" }), capability: text("capability").notNull(), resourceId: integer("resourceId").references( () => resources.resourceId, - { onDelete: "cascade" } + { onDelete: "set null" } ), siteResourceId: integer("siteResourceId").references( () => siteResources.siteResourceId, - { onDelete: "cascade" } + { onDelete: "set null" } ), userId: text("userId").references(() => users.userId, { onDelete: "set null" }), + virtualApiKeyId: text("virtualApiKeyId").references( + () => virtualApiKeys.virtualApiKeyId, + { onDelete: "set null" } + ), requestedModel: text("requestedModel"), isStream: integer("isStream", { mode: "boolean" }) .notNull() @@ -1973,6 +1986,11 @@ export const aiSessionLog = sqliteTable( t.userId, t.createdAt ), + index("idx_ai_session_log_org_virtual_api_key_created").on( + t.orgId, + t.virtualApiKeyId, + t.createdAt + ), index("idx_ai_session_log_session").on(t.sessionId) ] ); diff --git a/server/lib/aiBudgetEnforcement.ts b/server/lib/aiBudgetEnforcement.ts index d73f5ba59..f6daf9694 100644 --- a/server/lib/aiBudgetEnforcement.ts +++ b/server/lib/aiBudgetEnforcement.ts @@ -1,4 +1,14 @@ -import { and, eq, gte, inArray, isNull, or, sql, SQL, type InferInsertModel } from "drizzle-orm"; +import { + and, + eq, + gte, + inArray, + isNull, + or, + sql, + SQL, + type InferInsertModel +} from "drizzle-orm"; import { AiBudget, aiBudgetBreachEvents, @@ -424,6 +434,7 @@ export type UsageRecordInput = { resourceId: number | null; siteResourceId: number | null; userId: string | null; + virtualApiKeyId: string | null; requestedModel: string; usage: AiUsage; costUsd: number | null; @@ -460,7 +471,10 @@ async function flushUsageRecords() { isUsageFlushInProgress = true; - const recordsToWrite = usageRecordBuffer.splice(0, usageRecordBuffer.length); + const recordsToWrite = usageRecordBuffer.splice( + 0, + usageRecordBuffer.length + ); try { // Use a transaction to ensure all inserts succeed or fail together @@ -472,16 +486,25 @@ async function flushUsageRecords() { await tx.insert(aiUsageRecords).values(batch); } }); - logger.debug(`Flushed ${recordsToWrite.length} AI usage records to database`); + logger.debug( + `Flushed ${recordsToWrite.length} AI usage records to database` + ); } catch (error) { logger.error("Error flushing AI usage records:", error); // On transaction error, put records back at the front of the buffer // to retry, but only if the buffer isn't too large - if (usageRecordBuffer.length < USAGE_MAX_BUFFER_SIZE - recordsToWrite.length) { + if ( + usageRecordBuffer.length < + USAGE_MAX_BUFFER_SIZE - recordsToWrite.length + ) { usageRecordBuffer.unshift(...recordsToWrite); - logger.info(`Re-queued ${recordsToWrite.length} AI usage records for retry`); + logger.info( + `Re-queued ${recordsToWrite.length} AI usage records for retry` + ); } else { - logger.error(`Buffer full, dropped ${recordsToWrite.length} AI usage records`); + logger.error( + `Buffer full, dropped ${recordsToWrite.length} AI usage records` + ); } } finally { isUsageFlushInProgress = false; @@ -544,6 +567,7 @@ export async function recordUsage(input: UsageRecordInput): Promise { resourceId: input.resourceId, siteResourceId: input.siteResourceId, userId: input.userId, + virtualApiKeyId: input.virtualApiKeyId, sessionId: input.sessionId, requestedModel: input.requestedModel, promptTokens: usage.promptTokens, diff --git a/server/routers/aiGateway/logAiSession.ts b/server/routers/aiGateway/logAiSession.ts index ee05517f5..5a2e29bdb 100644 --- a/server/routers/aiGateway/logAiSession.ts +++ b/server/routers/aiGateway/logAiSession.ts @@ -180,6 +180,7 @@ export function logAiSession(data: { resourceId: number | null; siteResourceId: number | null; requestUserId: string | null; + virtualApiKeyId: string | null; }): void { (async () => { try { @@ -237,6 +238,9 @@ export function logAiSession(data: { resourceId: data.resourceId ?? undefined, siteResourceId: data.siteResourceId ?? undefined, userId: sanitizeString(data.requestUserId ?? undefined), + virtualApiKeyId: sanitizeString( + data.virtualApiKeyId ?? undefined + ), requestedModel: sanitizeString(data.requestedModel), isStream: data.isStream, requestBody: sanitizeString(requestBodyText.value), diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 5aa69d7fc..2be7a2592 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -161,6 +161,15 @@ export type RequestUser = { roleIds: number[]; }; +// Identity resolved for a gateway request: the app/session or virtual-API-key +// user (if any) plus the virtual API key that authenticated the request (if +// any) - a manual virtual API key with no associated user has a +// virtualApiKeyId but no user. +export type RequestIdentity = { + user: RequestUser | null; + virtualApiKeyId: string | null; +}; + // Identity headers forwarded to the upstream inference endpoint when the // requesting user is known. Omitted entirely (not sent empty) when we // couldn't resolve a user for the request. @@ -223,10 +232,12 @@ async function resolveRequestUser( req: Request, resourceId: number | null, orgId: string | null -): Promise { +): Promise { // Public inference: identity comes from Badger via Remote-* only when the // Traefik trust header proves the request passed verify-session (VAK). if (isAiGatewayTrustHeaderValid(req.headers as Record)) { + const virtualApiKeyId = + getRequestHeader(req, "remote-virtual-api-key-id") || null; const userId = getRequestHeader(req, "remote-user-id"); if (userId) { const username = getRequestHeader(req, "remote-user") || userId; @@ -236,32 +247,41 @@ async function resolveRequestUser( const orgRoles = orgId ? await getUserOrgRoles(userId, orgId) : []; return { - userId, - username, - email: email || null, - name: name || null, - role: - role || orgRoles.map((r) => r.roleName).join(", ") || null, - roleIds: orgRoles.map((r) => r.roleId) + user: { + userId, + username, + email: email || null, + name: name || null, + role: + role || + orgRoles.map((r) => r.roleName).join(", ") || + null, + roleIds: orgRoles.map((r) => r.roleId) + }, + virtualApiKeyId }; } - // Trusted request with no associated user (manual key without userId). + // Trusted request with no associated user (manual key without + // userId) - still attribute usage to the virtual API key itself. if (resourceId != null) { - return null; + return { user: null, virtualApiKeyId }; } } // Public inference must come through Badger; do not authorize via app session. if (resourceId != null) { - return null; + return { user: null, virtualApiKeyId: null }; } const sessionToken = req.cookies?.[SESSION_COOKIE_NAME]; if (sessionToken) { const { session, user } = await validateSessionToken(sessionToken); if (session && user) { - return buildRequestUser(user.userId, orgId); + return { + user: await buildRequestUser(user.userId, orgId), + virtualApiKeyId: null + }; } } @@ -269,7 +289,7 @@ async function resolveRequestUser( const ip = req.ip; if (!ip) { - return null; + return { user: null, virtualApiKeyId: null }; } const exitNodeRanges = await getExitNodeRanges(); @@ -277,15 +297,18 @@ async function resolveRequestUser( isIpInCidr(ip, range) ); if (!inExitNodeRange) { - return null; + return { user: null, virtualApiKeyId: null }; } const client = await findClientByIp(ip); if (!client || !client.userId) { - return null; + return { user: null, virtualApiKeyId: null }; } - return buildRequestUser(client.userId, orgId); + return { + user: await buildRequestUser(client.userId, orgId), + virtualApiKeyId: null + }; } function getRequestHeader(req: Request, name: string): string | undefined { @@ -608,6 +631,7 @@ export function recordAiGatewayCompletion(args: { resourceId: number | null; siteResourceId: number | null; requestUserId: string | null; + virtualApiKeyId: string | null; budgets: AiBudget[]; }): void { const { @@ -623,6 +647,7 @@ export function recordAiGatewayCompletion(args: { resourceId, siteResourceId, requestUserId, + virtualApiKeyId, budgets } = args; @@ -668,6 +693,7 @@ export function recordAiGatewayCompletion(args: { resourceId, siteResourceId, userId: requestUserId, + virtualApiKeyId, requestedModel: model ?? "unknown", usage, costUsd: cost?.totalCost ?? null, @@ -699,7 +725,8 @@ export function recordAiGatewayCompletion(args: { orgId, resourceId, siteResourceId, - requestUserId + requestUserId, + virtualApiKeyId }); } @@ -770,7 +797,7 @@ export async function handleAiGatewayProxy( const requestedModel = def.extractModel(req); - const [requestUser, selection] = await Promise.all([ + const [identity, selection] = await Promise.all([ resolveRequestUser(req, resourceId, orgId), selectProvider( capableAttachments, @@ -778,6 +805,7 @@ export async function handleAiGatewayProxy( requestedModel ) ]); + const requestUser = identity.user; if (requestUser) { logger.debug( @@ -836,7 +864,8 @@ export async function handleAiGatewayProxy( resourceId, siteResourceId, requestedModel, - budgets: appliedBudgets + budgets: appliedBudgets, + virtualApiKeyId: identity.virtualApiKeyId } ); } @@ -987,6 +1016,7 @@ export async function handleAiGatewayProxy( resourceId, siteResourceId, requestUserId: requestUser?.userId ?? null, + virtualApiKeyId: identity.virtualApiKeyId, budgets: appliedBudgets }); } diff --git a/server/routers/aiGateway/targetRouting.ts b/server/routers/aiGateway/targetRouting.ts index 9ffbf1d1b..8c4f244aa 100644 --- a/server/routers/aiGateway/targetRouting.ts +++ b/server/routers/aiGateway/targetRouting.ts @@ -179,6 +179,7 @@ export async function proxyAiGatewayToSiteTarget( siteResourceId: number | null; requestedModel: string | undefined; budgets: AiBudget[]; + virtualApiKeyId: string | null; } ): Promise { const providerTargets = await getProviderTargets(provider.providerId); @@ -319,6 +320,7 @@ export async function proxyAiGatewayToSiteTarget( resourceId: ctx.resourceId, siteResourceId: ctx.siteResourceId, requestUserId: requestUser?.userId ?? null, + virtualApiKeyId: ctx.virtualApiKeyId, budgets: ctx.budgets }); } diff --git a/server/routers/auditLogs/aiUsageAnalyticsShared.ts b/server/routers/auditLogs/aiUsageAnalyticsShared.ts index 95a6f825b..ada94a9b9 100644 --- a/server/routers/auditLogs/aiUsageAnalyticsShared.ts +++ b/server/routers/auditLogs/aiUsageAnalyticsShared.ts @@ -58,7 +58,8 @@ export const aiUsageAnalyticsFiltersQuery = z.object({ .transform(Number) .pipe(z.int().positive()) .optional(), - userId: z.string().optional() + userId: z.string().optional(), + virtualApiKeyId: z.string().optional() }); export const aiUsageAnalyticsParams = z.object({ @@ -114,6 +115,9 @@ export function buildAiUsageWhere( ) : undefined, data.userId ? eq(aiUsageRecords.userId, data.userId) : undefined, + data.virtualApiKeyId + ? eq(aiUsageRecords.virtualApiKeyId, data.virtualApiKeyId) + : undefined, roleUserIds ? inArray(aiUsageRecords.userId, roleUserIds) : undefined ); } diff --git a/server/routers/auditLogs/index.ts b/server/routers/auditLogs/index.ts index 12735d650..59969134f 100644 --- a/server/routers/auditLogs/index.ts +++ b/server/routers/auditLogs/index.ts @@ -8,3 +8,4 @@ export * from "./queryAiUsageOverview"; export * from "./queryAiUsageProviders"; export * from "./queryAiUsageResources"; export * from "./queryAiUsageUsersRoles"; +export * from "./queryAiUsageVirtualApiKeys"; diff --git a/server/routers/auditLogs/queryAiSessionLog.ts b/server/routers/auditLogs/queryAiSessionLog.ts index d61bdf5c8..437d74550 100644 --- a/server/routers/auditLogs/queryAiSessionLog.ts +++ b/server/routers/auditLogs/queryAiSessionLog.ts @@ -6,6 +6,7 @@ import { resources, siteResources, users, + virtualApiKeys, db, primaryDb } from "@server/db"; @@ -67,6 +68,7 @@ export const queryAiSessionLogsQuery = z.strictObject({ .pipe(z.int().positive()) .optional(), actor: z.string().optional(), + virtualApiKeyId: z.string().optional(), model: z.string().optional(), isStream: z .union([z.boolean(), z.string()]) @@ -117,7 +119,9 @@ function getWhere(data: Q) { data.providerId ? eq(aiSessionLog.providerId, data.providerId) : undefined, - data.capability ? eq(aiSessionLog.capability, data.capability) : undefined, + data.capability + ? eq(aiSessionLog.capability, data.capability) + : undefined, data.resourceId ? or( eq(aiSessionLog.resourceId, data.resourceId), @@ -125,9 +129,10 @@ function getWhere(data: Q) { ) : undefined, data.actor ? eq(aiSessionLog.userId, data.actor) : undefined, - data.model - ? eq(aiSessionLog.requestedModel, data.model) + data.virtualApiKeyId + ? eq(aiSessionLog.virtualApiKeyId, data.virtualApiKeyId) : undefined, + data.model ? eq(aiSessionLog.requestedModel, data.model) : undefined, data.isStream !== undefined ? eq(aiSessionLog.isStream, data.isStream) : undefined @@ -145,6 +150,7 @@ export function queryAiSession(data: Q) { resourceId: aiSessionLog.resourceId, siteResourceId: aiSessionLog.siteResourceId, userId: aiSessionLog.userId, + virtualApiKeyId: aiSessionLog.virtualApiKeyId, requestedModel: aiSessionLog.requestedModel, isStream: aiSessionLog.isStream, requestBody: aiSessionLog.requestBody, @@ -182,6 +188,14 @@ async function enrichWithDetails( ) ]; + const virtualApiKeyIds = [ + ...new Set( + logs + .map((log) => log.virtualApiKeyId) + .filter((id): id is string => id !== null && id !== undefined) + ) + ]; + const providerMap = new Map< number, { name: string | null; type: string | null } @@ -254,6 +268,28 @@ async function enrichWithDetails( } } + const virtualApiKeyMap = new Map< + string, + { name: string | null; lastChars: string } + >(); + if (virtualApiKeyIds.length > 0) { + const virtualApiKeyDetails = await db + .select({ + virtualApiKeyId: virtualApiKeys.virtualApiKeyId, + name: virtualApiKeys.name, + lastChars: virtualApiKeys.lastChars + }) + .from(virtualApiKeys) + .where(inArray(virtualApiKeys.virtualApiKeyId, virtualApiKeyIds)); + + for (const k of virtualApiKeyDetails) { + virtualApiKeyMap.set(k.virtualApiKeyId, { + name: k.name, + lastChars: k.lastChars + }); + } + } + const usageMap = new Map< string, { @@ -328,6 +364,12 @@ async function enrichWithDetails( resourceName, resourceNiceId, userEmail: log.userId ? (userMap.get(log.userId) ?? null) : null, + virtualApiKeyName: log.virtualApiKeyId + ? (virtualApiKeyMap.get(log.virtualApiKeyId)?.name ?? null) + : null, + virtualApiKeyLastChars: log.virtualApiKeyId + ? (virtualApiKeyMap.get(log.virtualApiKeyId)?.lastChars ?? null) + : null, usage: usageMap.get(log.sessionId) ?? null }; }); @@ -385,7 +427,8 @@ async function queryUniqueFilterAttributes( uniqueUsers, uniqueResources, uniqueSiteResources, - uniqueModels + uniqueModels, + uniqueVirtualApiKeys ] = await Promise.all([ logsDb .selectDistinct({ id: aiSessionLog.providerId }) @@ -411,6 +454,11 @@ async function queryUniqueFilterAttributes( .selectDistinct({ model: aiSessionLog.requestedModel }) .from(aiSessionLog) .where(baseConditions) + .limit(DISTINCT_LIMIT + 1), + logsDb + .selectDistinct({ id: aiSessionLog.virtualApiKeyId }) + .from(aiSessionLog) + .where(baseConditions) .limit(DISTINCT_LIMIT + 1) ]); @@ -498,10 +546,37 @@ async function queryUniqueFilterAttributes( ]; } + const virtualApiKeyIds = uniqueVirtualApiKeys + .map((row) => row.id) + .filter((id): id is string => id !== null); + + let virtualApiKeyList: Array<{ + id: string; + name: string | null; + lastChars: string | null; + }> = []; + if (virtualApiKeyIds.length > 0) { + const virtualApiKeyDetails = await primaryDb + .select({ + virtualApiKeyId: virtualApiKeys.virtualApiKeyId, + name: virtualApiKeys.name, + lastChars: virtualApiKeys.lastChars + }) + .from(virtualApiKeys) + .where(inArray(virtualApiKeys.virtualApiKeyId, virtualApiKeyIds)); + + virtualApiKeyList = virtualApiKeyDetails.map((k) => ({ + id: k.virtualApiKeyId, + name: k.name, + lastChars: k.lastChars + })); + } + return { providers: sortNamedFilterOptions(providers), resources: sortNamedFilterOptions(resourcesWithNames), users: userList, + virtualApiKeys: virtualApiKeyList, models: models.sort() }; } diff --git a/server/routers/auditLogs/queryAiUsageFilterOptions.ts b/server/routers/auditLogs/queryAiUsageFilterOptions.ts index 46cd3199e..6d7191187 100644 --- a/server/routers/auditLogs/queryAiUsageFilterOptions.ts +++ b/server/routers/auditLogs/queryAiUsageFilterOptions.ts @@ -6,7 +6,8 @@ import { siteResources, users, roles, - userOrgRoles + userOrgRoles, + virtualApiKeys } from "@server/db"; import { registry } from "@server/openApi"; import { NextFunction } from "express"; @@ -43,8 +44,9 @@ const queryAiUsageFilterOptionsParams = z.object({ orgId: z.string() }); -const queryAiUsageFilterOptionsCombined = - queryAiUsageFilterOptionsQuery.merge(queryAiUsageFilterOptionsParams); +const queryAiUsageFilterOptionsCombined = queryAiUsageFilterOptionsQuery.merge( + queryAiUsageFilterOptionsParams +); type Q = z.infer; function sortNamedFilterOptions( @@ -73,7 +75,8 @@ async function query(data: Q) { uniqueModels, uniqueResources, uniqueSiteResources, - uniqueUsers + uniqueUsers, + uniqueVirtualApiKeys ] = await Promise.all([ db .selectDistinct({ id: aiUsageRecords.providerId }) @@ -105,6 +108,13 @@ async function query(data: Q) { .selectDistinct({ userId: aiUsageRecords.userId }) .from(aiUsageRecords) .where(and(baseConditions, not(isNull(aiUsageRecords.userId)))) + .limit(DISTINCT_LIMIT + 1), + db + .selectDistinct({ id: aiUsageRecords.virtualApiKeyId }) + .from(aiUsageRecords) + .where( + and(baseConditions, not(isNull(aiUsageRecords.virtualApiKeyId))) + ) .limit(DISTINCT_LIMIT + 1) ]); @@ -120,11 +130,17 @@ async function query(data: Q) { let providers: Array<{ id: number; name: string | null }> = []; if (providerIds.length > 0) { const providerDetails = await db - .select({ providerId: aiProviders.providerId, name: aiProviders.name }) + .select({ + providerId: aiProviders.providerId, + name: aiProviders.name + }) .from(aiProviders) .where(inArray(aiProviders.providerId, providerIds)); - providers = providerDetails.map((p) => ({ id: p.providerId, name: p.name })); + providers = providerDetails.map((p) => ({ + id: p.providerId, + name: p.name + })); } const resourceIds = uniqueResources @@ -155,7 +171,10 @@ async function query(data: Q) { .where(inArray(siteResources.siteResourceId, siteResourceIds)); resourcesWithNames = resourcesWithNames.concat( - siteResourceDetails.map((r) => ({ id: r.siteResourceId, name: r.name })) + siteResourceDetails.map((r) => ({ + id: r.siteResourceId, + name: r.name + })) ); } @@ -190,11 +209,37 @@ async function query(data: Q) { roleList = [...roleMap.entries()].map(([id, name]) => ({ id, name })); } + const virtualApiKeyIds = uniqueVirtualApiKeys + .map((row) => row.id) + .filter((id): id is string => id !== null); + + let virtualApiKeyList: Array<{ + id: string; + name: string | null; + lastChars: string; + }> = []; + if (virtualApiKeyIds.length > 0) { + const virtualApiKeyDetails = await db + .select({ + virtualApiKeyId: virtualApiKeys.virtualApiKeyId, + name: virtualApiKeys.name, + lastChars: virtualApiKeys.lastChars + }) + .from(virtualApiKeys) + .where(inArray(virtualApiKeys.virtualApiKeyId, virtualApiKeyIds)); + virtualApiKeyList = virtualApiKeyDetails.map((k) => ({ + id: k.virtualApiKeyId, + name: k.name, + lastChars: k.lastChars + })); + } + return { providers: sortNamedFilterOptions(providers), resources: sortNamedFilterOptions(resourcesWithNames), roles: sortNamedFilterOptions(roleList), users: userList, + virtualApiKeys: virtualApiKeyList, models }; } @@ -227,7 +272,9 @@ registry.registerPath({ } }); -export type QueryAiUsageFilterOptionsResponse = Awaited>; +export type QueryAiUsageFilterOptionsResponse = Awaited< + ReturnType +>; export async function queryAiUsageFilterOptions( req: Request, @@ -238,7 +285,10 @@ export async function queryAiUsageFilterOptions( const parsedQuery = queryAiUsageFilterOptionsQuery.safeParse(req.query); if (!parsedQuery.success) { return next( - createHttpError(HttpCode.BAD_REQUEST, fromError(parsedQuery.error)) + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error) + ) ); } @@ -247,7 +297,10 @@ export async function queryAiUsageFilterOptions( ); if (!parsedParams.success) { return next( - createHttpError(HttpCode.BAD_REQUEST, fromError(parsedParams.error)) + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error) + ) ); } diff --git a/server/routers/auditLogs/queryAiUsageVirtualApiKeys.ts b/server/routers/auditLogs/queryAiUsageVirtualApiKeys.ts new file mode 100644 index 000000000..9d1b52ad2 --- /dev/null +++ b/server/routers/auditLogs/queryAiUsageVirtualApiKeys.ts @@ -0,0 +1,222 @@ +import { db, aiUsageRecords, virtualApiKeys } from "@server/db"; +import { registry } from "@server/openApi"; +import { NextFunction } from "express"; +import { Request, Response } from "express"; +import { count, desc, inArray, sql } from "drizzle-orm"; +import { OpenAPITags } from "@server/openApi"; +import { z } from "zod"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; +import { fromError } from "zod-validation-error"; +import response from "@server/lib/response"; +import logger from "@server/logger"; +import { + aiUsageAnalyticsFiltersQuery, + aiUsageAnalyticsParams, + buildAiUsageWhere, + resolveRoleUserIds, + dayBucketExpr, + pickTopNKeys, + bucketTopNPerDay, + DISTINCT_LIMIT, + type AiUsageAnalyticsQuery +} from "./aiUsageAnalyticsShared"; + +type Q = AiUsageAnalyticsQuery; + +const UNKNOWN_VIRTUAL_API_KEY_KEY = "unknown"; + +async function query(data: Q) { + const roleUserIds = await resolveRoleUserIds(data.orgId, data.roleId); + const baseConditions = buildAiUsageWhere(data, roleUserIds); + const dayExpr = dayBucketExpr(); + + const virtualApiKeyByDay = await db + .select({ + day: dayExpr.as("day"), + virtualApiKeyId: aiUsageRecords.virtualApiKeyId, + cost: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`, + tokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(dayExpr, aiUsageRecords.virtualApiKeyId) + .orderBy(dayExpr); + + const virtualApiKeyTotalsRaw = await db + .select({ + virtualApiKeyId: aiUsageRecords.virtualApiKeyId, + requests: count(), + totalTokens: sql`COALESCE(SUM(${aiUsageRecords.totalTokens}), 0)`, + costUsd: sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)` + }) + .from(aiUsageRecords) + .where(baseConditions) + .groupBy(aiUsageRecords.virtualApiKeyId) + .orderBy(desc(sql`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`)) + .limit(DISTINCT_LIMIT + 1); + + if (virtualApiKeyTotalsRaw.length > DISTINCT_LIMIT) { + throw createHttpError( + HttpCode.BAD_REQUEST, + "Too many distinct virtual API keys. Please narrow your query." + ); + } + + const virtualApiKeyIds = virtualApiKeyTotalsRaw + .map((r) => r.virtualApiKeyId) + .filter((id): id is string => id !== null); + + const detailsMap = new Map< + string, + { name: string | null; lastChars: string; kind: "user" | "manual" } + >(); + if (virtualApiKeyIds.length > 0) { + const details = await db + .select({ + virtualApiKeyId: virtualApiKeys.virtualApiKeyId, + name: virtualApiKeys.name, + lastChars: virtualApiKeys.lastChars, + kind: virtualApiKeys.kind + }) + .from(virtualApiKeys) + .where(inArray(virtualApiKeys.virtualApiKeyId, virtualApiKeyIds)); + for (const k of details) { + detailsMap.set(k.virtualApiKeyId, { + name: k.name, + lastChars: k.lastChars, + kind: k.kind + }); + } + } + + const topVirtualApiKeys = virtualApiKeyTotalsRaw.map((r) => { + const details = r.virtualApiKeyId + ? detailsMap.get(r.virtualApiKeyId) + : undefined; + return { + virtualApiKeyId: r.virtualApiKeyId, + name: details?.name ?? null, + lastChars: details?.lastChars ?? null, + kind: details?.kind ?? null, + requests: r.requests, + totalTokens: r.totalTokens, + costUsd: r.costUsd + }; + }); + + const virtualApiKeyCostTotals = new Map(); + const virtualApiKeyTokenTotals = new Map(); + for (const row of virtualApiKeyByDay) { + const key = row.virtualApiKeyId ?? UNKNOWN_VIRTUAL_API_KEY_KEY; + virtualApiKeyCostTotals.set( + key, + (virtualApiKeyCostTotals.get(key) ?? 0) + row.cost + ); + virtualApiKeyTokenTotals.set( + key, + (virtualApiKeyTokenTotals.get(key) ?? 0) + row.tokens + ); + } + const topVirtualApiKeysByCost = pickTopNKeys(virtualApiKeyCostTotals); + const topVirtualApiKeysByTokens = pickTopNKeys(virtualApiKeyTokenTotals); + + const virtualApiKeyCostPerDay = bucketTopNPerDay( + virtualApiKeyByDay.map((r) => ({ + day: r.day, + key: r.virtualApiKeyId ?? UNKNOWN_VIRTUAL_API_KEY_KEY, + value: r.cost + })), + topVirtualApiKeysByCost + ); + const virtualApiKeyTokensPerDay = bucketTopNPerDay( + virtualApiKeyByDay.map((r) => ({ + day: r.day, + key: r.virtualApiKeyId ?? UNKNOWN_VIRTUAL_API_KEY_KEY, + value: r.tokens + })), + topVirtualApiKeysByTokens + ); + + return { + topVirtualApiKeys, + virtualApiKeyCostPerDay, + virtualApiKeyTokensPerDay + }; +} + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/logs/ai/usage/virtual-api-keys", + description: + "Query the AI usage analytics virtual API key breakdown for an organization", + tags: [OpenAPITags.Logs], + request: { + query: aiUsageAnalyticsFiltersQuery, + params: aiUsageAnalyticsParams + }, + responses: { + 200: { + description: "Successful response", + content: { + "application/json": { + schema: z.object({ + data: z.record(z.string(), z.any()).nullable(), + success: z.boolean(), + error: z.boolean(), + message: z.string(), + status: z.number() + }) + } + } + } + } +}); + +export type QueryAiUsageVirtualApiKeysResponse = Awaited< + ReturnType +>; + +export async function queryAiUsageVirtualApiKeys( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = aiUsageAnalyticsFiltersQuery.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error) + ) + ); + } + + const parsedParams = aiUsageAnalyticsParams.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error) + ) + ); + } + + const data = await query({ ...parsedQuery.data, ...parsedParams.data }); + + return response(res, { + data, + success: true, + error: false, + message: + "AI usage virtual API key breakdown retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/auditLogs/types.ts b/server/routers/auditLogs/types.ts index b63acbf30..dc8d4015f 100644 --- a/server/routers/auditLogs/types.ts +++ b/server/routers/auditLogs/types.ts @@ -110,6 +110,9 @@ export type QueryAiSessionLogResponse = { resourceType: "public" | "site" | null; userId: string | null; userEmail: string | null; + virtualApiKeyId: string | null; + virtualApiKeyName: string | null; + virtualApiKeyLastChars: string | null; requestedModel: string | null; isStream: boolean; requestBody: string | null; @@ -148,6 +151,11 @@ export type QueryAiSessionLogResponse = { id: string; email: string | null; }[]; + virtualApiKeys: { + id: string; + name: string | null; + lastChars: string | null; + }[]; models: string[]; }; }; diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index ee69ce163..525128fc8 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -96,6 +96,10 @@ export type VerifyUserResponse = { userData?: BasicUserData; pangolinVersion?: string; dontStripSession?: boolean; + // Set independently of userData so a manual virtual API key with no + // associated user still gets attributed to the key that authenticated + // the request (see the mode === "inference" branch below). + virtualApiKeyId?: string; }; export async function verifyResourceSession( @@ -401,7 +405,12 @@ export async function verifyResourceSession( parsedBody.data ); - return allowed(res, vakUserData, dontStripSession); + return allowed( + res, + vakUserData, + dontStripSession, + key.virtualApiKeyId + ); } } @@ -987,16 +996,20 @@ async function notAllowed( function allowed( res: Response, userData?: BasicUserData, - dontStripSession?: boolean + dontStripSession?: boolean, + virtualApiKeyId?: string ) { const baseData = userData !== undefined && userData !== null ? { valid: true, ...userData, pangolinVersion: APP_VERSION } : { valid: true, pangolinVersion: APP_VERSION }; + const withVirtualApiKey = virtualApiKeyId + ? { ...baseData, virtualApiKeyId } + : baseData; const data = { data: dontStripSession - ? { ...baseData, dontStripSession: true } - : baseData, + ? { ...withVirtualApiKey, dontStripSession: true } + : withVirtualApiKey, success: true, error: false, message: "Access allowed", diff --git a/server/routers/external.ts b/server/routers/external.ts index b50de6bea..6f44c1efe 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1537,6 +1537,13 @@ authenticated.get( logs.queryAiUsageUsersRoles ); +authenticated.get( + "/org/:orgId/logs/ai/usage/virtual-api-keys", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageVirtualApiKeys +); + authenticated.get( "/org/:orgId/blueprints", verifyOrgAccess, diff --git a/server/routers/integration.ts b/server/routers/integration.ts index 02e1ab987..cc283990e 100644 --- a/server/routers/integration.ts +++ b/server/routers/integration.ts @@ -1580,6 +1580,13 @@ authenticated.get( logs.queryAiUsageUsersRoles ); +authenticated.get( + "/org/:orgId/logs/ai/usage/virtual-api-keys", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.viewLogs), + logs.queryAiUsageVirtualApiKeys +); + authenticated.get( "/org/:orgId/logs/analytics", verifyApiKeyOrgAccess, diff --git a/src/app/[orgId]/settings/logs/ai/page.tsx b/src/app/[orgId]/settings/logs/ai/page.tsx index 5d51c118b..15c573337 100644 --- a/src/app/[orgId]/settings/logs/ai/page.tsx +++ b/src/app/[orgId]/settings/logs/ai/page.tsx @@ -12,6 +12,7 @@ import { useTranslations } from "next-intl"; import { getSevenDaysAgo } from "@app/lib/getSevenDaysAgo"; import { getPrivateResourceSettingsHref } from "@app/lib/launcherResourceAdminHref"; import { logQueries } from "@app/lib/queries"; +import { formatVirtualApiKeyPreview } from "@app/lib/virtualApiKeyFormat"; import { ColumnDef } from "@tanstack/react-table"; import { useQuery } from "@tanstack/react-query"; import axios from "axios"; @@ -50,6 +51,7 @@ export default function AiSessionLogsPage() { capability?: string; resourceId?: string; actor?: string; + virtualApiKeyId?: string; model?: string; isStream?: string; }>({ @@ -57,6 +59,7 @@ export default function AiSessionLogsPage() { capability: searchParams.get("capability") || undefined, resourceId: searchParams.get("resourceId") || undefined, actor: searchParams.get("actor") || undefined, + virtualApiKeyId: searchParams.get("virtualApiKeyId") || undefined, model: searchParams.get("model") || undefined, isStream: searchParams.get("isStream") || undefined }); @@ -135,6 +138,7 @@ export default function AiSessionLogsPage() { providers: [], resources: [], users: [], + virtualApiKeys: [], models: [] }; @@ -358,7 +362,9 @@ export default function AiSessionLogsPage() { }, cell: ({ row }) => { if (!row.original.resourceNiceId) { - return -; + return ( + - + ); } return ( ); } + }, + { + accessorKey: "virtualApiKeyId", + header: ({ column }) => { + return ( +
+ ({ + value: key.id, + label: + key.name ?? + (key.lastChars + ? formatVirtualApiKeyPreview( + key.id, + key.lastChars + ) + : key.id) + }) + )} + selectedValue={filters.virtualApiKeyId} + onValueChange={(value) => + handleFilterChange("virtualApiKeyId", value) + } + label={t("virtualApiKey")} + searchPlaceholder={t("searchPlaceholder")} + emptyMessage={t("emptySearchOptions")} + /> +
+ ); + }, + cell: ({ row }) => { + if (!row.original.virtualApiKeyId) { + return ( + - + ); + } + return ( +
+ + {row.original.virtualApiKeyName ?? + t("aiUsageUnnamedVirtualApiKey")} + + {row.original.virtualApiKeyLastChars && ( + + {formatVirtualApiKeyPreview( + row.original.virtualApiKeyId, + row.original.virtualApiKeyLastChars + )} + + )} +
+ ); + } } ]; const renderExpandedRow = (row: any) => { return (
-
+
{t("aiSessionId")}

@@ -490,6 +550,30 @@ export default function AiSessionLogsPage() { : "N/A"}

+
+ {t("virtualApiKey")} +

+ {row.virtualApiKeyId ? ( + <> + {row.virtualApiKeyName ?? + t("aiUsageUnnamedVirtualApiKey")} + {row.virtualApiKeyLastChars && ( + <> + {" "} + ( + {formatVirtualApiKeyPreview( + row.virtualApiKeyId, + row.virtualApiKeyLastChars + )} + ) + + )} + + ) : ( + t("noVirtualApiKey") + )} +

+
{row.usage && (
@@ -599,15 +683,25 @@ function generateSampleAiSessionLogs(): QueryAiSessionLogResponse["log"] { ]; const actors = ["alice@example.com", "bob@example.com", null]; const models = ["gpt-4o", "claude-sonnet-5", "gemini-2.5-pro"]; + const virtualApiKeysSample = [ + { id: "vak00001", name: "CI pipeline", lastChars: "ab12" }, + { id: "vak00002", name: null, lastChars: "cd34" }, + null + ]; const now = Date.now(); const sevenDaysAgoMs = now - 7 * 24 * 60 * 60 * 1000; return Array.from({ length: 10 }, (_, i) => { - const provider = providers[Math.floor(Math.random() * providers.length)]; + const provider = + providers[Math.floor(Math.random() * providers.length)]; const resource = resourcesSample[Math.floor(Math.random() * resourcesSample.length)]; const actor = actors[Math.floor(Math.random() * actors.length)]; + const virtualApiKey = + virtualApiKeysSample[ + Math.floor(Math.random() * virtualApiKeysSample.length) + ]; return { id: i, @@ -625,6 +719,9 @@ function generateSampleAiSessionLogs(): QueryAiSessionLogResponse["log"] { resourceType: "public", userId: actor ? `user-${i}` : null, userEmail: actor, + virtualApiKeyId: virtualApiKey?.id ?? null, + virtualApiKeyName: virtualApiKey?.name ?? null, + virtualApiKeyLastChars: virtualApiKey?.lastChars ?? null, requestedModel: models[Math.floor(Math.random() * models.length)], isStream: Math.random() > 0.5, requestBody: null, diff --git a/src/components/AiUsageAnalyticsData.tsx b/src/components/AiUsageAnalyticsData.tsx index c19ca2346..a5d3db2b8 100644 --- a/src/components/AiUsageAnalyticsData.tsx +++ b/src/components/AiUsageAnalyticsData.tsx @@ -29,6 +29,8 @@ import { ProvidersTab } from "./ai-usage-analytics/ProvidersTab"; import { ResourcesTab } from "./ai-usage-analytics/ResourcesTab"; import { RolesTab } from "./ai-usage-analytics/RolesTab"; import { UsersTab } from "./ai-usage-analytics/UsersTab"; +import { VirtualApiKeysTab } from "./ai-usage-analytics/VirtualApiKeysTab"; +import { formatVirtualApiKeyPreview } from "@app/lib/virtualApiKeyFormat"; export type AiUsageAnalyticsDataProps = { orgId: string; @@ -140,13 +142,20 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { value: u.id, label: u.email ?? u.id })); + const virtualApiKeyOptions = (filterOptions?.virtualApiKeys ?? []).map( + (k) => ({ + value: k.id, + label: k.name ?? formatVirtualApiKeyPreview(k.id, k.lastChars) + }) + ); const tabs: TabItem[] = [ { title: t("aiUsageTabOverview"), href: "#" }, { title: t("aiUsageTabProviders"), href: "#" }, { title: t("aiUsageTabResources"), href: "#" }, { title: t("aiUsageRolesTab"), href: "#" }, - { title: t("aiUsageUsersTab"), href: "#" } + { title: t("aiUsageUsersTab"), href: "#" }, + { title: t("aiUsageVirtualApiKeysTab"), href: "#" } ]; return ( @@ -180,7 +189,9 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { value={filters.providerId?.toString()} options={providerOptions} placeholder={t("aiUsageFilterAllProviders")} - onValueChange={(v) => setFilter("providerId", v)} + onValueChange={(v) => + setFilter("providerId", v) + } /> setFilter("resourceId", v)} + onValueChange={(v) => + setFilter("resourceId", v) + } /> setFilter("userId", v)} /> + + setFilter("virtualApiKeyId", v) + } + /> {!isEmptySearchParams && (
); diff --git a/src/components/ai-usage-analytics/VirtualApiKeysTab.tsx b/src/components/ai-usage-analytics/VirtualApiKeysTab.tsx new file mode 100644 index 000000000..9ed8edb3d --- /dev/null +++ b/src/components/ai-usage-analytics/VirtualApiKeysTab.tsx @@ -0,0 +1,108 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { aiUsageAnalyticsQueries } from "@app/lib/queries"; +import type { AiUsageAnalyticsFilters } from "@app/lib/queries"; +import { formatVirtualApiKeyPreview } from "@app/lib/virtualApiKeyFormat"; +import { useTranslations } from "next-intl"; +import { Card, CardContent, CardHeader } from "@app/components/ui/card"; +import { ToggleableTrendChart } from "./ToggleableTrendChart"; +import { TopEntitiesList, type TopEntity } from "./TopEntitiesList"; +import { buildSeriesFromData, formatCost } from "./shared"; + +type VirtualApiKeysTabProps = { + orgId: string; + filters: AiUsageAnalyticsFilters; +}; + +const UNKNOWN_VIRTUAL_API_KEY_KEY = "unknown"; + +export function VirtualApiKeysTab(props: VirtualApiKeysTabProps) { + const t = useTranslations(); + const { data, isLoading } = useQuery( + aiUsageAnalyticsQueries.virtualApiKeys({ + orgId: props.orgId, + filters: props.filters + }) + ); + + const labelByKey = new Map(); + for (const k of data?.topVirtualApiKeys ?? []) { + if (k.virtualApiKeyId) { + labelByKey.set(k.virtualApiKeyId, k.name ?? k.virtualApiKeyId); + } + } + const virtualApiKeyLabelFor = (key: string) => + key === UNKNOWN_VIRTUAL_API_KEY_KEY + ? t("aiUsageUnknownVirtualApiKey") + : (labelByKey.get(key) ?? key); + + const virtualApiKeyCostSeries = buildSeriesFromData( + data?.virtualApiKeyCostPerDay ?? [], + virtualApiKeyLabelFor, + t("aiUsageOther") + ); + const virtualApiKeyTokensSeries = buildSeriesFromData( + data?.virtualApiKeyTokensPerDay ?? [], + virtualApiKeyLabelFor, + t("aiUsageOther") + ); + + const topVirtualApiKeys: TopEntity[] = (data?.topVirtualApiKeys ?? []).map( + (k) => ({ + key: k.virtualApiKeyId ?? UNKNOWN_VIRTUAL_API_KEY_KEY, + label: k.virtualApiKeyId + ? (k.name ?? t("aiUsageUnnamedVirtualApiKey")) + : t("aiUsageUnknownVirtualApiKey"), + sublabel: + k.virtualApiKeyId && k.lastChars + ? formatVirtualApiKeyPreview(k.virtualApiKeyId, k.lastChars) + : undefined, + requests: k.requests, + totalTokens: k.totalTokens, + costUsd: k.costUsd + }) + ); + + return ( +
+ + +

+ {t("aiUsageTopVirtualApiKeys")} +

+ + + + + +
+ + + formatCost(v)} + /> + + + + + + + +
+
+ ); +} diff --git a/src/lib/queries.ts b/src/lib/queries.ts index bf7f69b8f..56a41cee8 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -12,7 +12,8 @@ import type { QueryAiUsageOverviewResponse, QueryAiUsageProvidersResponse, QueryAiUsageResourcesResponse, - QueryAiUsageUsersRolesResponse + QueryAiUsageUsersRolesResponse, + QueryAiUsageVirtualApiKeysResponse } from "@server/routers/auditLogs"; import type { QueryAccessAuditLogResponse, @@ -954,7 +955,8 @@ export const aiUsageAnalyticsFiltersSchema = z.object({ model: z.string().optional().catch(undefined), resourceId: z.coerce.number().optional().catch(undefined), roleId: z.coerce.number().optional().catch(undefined), - userId: z.string().optional().catch(undefined) + userId: z.string().optional().catch(undefined), + virtualApiKeyId: z.string().optional().catch(undefined) }); export type AiUsageAnalyticsFilters = z.output< @@ -1086,6 +1088,7 @@ export const aiSessionLogsFiltersSchema = z.object({ capability: z.string().optional().catch(undefined), resourceId: z.string().optional().catch(undefined), actor: z.string().optional().catch(undefined), + virtualApiKeyId: z.string().optional().catch(undefined), model: z.string().optional().catch(undefined), isStream: z.string().optional().catch(undefined) }); @@ -1284,7 +1287,12 @@ export const aiUsageAnalyticsQueries = { filters: Pick; }) => queryOptions({ - queryKey: ["AI_USAGE_ANALYTICS", orgId, "FILTERS", filters] as const, + queryKey: [ + "AI_USAGE_ANALYTICS", + orgId, + "FILTERS", + filters + ] as const, queryFn: async ({ signal, meta }) => { const res = await meta!.api.get< AxiosResponse @@ -1304,7 +1312,12 @@ export const aiUsageAnalyticsQueries = { filters: AiUsageAnalyticsFilters; }) => queryOptions({ - queryKey: ["AI_USAGE_ANALYTICS", orgId, "OVERVIEW", filters] as const, + queryKey: [ + "AI_USAGE_ANALYTICS", + orgId, + "OVERVIEW", + filters + ] as const, queryFn: async ({ signal, meta }) => { const res = await meta!.api.get< AxiosResponse @@ -1330,7 +1343,12 @@ export const aiUsageAnalyticsQueries = { filters: AiUsageAnalyticsFilters; }) => queryOptions({ - queryKey: ["AI_USAGE_ANALYTICS", orgId, "PROVIDERS", filters] as const, + queryKey: [ + "AI_USAGE_ANALYTICS", + orgId, + "PROVIDERS", + filters + ] as const, queryFn: async ({ signal, meta }) => { const res = await meta!.api.get< AxiosResponse @@ -1356,7 +1374,12 @@ export const aiUsageAnalyticsQueries = { filters: AiUsageAnalyticsFilters; }) => queryOptions({ - queryKey: ["AI_USAGE_ANALYTICS", orgId, "RESOURCES", filters] as const, + queryKey: [ + "AI_USAGE_ANALYTICS", + orgId, + "RESOURCES", + filters + ] as const, queryFn: async ({ signal, meta }) => { const res = await meta!.api.get< AxiosResponse @@ -1403,6 +1426,37 @@ export const aiUsageAnalyticsQueries = { } return false; } + }), + + virtualApiKeys: ({ + orgId, + filters + }: { + orgId: string; + filters: AiUsageAnalyticsFilters; + }) => + queryOptions({ + queryKey: [ + "AI_USAGE_ANALYTICS", + orgId, + "VIRTUAL_API_KEYS", + filters + ] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/org/${orgId}/logs/ai/usage/virtual-api-keys`, { + params: filters, + signal + }); + return res.data.data; + }, + refetchInterval: (query) => { + if (query.state.data) { + return durationToMs(30, "seconds"); + } + return false; + } }) }; From 115c3cbf077bb8be67c436bdaef08441e0c6aa57 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 12 Aug 2026 11:32:22 -0400 Subject: [PATCH 140/296] show sso page in browser for gateway resource --- messages/en-US.json | 10 +++--- server/routers/badger/verifySession.ts | 14 +++------ .../virtualApiKey/listMyVirtualApiKeys.ts | 8 +++-- server/routers/virtualApiKey/types.ts | 1 + .../resource/[resourceGuid]/keys/page.tsx | 10 ++---- src/app/auth/resource/[resourceGuid]/page.tsx | 22 ++++++------- src/components/UserVirtualApiKeys.tsx | 31 +++++++------------ .../LauncherInferenceApiKeysSection.tsx | 7 ++++- 8 files changed, 45 insertions(+), 58 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 5da27a834..abf67bb7a 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1699,16 +1699,16 @@ "virtualApiKeysFilterUnassigned": "Unassigned", "myVirtualApiKeysTitle": "Your API Keys", "myVirtualApiKeysDescription": "View your identity key and any virtual API keys attributed to you in this organization", - "myVirtualApiKeysResourceTitle": "Your API Keys for This Resource", - "myVirtualApiKeysResourceDescription": "View your identity key and virtual API keys attributed to you that can access this resource", + "myVirtualApiKeysResourceTitle": "Your API Keys for {resourceName}", + "myVirtualApiKeysResourceDescription": "View your identity key and virtual API keys attributed to you that can access {resourceName}", "myVirtualApiKeysIdentityTitle": "Identity Key", "myVirtualApiKeysIdentityHeadline": "Your Personal API Key", "myVirtualApiKeysIdentityDescription": "Your personal key for this organization. It is unique to your account and used to identify you when calling AI Gateway resources.", - "myVirtualApiKeysIdentityResourceHeadline": "Your Personal API Key for This Resource", - "myVirtualApiKeysIdentityResourceDescription": "Your personal key for this organization. Use it to call this AI Gateway resource.", + "myVirtualApiKeysIdentityResourceHeadline": "Your Personal API Key for {resourceName}", + "myVirtualApiKeysIdentityResourceDescription": "Your personal key for this organization. Use it to call {resourceName}.", "myVirtualApiKeysManualTitle": "Attributed Keys", "myVirtualApiKeysManualDescription": "Manual virtual API keys an admin associated with your account", - "myVirtualApiKeysManualResourceDescription": "Manual virtual API keys associated with your account that can access this resource", + "myVirtualApiKeysManualResourceDescription": "Manual virtual API keys associated with your account that can access {resourceName}", "myVirtualApiKeysManualEmpty": "No attributed keys yet", "myVirtualApiKeysKindUser": "Identity", "myVirtualApiKeysKindManual": "Manual", diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index 0b7cbeab0..f5b6a25b9 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -329,16 +329,10 @@ export async function verifyResourceSession( // Only offer a browser redirect to clients that can actually follow one and log in // (an interactive browser). Non-browser clients (curl, scripts, bots, etc.) just get // an unauthorized response from Badger instead of a login redirect URL. - // Inference browsers go to the dashboard keys page (not back to the inference host) - // so a valid session cannot create a redirect loop. const redirectPath = clientIsBrowser - ? mode === "inference" - ? `/${resource.orgId}/resource/${encodeURIComponent( - resource.resourceGuid - )}/keys` - : `/auth/resource/${encodeURIComponent( - resource.resourceGuid - )}?redirect=${encodeURIComponent(originalRequestURL)}` + ? `/auth/resource/${encodeURIComponent( + resource.resourceGuid + )}?redirect=${encodeURIComponent(originalRequestURL)}` : undefined; // Virtual API keys for public inference resources (provider-style auth headers). @@ -414,7 +408,7 @@ export async function verifyResourceSession( parsedBody.data ); - return notAllowed(res, redirectPath); + return notAllowed(res, redirectPath, resource.orgId); } // check for access token in headers diff --git a/server/routers/virtualApiKey/listMyVirtualApiKeys.ts b/server/routers/virtualApiKey/listMyVirtualApiKeys.ts index 244377ef1..5177a975a 100644 --- a/server/routers/virtualApiKey/listMyVirtualApiKeys.ts +++ b/server/routers/virtualApiKey/listMyVirtualApiKeys.ts @@ -123,10 +123,12 @@ export async function listMyVirtualApiKeys( } let resourceId: number | undefined; + let resourceName: string | undefined; if (resourceGuid) { const [resource] = await db .select({ - resourceId: resources.resourceId + resourceId: resources.resourceId, + name: resources.name }) .from(resources) .where( @@ -147,6 +149,7 @@ export async function listMyVirtualApiKeys( } resourceId = resource.resourceId; + resourceName = resource.name; } const { key: userKeyRow } = await getOrCreateUserVirtualApiKey({ @@ -203,7 +206,8 @@ export async function listMyVirtualApiKeys( userKey: toKeyWithResources(userKeyRow, resourceIdsByKey), manualKeys: manualRows.map((row) => toKeyWithResources(row, resourceIdsByKey) - ) + ), + ...(resourceName !== undefined ? { resourceName } : {}) }, success: true, error: false, diff --git a/server/routers/virtualApiKey/types.ts b/server/routers/virtualApiKey/types.ts index 73e475632..91bf17c0a 100644 --- a/server/routers/virtualApiKey/types.ts +++ b/server/routers/virtualApiKey/types.ts @@ -22,6 +22,7 @@ export type CreateOrEditVirtualApiKeyResponse = { export type ListMyVirtualApiKeysResponse = { userKey: VirtualApiKeyWithResources; manualKeys: VirtualApiKeyWithResources[]; + resourceName?: string | null; }; export type GetMyVirtualApiKeyResponse = { diff --git a/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx b/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx index 98d4b6582..dd673ac85 100644 --- a/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx +++ b/src/app/[orgId]/resource/[resourceGuid]/keys/page.tsx @@ -18,7 +18,7 @@ import { cache } from "react"; export async function generateMetadata(): Promise { const t = await getTranslations(); return { - title: t("myVirtualApiKeysResourceTitle") + title: t("myVirtualApiKeysTitle") }; } @@ -42,7 +42,7 @@ export default async function ResourceKeysPage(props: ResourceKeysPageProps) { if (!user) { redirect( - `/auth/login?redirect=/${orgId}/resource/${resourceGuid}/keys` + `/auth/resource/${encodeURIComponent(resourceGuid)}?redirect=${encodeURIComponent(`/${orgId}/resource/${resourceGuid}/keys`)}` ); } @@ -113,11 +113,7 @@ export default async function ResourceKeysPage(props: ResourceKeysPageProps) { launcherMode showViewAsAdmin={isAdminOrOwner} > - + ); diff --git a/src/app/auth/resource/[resourceGuid]/page.tsx b/src/app/auth/resource/[resourceGuid]/page.tsx index 4376bebc2..3c4fd2691 100644 --- a/src/app/auth/resource/[resourceGuid]/page.tsx +++ b/src/app/auth/resource/[resourceGuid]/page.tsx @@ -163,17 +163,11 @@ export default async function ResourceAuthPage(props: { if (user && !user.emailVerified && env.flags.emailVerificationRequired) { redirect( `/auth/verify-email?redirect=${encodeURIComponent( - isInference - ? keysPath - : `/auth/resource/${authInfo.resourceGuid}` + `/auth/resource/${authInfo.resourceGuid}` )}` ); } - if (isInference && !user) { - redirect(`/auth/login?redirect=${encodeURIComponent(keysPath)}`); - } - const cookie = await authCookieHeader(); // Check org policy compliance before proceeding @@ -192,9 +186,7 @@ export default async function ResourceAuthPage(props: { // If user is not compliant with org policies, show policy requirements if (orgPolicyCheck && !orgPolicyCheck.allowed && orgPolicyCheck.policies) { - const resourceAuthPageUrl = isInference - ? keysPath - : `/auth/resource/${authInfo.resourceGuid}${redirectUrl !== authInfo.url ? `?redirect=${encodeURIComponent(redirectUrl)}` : ""}`; + const resourceAuthPageUrl = `/auth/resource/${authInfo.resourceGuid}${redirectUrl !== authInfo.url ? `?redirect=${encodeURIComponent(redirectUrl)}` : ""}`; return (
); @@ -334,7 +330,7 @@ export default async function ResourceAuthPage(props: { name: authInfo.resourceName, id: authInfo.resourceId }} - redirect={redirectUrl} + redirect={postAuthRedirect} idps={loginIdps} orgId={build === "saas" ? authInfo.orgId : undefined} branding={ diff --git a/src/components/UserVirtualApiKeys.tsx b/src/components/UserVirtualApiKeys.tsx index d90241d0b..578f007b7 100644 --- a/src/components/UserVirtualApiKeys.tsx +++ b/src/components/UserVirtualApiKeys.tsx @@ -4,11 +4,9 @@ import { useState } from "react"; import { useTranslations } from "next-intl"; import { AxiosResponse } from "axios"; import moment from "moment"; -import { Badge } from "@app/components/ui/badge"; import { Button } from "@app/components/ui/button"; import CopyTextBox from "@app/components/CopyTextBox"; import CopyToClipboard from "@app/components/CopyToClipboard"; -import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; import { SettingsContainer, SettingsFormCell, @@ -34,7 +32,6 @@ import { type UserVirtualApiKeysProps = { orgId: string; - resourceGuid?: string; initialData: ListMyVirtualApiKeysResponse; }; @@ -131,12 +128,12 @@ function IdentityKeyCenterpiece({ orgId, virtualApiKeyId, lastChars, - resourceGuid + resourceName }: { orgId: string; virtualApiKeyId: string; lastChars: string; - resourceGuid?: string; + resourceName?: string | null; }) { const t = useTranslations(); const preview = formatVirtualApiKeyPreview(virtualApiKeyId, lastChars); @@ -145,11 +142,11 @@ function IdentityKeyCenterpiece({ virtualApiKeyId ); const displayValue = credential ?? preview; - const headline = resourceGuid - ? t("myVirtualApiKeysIdentityResourceHeadline") + const headline = resourceName + ? t("myVirtualApiKeysIdentityResourceHeadline", { resourceName }) : t("myVirtualApiKeysIdentityHeadline"); - const description = resourceGuid - ? t("myVirtualApiKeysIdentityResourceDescription") + const description = resourceName + ? t("myVirtualApiKeysIdentityResourceDescription", { resourceName }) : t("myVirtualApiKeysIdentityDescription"); return ( @@ -220,17 +217,10 @@ function ManualKeyRow({ export default function UserVirtualApiKeys({ orgId, - resourceGuid, initialData }: UserVirtualApiKeysProps) { const t = useTranslations(); - - const title = resourceGuid - ? t("myVirtualApiKeysResourceTitle") - : t("myVirtualApiKeysTitle"); - const description = resourceGuid - ? t("myVirtualApiKeysResourceDescription") - : t("myVirtualApiKeysDescription"); + const resourceName = initialData.resourceName; return ( <> @@ -239,7 +229,7 @@ export default function UserVirtualApiKeys({ orgId={orgId} virtualApiKeyId={initialData.userKey.virtualApiKeyId} lastChars={initialData.userKey.lastChars} - resourceGuid={resourceGuid} + resourceName={resourceName} /> {initialData.manualKeys.length > 0 ? ( @@ -249,9 +239,10 @@ export default function UserVirtualApiKeys({ {t("myVirtualApiKeysManualTitle")} - {resourceGuid + {resourceName ? t( - "myVirtualApiKeysManualResourceDescription" + "myVirtualApiKeysManualResourceDescription", + { resourceName } ) : t("myVirtualApiKeysManualDescription")} diff --git a/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx b/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx index 2434ed575..9624a3a43 100644 --- a/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx +++ b/src/components/resource-launcher/LauncherInferenceApiKeysSection.tsx @@ -202,7 +202,12 @@ export function LauncherInferenceApiKeysSection({ {t( - "myVirtualApiKeysManualResourceDescription" + "myVirtualApiKeysManualResourceDescription", + { + resourceName: + data.resourceName ?? + t("resource") + } )} From 93cba1d098e55bdf719cd7b61a8aaf4fb1535870 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Wed, 12 Aug 2026 12:34:25 -0400 Subject: [PATCH 141/296] support sending capability specific error codes --- server/lib/aiCapabilities.ts | 112 +++++++++++++++++++++++-- server/lib/aiGatewayAuthError.ts | 27 ++++++ server/routers/aiGateway/pipeline.ts | 16 ++-- server/routers/badger/verifySession.ts | 37 +++++++- 4 files changed, 175 insertions(+), 17 deletions(-) create mode 100644 server/lib/aiGatewayAuthError.ts diff --git a/server/lib/aiCapabilities.ts b/server/lib/aiCapabilities.ts index 268acd36e..c9f78f10e 100644 --- a/server/lib/aiCapabilities.ts +++ b/server/lib/aiCapabilities.ts @@ -18,8 +18,41 @@ export type AiCapabilityDefinition = { model: string ) => string; isStreaming: (req: Request, contentType: string) => boolean; + /** Protocol-shaped body returned when gateway auth fails for this capability. */ + authErrorBody: Record; }; +const AUTH_MESSAGE = "Invalid API key provided."; + +export const OPENAI_AUTH_ERROR_BODY = { + error: { + message: AUTH_MESSAGE, + type: "authentication_error", + param: null, + code: "invalid_api_key" + } +} as const; + +const ANTHROPIC_AUTH_ERROR_BODY = { + type: "error", + error: { + type: "authentication_error", + message: AUTH_MESSAGE + } +} as const; + +const GOOGLE_AUTH_ERROR_BODY = { + error: { + code: 401, + message: AUTH_MESSAGE, + status: "UNAUTHENTICATED" + } +} as const; + +const BEDROCK_AUTH_ERROR_BODY = { + message: "The security token included in the request is invalid." +} as const; + function bodyModel(req: Request): string | undefined { return typeof req.body?.model === "string" ? req.body.model : undefined; } @@ -111,7 +144,8 @@ export const AI_CAPABILITY_DEFS: Record = extractModel: bodyModel, resolveUpstreamUrl: (base, req) => joinUpstreamUrl(base, pathFromRequest(req)), - isStreaming: isBodyOrSseStreaming + isStreaming: isBodyOrSseStreaming, + authErrorBody: OPENAI_AUTH_ERROR_BODY }, openai_responses: { id: "openai_responses", @@ -119,7 +153,8 @@ export const AI_CAPABILITY_DEFS: Record = extractModel: bodyModel, resolveUpstreamUrl: (base, req) => joinUpstreamUrl(base, pathFromRequest(req)), - isStreaming: isBodyOrSseStreaming + isStreaming: isBodyOrSseStreaming, + authErrorBody: OPENAI_AUTH_ERROR_BODY }, anthropic_messages: { id: "anthropic_messages", @@ -127,7 +162,8 @@ export const AI_CAPABILITY_DEFS: Record = extractModel: bodyModel, resolveUpstreamUrl: (base, req) => joinUpstreamUrl(base, pathFromRequest(req)), - isStreaming: isBodyOrSseStreaming + isStreaming: isBodyOrSseStreaming, + authErrorBody: ANTHROPIC_AUTH_ERROR_BODY }, gemini_generate_content: { id: "gemini_generate_content", @@ -144,7 +180,8 @@ export const AI_CAPABILITY_DEFS: Record = extractModel: paramModel, resolveUpstreamUrl: (base, req) => joinUpstreamUrl(base, pathFromRequest(req)), - isStreaming: isGeminiStyleStreaming + isStreaming: isGeminiStyleStreaming, + authErrorBody: GOOGLE_AUTH_ERROR_BODY }, google_generate_content: { id: "google_generate_content", @@ -162,7 +199,8 @@ export const AI_CAPABILITY_DEFS: Record = extractModel: paramModel, resolveUpstreamUrl: (base, req) => joinUpstreamUrl(base, pathFromRequest(req)), - isStreaming: isGeminiStyleStreaming + isStreaming: isGeminiStyleStreaming, + authErrorBody: GOOGLE_AUTH_ERROR_BODY }, google_raw_predict: { id: "google_raw_predict", @@ -182,7 +220,8 @@ export const AI_CAPABILITY_DEFS: Record = isStreaming: (req, contentType) => pathIncludes(req, "streamRawPredict") || pathIncludes(req, "alt=sse") || - contentTypeIsSse(contentType) + contentTypeIsSse(contentType), + authErrorBody: GOOGLE_AUTH_ERROR_BODY }, bedrock_model_invoke: { id: "bedrock_model_invoke", @@ -199,7 +238,8 @@ export const AI_CAPABILITY_DEFS: Record = isStreaming: (req, contentType) => pathIncludes(req, "invoke-with-response-stream") || contentTypeIsAmazonEventStream(contentType) || - contentTypeIsSse(contentType) + contentTypeIsSse(contentType), + authErrorBody: BEDROCK_AUTH_ERROR_BODY }, bedrock_converse: { id: "bedrock_converse", @@ -213,7 +253,8 @@ export const AI_CAPABILITY_DEFS: Record = isStreaming: (req, contentType) => pathIncludes(req, "converse-stream") || contentTypeIsAmazonEventStream(contentType) || - contentTypeIsSse(contentType) + contentTypeIsSse(contentType), + authErrorBody: BEDROCK_AUTH_ERROR_BODY } }; @@ -224,6 +265,61 @@ export function isAiCapability(value: unknown): value is AiCapability { ); } +/** + * Convert an Express-style route path from AI_CAPABILITY_DEFS into a RegExp. + * Handles `:param` segments and escaped literal colons (`\:`). + */ +export function routePatternToRegExp(routePath: string): RegExp { + let pattern = ""; + for (let i = 0; i < routePath.length; i++) { + const ch = routePath[i]; + if ( + ch === "\\" && + i + 1 < routePath.length && + routePath[i + 1] === ":" + ) { + pattern += ":"; + i++; + continue; + } + if (ch === ":") { + // Named param: consume until next / or end + i++; + while ( + i < routePath.length && + routePath[i] !== "/" && + !(routePath[i] === "\\" && routePath[i + 1] === ":") + ) { + i++; + } + i--; // loop will ++ + pattern += "[^/]+"; + continue; + } + // Escape regex special chars + if (/[.*+?^${}()|[\]\\]/.test(ch)) { + pattern += "\\" + ch; + } else { + pattern += ch; + } + } + return new RegExp(`^${pattern}$`); +} + +export function resolveAiCapabilityFromPath(path: string): AiCapability | null { + const pathname = path.split("?")[0] || "/"; + const normalized = pathname.startsWith("/") ? pathname : `/${pathname}`; + + for (const def of Object.values(AI_CAPABILITY_DEFS)) { + for (const route of def.routes) { + if (routePatternToRegExp(route.path).test(normalized)) { + return def.id; + } + } + } + return null; +} + export function parseCapabilities(raw: unknown): AiCapability[] { if (raw == null) { return []; diff --git a/server/lib/aiGatewayAuthError.ts b/server/lib/aiGatewayAuthError.ts new file mode 100644 index 000000000..55e4d5e28 --- /dev/null +++ b/server/lib/aiGatewayAuthError.ts @@ -0,0 +1,27 @@ +import { + AI_CAPABILITY_DEFS, + OPENAI_AUTH_ERROR_BODY, + type AiCapability +} from "@server/lib/aiCapabilities"; +import HttpCode from "@server/types/HttpCode"; + +export type ClientErrorResponse = { + statusCode?: number; + contentType?: string; + body: string; +}; + +export function buildInferenceAuthClientError( + capability: AiCapability | null +): ClientErrorResponse { + const body = + capability != null + ? AI_CAPABILITY_DEFS[capability].authErrorBody + : OPENAI_AUTH_ERROR_BODY; + + return { + statusCode: HttpCode.UNAUTHORIZED, + contentType: "application/json", + body: JSON.stringify(body) + }; +} diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 5aa69d7fc..d77937cd3 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -366,10 +366,6 @@ async function resolveTarget(host: string): Promise { .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)) ]); - if (attachmentRows.length === 0) { - return null; - } - return { resourceId: resourceRow.resourceId, siteResourceId: null, @@ -427,10 +423,6 @@ async function resolveTarget(host: string): Promise { ) ]); - if (attachmentRows.length === 0) { - return null; - } - return { resourceId: null, siteResourceId: siteResourceRow.siteResourceId, @@ -756,6 +748,14 @@ export async function handleAiGatewayProxy( }); } + if (attachments.length === 0) { + return res.status(HttpCode.FORBIDDEN).json({ + error: { + message: "No AI providers configured for this resource" + } + }); + } + const capableAttachments = attachments.filter((a) => providerHasCapability(a.provider.capabilities, capability) ); diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index f5b6a25b9..706626373 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -48,6 +48,11 @@ import { z } from "zod"; import { fromError } from "zod-validation-error"; import { getCountryCodeForIp } from "@server/lib/geoip"; import { getAsnForIp } from "@server/lib/asn"; +import { + buildInferenceAuthClientError, + type ClientErrorResponse +} from "@server/lib/aiGatewayAuthError"; +import { resolveAiCapabilityFromPath } from "@server/lib/aiCapabilities"; import { verifyPassword } from "@server/auth/password"; import { checkOrgAccessPolicy, @@ -89,6 +94,8 @@ type BasicUserData = { role: string | null; }; +export type { ClientErrorResponse }; + export type VerifyUserResponse = { valid: boolean; headerAuthChallenged?: boolean; @@ -96,8 +103,27 @@ export type VerifyUserResponse = { userData?: BasicUserData; pangolinVersion?: string; dontStripSession?: boolean; + clientError?: ClientErrorResponse; }; +function notAllowedWithClientError( + res: Response, + clientError: ClientErrorResponse +) { + const data = { + data: { + valid: false, + clientError, + pangolinVersion: APP_VERSION + }, + success: true, + error: false, + message: "Access denied", + status: HttpCode.OK + }; + return response(res, data); +} + export async function verifyResourceSession( req: Request, res: Response, @@ -408,7 +434,16 @@ export async function verifyResourceSession( parsedBody.data ); - return notAllowed(res, redirectPath, resource.orgId); + // Browsers go to the resource auth / API key page. API clients get + // a capability-shaped JSON auth error instead of a redirect. + if (clientIsBrowser) { + return notAllowed(res, redirectPath, resource.orgId); + } + + return notAllowedWithClientError( + res, + buildInferenceAuthClientError(resolveAiCapabilityFromPath(path)) + ); } // check for access token in headers From c6bd657ee62491e10b76798b419c18debce2e9be Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 12 Aug 2026 18:48:14 +0200 Subject: [PATCH 142/296] =?UTF-8?q?=E2=9C=A8=20send=20webhook=20action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../private/lib/alerts/processTestAlerts.ts | 16 ++++---- server/private/lib/alerts/sendAlertEmail.ts | 5 +-- server/private/lib/alerts/sendAlertWebhook.ts | 41 ++++++++++++++----- .../AlertRuleGraphEditor.tsx | 28 ++++--------- src/lib/queries.ts | 2 +- 5 files changed, 50 insertions(+), 42 deletions(-) diff --git a/server/private/lib/alerts/processTestAlerts.ts b/server/private/lib/alerts/processTestAlerts.ts index 1a6c1d7c6..f7fa47b20 100644 --- a/server/private/lib/alerts/processTestAlerts.ts +++ b/server/private/lib/alerts/processTestAlerts.ts @@ -7,15 +7,13 @@ import type { } from "@server/routers/alertRule/types"; import { eq, inArray } from "drizzle-orm"; import { sendAlertEmail } from "./sendAlertEmail"; -import { decrypt } from "@server/lib/crypto"; -import config from "@server/lib/config"; import { sendAlertWebhook } from "./sendAlertWebhook"; export async function processTestAlerts(context: TestAlertContext) { + // Process email actions const emailActions = context.actions.filter( (action) => action.type === "email" ); - // Process email actions for (const action of emailActions) { try { const recipients = await resolveEmailRecipients(action); @@ -30,10 +28,10 @@ export async function processTestAlerts(context: TestAlertContext) { } } + // Process webhook actions const webhookActions = context.actions.filter( (action) => action.type === "webhook" ); - const serverSecret = config.getRawConfig().server.secret!; for (const action of webhookActions) { try { @@ -41,8 +39,9 @@ export async function processTestAlerts(context: TestAlertContext) { if (action.config) { try { - const decrypted = decrypt(action.config, serverSecret); - webhookConfig = JSON.parse(decrypted) as WebhookAlertConfig; + webhookConfig = JSON.parse( + action.config + ) as WebhookAlertConfig; } catch (err) { logger.error( `processTestAlerts: failed to decrypt webhook`, @@ -52,7 +51,10 @@ export async function processTestAlerts(context: TestAlertContext) { } } - await sendAlertWebhook(action.webhookUrl, webhookConfig, context); + await sendAlertWebhook(action.webhookUrl, webhookConfig, { + ...context, + isTest: true + }); } catch (err) { logger.error( `processTestAlerts: failed to send alert webhook `, diff --git a/server/private/lib/alerts/sendAlertEmail.ts b/server/private/lib/alerts/sendAlertEmail.ts index 0eef6fb5c..ab7d49acc 100644 --- a/server/private/lib/alerts/sendAlertEmail.ts +++ b/server/private/lib/alerts/sendAlertEmail.ts @@ -15,10 +15,7 @@ import { sendEmail } from "@server/emails"; import AlertNotification from "@server/emails/templates/AlertNotification"; import config from "@server/lib/config"; import logger from "@server/logger"; -import { - AlertContext, - type AlertEventType -} from "@server/routers/alertRule/types"; +import { type AlertEventType } from "@server/routers/alertRule/types"; type EmailAlertContext = { eventType: AlertEventType; diff --git a/server/private/lib/alerts/sendAlertWebhook.ts b/server/private/lib/alerts/sendAlertWebhook.ts index 27e142cc4..d3a5d421b 100644 --- a/server/private/lib/alerts/sendAlertWebhook.ts +++ b/server/private/lib/alerts/sendAlertWebhook.ts @@ -14,13 +14,28 @@ import logger from "@server/logger"; import { AlertContext, - WebhookAlertConfig + WebhookAlertConfig, + type AlertEventType } from "@server/routers/alertRule/types"; const REQUEST_TIMEOUT_MS = 15_000; const MAX_RETRIES = 3; const RETRY_BASE_DELAY_MS = 500; +type WebhookAlertContext = { + eventType: AlertEventType; + orgId: string; + /** Set for site_online / site_offline events */ + siteId?: number; + /** Set for health_check_* events */ + healthCheckId?: number; + /** Set for resource_* events */ + resourceId?: number; + /** Human-readable context data included in emails and webhook payloads */ + data: Record; + isTest?: boolean; +}; + /** * Sends a single webhook POST for an alert event. * @@ -40,14 +55,14 @@ const RETRY_BASE_DELAY_MS = 500; export async function sendAlertWebhook( url: string, webhookConfig: WebhookAlertConfig, - context: AlertContext + context: WebhookAlertContext ): Promise { const eventType = context.eventType; const timestamp = new Date().toISOString(); const status = deriveStatus(eventType, context.data); const data = { orgId: context.orgId, ...context.data }; - let body: string; + let body: Record; if (webhookConfig.useBodyTemplate && webhookConfig.bodyTemplate?.trim()) { body = renderTemplate(webhookConfig.bodyTemplate, { event: eventType, @@ -56,7 +71,11 @@ export async function sendAlertWebhook( data }); } else { - body = JSON.stringify({ event: eventType, timestamp, status, data }); + body = { event: eventType, timestamp, status, data }; + } + + if (body.data && context.isTest) { + body.data.test = true; } const headers = buildHeaders(webhookConfig); @@ -75,7 +94,7 @@ export async function sendAlertWebhook( response = await fetch(url, { method: webhookConfig.method ?? "POST", headers, - body, + body: JSON.stringify(body), signal: controller.signal }); } catch (err: unknown) { @@ -247,7 +266,10 @@ interface TemplateContext { * left untouched. * 3. The fixed top-level keys: event, timestamp, status. */ -function renderTemplate(template: string, ctx: TemplateContext): string { +function renderTemplate( + template: string, + ctx: TemplateContext +): Record { // Step 1 – expand {{data}} first so its contents are already serialised // and won't be touched by later passes. let rendered = template.replace(/\{\{data\}\}/g, JSON.stringify(ctx.data)); @@ -280,20 +302,19 @@ function renderTemplate(template: string, ctx: TemplateContext): string { // Validate the rendered result is valid JSON; if not, log a warning and // fall back to the default payload so the webhook still fires. try { - JSON.parse(rendered); - return rendered; + return JSON.parse(rendered); } catch { logger.warn( `sendAlertWebhook: body template produced invalid JSON for event ` + `"${ctx.event}" destined for a webhook. Falling back to default ` + `payload. Check that {{data}} is NOT wrapped in quotes in your template.` ); - return JSON.stringify({ + return { event: ctx.event, timestamp: ctx.timestamp, status: ctx.status, data: ctx.data - }); + }; } } diff --git a/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx b/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx index 243ce82f8..a3e5e89cb 100644 --- a/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx +++ b/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx @@ -180,8 +180,9 @@ export default function AlertRuleGraphEditor({ const testAlert = async () => { const isValid = await form.trigger(); + const values = form.getValues(); + if (!isValid) { - const values = form.getValues(); if (values.actions.length === 0) { toast({ variant: "warning", @@ -193,27 +194,14 @@ export default function AlertRuleGraphEditor({ return; } - const values = form.getValues(); try { const payload = formValuesToApiPayload(values); - if (isNew) { - const res = await api.post< - AxiosResponse - >(`/org/${orgId}/test-alert-rule`, payload); - toast({ - title: t("alertingTestAlertSent"), - description: t("alertingTestAlertSentDescription") - }); - } else { - await api.post( - `/org/${orgId}/alert-rule/${alertRuleId}`, - payload - ); - toast({ - title: t("alertingTestAlertSent"), - description: t("alertingTestAlertSentDescription") - }); - } + await api.post(`/org/${orgId}/test-alert-rule`, payload); + + toast({ + title: t("alertingTestAlertSent"), + description: t("alertingTestAlertSentDescription") + }); } catch (e) { toast({ title: t("error"), diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 96024c3a0..e9e324c08 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -1371,7 +1371,7 @@ export const approvalQueries = { }, refetchInterval: (query) => { if (query.state.data) { - return durationToMs(30, "seconds"); + return durationToMs(1.5, "minutes"); } return false; } From 4b61e12ca6ae837e4b7341222afb60ffb824d8b3 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 12 Aug 2026 18:56:56 +0200 Subject: [PATCH 143/296] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20trigger=20alert=20?= =?UTF-8?q?correctly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../routers/alertRule/testAlertRule.ts | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/server/private/routers/alertRule/testAlertRule.ts b/server/private/routers/alertRule/testAlertRule.ts index 6104dbc6b..e870e9488 100644 --- a/server/private/routers/alertRule/testAlertRule.ts +++ b/server/private/routers/alertRule/testAlertRule.ts @@ -11,29 +11,17 @@ * This file is not licensed under the AGPLv3. */ -import { Request, Response, NextFunction } from "express"; -import { z } from "zod"; -import { db } from "@server/db"; -import { - alertRules, - alertSites, - alertHealthChecks, - alertResources -} from "@server/db"; -import response from "@server/lib/response"; -import HttpCode from "@server/types/HttpCode"; -import createHttpError from "http-errors"; -import logger from "@server/logger"; -import { fromError } from "zod-validation-error"; -import { OpenAPITags, registry } from "@server/openApi"; -import { and, asc, desc, eq, inArray, like, or, sql } from "drizzle-orm"; -import { - ListAlertRulesResponse, - type AlertAction, - type EmailAlertAction -} from "@server/routers/alertRule/types"; -import { processTestAlerts } from "@server/private/lib/alerts/processTestAlerts"; import { getRandomItemInArray } from "@app/lib/getRandomItemInArray"; +import response from "@server/lib/response"; +import logger from "@server/logger"; +import { processTestAlerts } from "@server/private/lib/alerts/processTestAlerts"; +import { type AlertAction } from "@server/routers/alertRule/types"; +import HttpCode from "@server/types/HttpCode"; +import { NextFunction, Request, Response } from "express"; +import createHttpError from "http-errors"; +import { z } from "zod"; +import { fromError } from "zod-validation-error"; +import type { TriggerSiteAlertResponse } from "../alertEvents"; const paramsSchema = z.strictObject({ orgId: z.string().nonempty() @@ -203,6 +191,14 @@ export async function testAlertRule( actions: collectedActions, data }); + + return response(res, { + data: { success: true }, + success: true, + error: false, + message: "Alert triggered successfully", + status: HttpCode.OK + }); } catch (error) { logger.error(error); return next( From 47ae017f94e7a6c2f45c85540f80dbe7ed25a457 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 12 Aug 2026 13:21:46 -0400 Subject: [PATCH 144/296] add virtual api keys to budgets --- messages/en-US.json | 2 + server/db/pg/schema/schema.ts | 11 +- server/db/sqlite/schema/schema.ts | 11 +- server/lib/aiBudgetEnforcement.ts | 20 +++- server/routers/aiBudget/createAiBudget.ts | 41 +++++-- server/routers/aiBudget/index.ts | 1 + .../aiBudget/listAiBudgetsForVirtualApiKey.ts | 69 ++++++++++++ server/routers/aiBudget/updateAiBudget.ts | 63 +++++++++-- server/routers/aiBudget/validation.ts | 6 +- server/routers/aiGateway/pipeline.ts | 3 +- server/routers/external.ts | 7 ++ src/components/CreateVirtualApiKeyForm.tsx | 92 +++++++++++++++- src/components/EditVirtualApiKeyForm.tsx | 104 +++++++++++++++++- src/lib/aiBudgetScope.ts | 17 ++- 14 files changed, 415 insertions(+), 32 deletions(-) create mode 100644 server/routers/aiBudget/listAiBudgetsForVirtualApiKey.ts diff --git a/messages/en-US.json b/messages/en-US.json index b42aa4f10..0abe669ea 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1697,6 +1697,8 @@ "virtualApiKeysErrorFetchSecret": "Error loading secret", "virtualApiKeysErrorFetchSecretDescription": "Failed to load the virtual API key secret", "virtualApiKeysFilterUnassigned": "Unassigned", + "virtualApiKeysInferenceBudget": "Inference Budget", + "virtualApiKeysInferenceBudgetDescription": "Configure how this key restricts AI usage based on spending or token limits", "myVirtualApiKeysTitle": "Your API Keys", "myVirtualApiKeysDescription": "View your identity key and any virtual API keys attributed to you in this organization", "myVirtualApiKeysResourceTitle": "Your API Keys for {resourceName}", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index a5ebfa530..8efebc619 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1776,6 +1776,10 @@ export const aiBudgets = pgTable( roleId: integer("roleId").references(() => roles.roleId, { onDelete: "cascade" }), + virtualApiKeyId: varchar("virtualApiKeyId").references( + () => virtualApiKeys.virtualApiKeyId, + { onDelete: "cascade" } + ), amount: real("amount").notNull(), unit: varchar("unit").$type<"usd" | "tokens">().notNull(), period: varchar("period") @@ -1806,7 +1810,12 @@ export const aiBudgets = pgTable( t.unit, t.period ), - unique("ai_budget_role_uniq").on(t.roleId, t.unit, t.period) + unique("ai_budget_role_uniq").on(t.roleId, t.unit, t.period), + unique("ai_budget_virtual_api_key_uniq").on( + t.virtualApiKeyId, + t.unit, + t.period + ) ] ); diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 450b99fce..e46aef8d2 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1762,6 +1762,10 @@ export const aiBudgets = sqliteTable( roleId: integer("roleId").references(() => roles.roleId, { onDelete: "cascade" }), + virtualApiKeyId: text("virtualApiKeyId").references( + () => virtualApiKeys.virtualApiKeyId, + { onDelete: "cascade" } + ), amount: real("amount").notNull(), unit: text("unit").$type<"usd" | "tokens">().notNull(), period: text("period") @@ -1794,7 +1798,12 @@ export const aiBudgets = sqliteTable( t.unit, t.period ), - unique("ai_budget_role_uniq").on(t.roleId, t.unit, t.period) + unique("ai_budget_role_uniq").on(t.roleId, t.unit, t.period), + unique("ai_budget_virtual_api_key_uniq").on( + t.virtualApiKeyId, + t.unit, + t.period + ) ] ); diff --git a/server/lib/aiBudgetEnforcement.ts b/server/lib/aiBudgetEnforcement.ts index f6daf9694..05decd6ae 100644 --- a/server/lib/aiBudgetEnforcement.ts +++ b/server/lib/aiBudgetEnforcement.ts @@ -53,7 +53,8 @@ function applicableBudgetsCacheKey(ctx: BudgetScopeContext): string { ctx.requestedModel, ctx.resourceId ?? "", ctx.siteResourceId ?? "", - roleKey + roleKey, + ctx.virtualApiKeyId ?? "" ].join(":"); } @@ -83,6 +84,7 @@ export type BudgetScopeContext = { siteResourceId: number | null; roleIds: number[]; requestUserId: string | null; + virtualApiKeyId: string | null; }; /** @@ -142,6 +144,11 @@ async function fetchApplicableBudgets( if (ctx.roleIds.length > 0) { scopeConditions.push(inArray(aiBudgets.roleId, ctx.roleIds)); } + if (ctx.virtualApiKeyId != null) { + scopeConditions.push( + eq(aiBudgets.virtualApiKeyId, ctx.virtualApiKeyId) + ); + } return db .select() @@ -274,6 +281,17 @@ export async function sumUsageForBudget( ); } + if (budget.virtualApiKeyId != null) { + return sumUsageAmount( + and( + eq(aiUsageRecords.orgId, ctx.orgId), + eq(aiUsageRecords.virtualApiKeyId, budget.virtualApiKeyId), + gte(aiUsageRecords.createdAt, start) + )!, + budget.unit + ); + } + return 0; } diff --git a/server/routers/aiBudget/createAiBudget.ts b/server/routers/aiBudget/createAiBudget.ts index 3ddd76174..0972f9d29 100644 --- a/server/routers/aiBudget/createAiBudget.ts +++ b/server/routers/aiBudget/createAiBudget.ts @@ -7,7 +7,8 @@ import { db, resources, roles, - siteResources + siteResources, + virtualApiKeys } from "@server/db"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -35,6 +36,7 @@ const bodySchema = z resourceId: z.coerce.number().int().positive().optional(), siteResourceId: z.coerce.number().int().positive().optional(), roleId: z.coerce.number().int().positive().optional(), + virtualApiKeyId: z.string().nonempty().optional(), amount: z.number().positive(), unit: aiBudgetUnitSchema, period: aiBudgetPeriodSchema.optional().default("monthly"), @@ -98,6 +100,7 @@ export async function createAiBudget( resourceId, siteResourceId, roleId, + virtualApiKeyId, amount, unit, period, @@ -189,6 +192,22 @@ export async function createAiBudget( } } + if (virtualApiKeyId !== undefined) { + const [key] = await db + .select({ orgId: virtualApiKeys.orgId }) + .from(virtualApiKeys) + .where(eq(virtualApiKeys.virtualApiKeyId, virtualApiKeyId)) + .limit(1); + if (!key || key.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Virtual API key with ID ${virtualApiKeyId} not found in this organization` + ) + ); + } + } + const scopeCondition = providerId !== undefined ? eq(aiBudgets.providerId, providerId) @@ -200,14 +219,17 @@ export async function createAiBudget( ? eq(aiBudgets.siteResourceId, siteResourceId) : roleId !== undefined ? eq(aiBudgets.roleId, roleId) - : and( - eq(aiBudgets.orgId, orgId), - isNull(aiBudgets.providerId), - isNull(aiBudgets.modelId), - isNull(aiBudgets.resourceId), - isNull(aiBudgets.siteResourceId), - isNull(aiBudgets.roleId) - ); + : virtualApiKeyId !== undefined + ? eq(aiBudgets.virtualApiKeyId, virtualApiKeyId) + : and( + eq(aiBudgets.orgId, orgId), + isNull(aiBudgets.providerId), + isNull(aiBudgets.modelId), + isNull(aiBudgets.resourceId), + isNull(aiBudgets.siteResourceId), + isNull(aiBudgets.roleId), + isNull(aiBudgets.virtualApiKeyId) + ); const [existing] = await db .select({ budgetId: aiBudgets.budgetId }) @@ -239,6 +261,7 @@ export async function createAiBudget( resourceId: resourceId ?? null, siteResourceId: siteResourceId ?? null, roleId: roleId ?? null, + virtualApiKeyId: virtualApiKeyId ?? null, amount, unit, period, diff --git a/server/routers/aiBudget/index.ts b/server/routers/aiBudget/index.ts index 36eed7ba4..7052dbe4e 100644 --- a/server/routers/aiBudget/index.ts +++ b/server/routers/aiBudget/index.ts @@ -5,6 +5,7 @@ export * from "./listAiBudgetsForModel"; export * from "./listAiBudgetsForResource"; export * from "./listAiBudgetsForSiteResource"; export * from "./listAiBudgetsForRole"; +export * from "./listAiBudgetsForVirtualApiKey"; export * from "./getAiBudget"; export * from "./updateAiBudget"; export * from "./deleteAiBudget"; diff --git a/server/routers/aiBudget/listAiBudgetsForVirtualApiKey.ts b/server/routers/aiBudget/listAiBudgetsForVirtualApiKey.ts new file mode 100644 index 000000000..be2e45775 --- /dev/null +++ b/server/routers/aiBudget/listAiBudgetsForVirtualApiKey.ts @@ -0,0 +1,69 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { aiBudgets, db } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { asc, eq } from "drizzle-orm"; +import type { ListAiBudgetsByScopeResponse } from "@server/routers/aiBudget/types"; + +const paramsSchema = z.strictObject({ + virtualApiKeyId: z.string().nonempty() +}); + +registry.registerPath({ + method: "get", + path: "/virtual-api-key/{virtualApiKeyId}/ai-budgets", + description: "List AI budgets scoped to a virtual API key.", + tags: [OpenAPITags.AiBudget], + request: { + params: paramsSchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listAiBudgetsForVirtualApiKey( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { virtualApiKeyId } = parsedParams.data; + + const budgets = await db + .select() + .from(aiBudgets) + .where(eq(aiBudgets.virtualApiKeyId, virtualApiKeyId)) + .orderBy(asc(aiBudgets.budgetId)); + + return response(res, { + data: { budgets }, + success: true, + error: false, + message: "AI budgets retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/aiBudget/updateAiBudget.ts b/server/routers/aiBudget/updateAiBudget.ts index 723b346c7..c7bf70cae 100644 --- a/server/routers/aiBudget/updateAiBudget.ts +++ b/server/routers/aiBudget/updateAiBudget.ts @@ -7,7 +7,8 @@ import { db, resources, roles, - siteResources + siteResources, + virtualApiKeys } from "@server/db"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -34,6 +35,7 @@ const bodySchema = z.strictObject({ resourceId: z.coerce.number().int().positive().nullable().optional(), siteResourceId: z.coerce.number().int().positive().nullable().optional(), roleId: z.coerce.number().int().positive().nullable().optional(), + virtualApiKeyId: z.string().nonempty().nullable().optional(), amount: z.number().positive().optional(), unit: aiBudgetUnitSchema.optional(), period: aiBudgetPeriodSchema.optional(), @@ -128,6 +130,10 @@ export async function updateAiBudget( : existing.siteResourceId; const nextRoleId = body.roleId !== undefined ? body.roleId : existing.roleId; + const nextVirtualApiKeyId = + body.virtualApiKeyId !== undefined + ? body.virtualApiKeyId + : existing.virtualApiKeyId; const nextUnit = body.unit !== undefined ? body.unit : existing.unit; const nextPeriod = body.period !== undefined ? body.period : existing.period; @@ -138,7 +144,8 @@ export async function updateAiBudget( modelId: z.number().nullable().optional(), resourceId: z.number().nullable().optional(), siteResourceId: z.number().nullable().optional(), - roleId: z.number().nullable().optional() + roleId: z.number().nullable().optional(), + virtualApiKeyId: z.string().nullable().optional() }) .superRefine((data, ctx) => refineBudgetScopeFields(data, ctx)) .safeParse({ @@ -146,7 +153,8 @@ export async function updateAiBudget( modelId: nextModelId, resourceId: nextResourceId, siteResourceId: nextSiteResourceId, - roleId: nextRoleId + roleId: nextRoleId, + virtualApiKeyId: nextVirtualApiKeyId }); if (!scopeValidation.success) { @@ -245,6 +253,30 @@ export async function updateAiBudget( } } + if ( + body.virtualApiKeyId !== undefined && + body.virtualApiKeyId !== null + ) { + const [key] = await db + .select({ orgId: virtualApiKeys.orgId }) + .from(virtualApiKeys) + .where( + eq( + virtualApiKeys.virtualApiKeyId, + body.virtualApiKeyId + ) + ) + .limit(1); + if (!key || key.orgId !== orgId) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Virtual API key with ID ${body.virtualApiKeyId} not found in this organization` + ) + ); + } + } + const scopeCondition = nextProviderId !== null ? eq(aiBudgets.providerId, nextProviderId) @@ -256,14 +288,20 @@ export async function updateAiBudget( ? eq(aiBudgets.siteResourceId, nextSiteResourceId) : nextRoleId !== null ? eq(aiBudgets.roleId, nextRoleId) - : and( - eq(aiBudgets.orgId, orgId), - isNull(aiBudgets.providerId), - isNull(aiBudgets.modelId), - isNull(aiBudgets.resourceId), - isNull(aiBudgets.siteResourceId), - isNull(aiBudgets.roleId) - ); + : nextVirtualApiKeyId !== null + ? eq( + aiBudgets.virtualApiKeyId, + nextVirtualApiKeyId + ) + : and( + eq(aiBudgets.orgId, orgId), + isNull(aiBudgets.providerId), + isNull(aiBudgets.modelId), + isNull(aiBudgets.resourceId), + isNull(aiBudgets.siteResourceId), + isNull(aiBudgets.roleId), + isNull(aiBudgets.virtualApiKeyId) + ); const [conflict] = await db .select({ budgetId: aiBudgets.budgetId }) @@ -305,6 +343,9 @@ export async function updateAiBudget( if (body.roleId !== undefined) { updateData.roleId = body.roleId; } + if (body.virtualApiKeyId !== undefined) { + updateData.virtualApiKeyId = body.virtualApiKeyId; + } if (body.amount !== undefined) { updateData.amount = body.amount; } diff --git a/server/routers/aiBudget/validation.ts b/server/routers/aiBudget/validation.ts index 303039948..e7cae351f 100644 --- a/server/routers/aiBudget/validation.ts +++ b/server/routers/aiBudget/validation.ts @@ -20,6 +20,7 @@ export function refineBudgetScopeFields( resourceId?: number | null; siteResourceId?: number | null; roleId?: number | null; + virtualApiKeyId?: string | null; }, ctx: z.RefinementCtx ) { @@ -28,7 +29,8 @@ export function refineBudgetScopeFields( data.modelId, data.resourceId, data.siteResourceId, - data.roleId + data.roleId, + data.virtualApiKeyId ]; const setCount = scopeFields.filter( @@ -39,7 +41,7 @@ export function refineBudgetScopeFields( ctx.addIssue({ code: "custom", message: - "Only one of providerId, modelId, resourceId, siteResourceId, or roleId may be set on a budget", + "Only one of providerId, modelId, resourceId, siteResourceId, roleId, or virtualApiKeyId may be set on a budget", path: ["providerId"] }); } diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 12b1cbb79..2d916a97a 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -830,7 +830,8 @@ export async function handleAiGatewayProxy( resourceId, siteResourceId, roleIds: requestUser?.roleIds ?? [], - requestUserId: requestUser?.userId ?? null + requestUserId: requestUser?.userId ?? null, + virtualApiKeyId: identity.virtualApiKeyId }); appliedBudgets = budgetCheck.budgets; diff --git a/server/routers/external.ts b/server/routers/external.ts index 6f44c1efe..c391665b9 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1791,6 +1791,13 @@ authenticated.get( aiBudget.listAiBudgetsForRole ); +authenticated.get( + "/virtual-api-key/:virtualApiKeyId/ai-budgets", + verifyVirtualApiKeyAccess, + verifyUserHasAction(ActionsEnum.listAiBudgets), + aiBudget.listAiBudgetsForVirtualApiKey +); + authenticated.get( "/org/:orgId/labels", verifyOrgAccess, diff --git a/src/components/CreateVirtualApiKeyForm.tsx b/src/components/CreateVirtualApiKeyForm.tsx index e67488b5d..c443ad4b9 100644 --- a/src/components/CreateVirtualApiKeyForm.tsx +++ b/src/components/CreateVirtualApiKeyForm.tsx @@ -48,6 +48,12 @@ import { formatMultiResourcesSelectorLabel } from "@app/components/multi-resource-selector"; import type { SelectedResource } from "@app/components/resource-selector"; +import { HorizontalTabs } from "@app/components/HorizontalTabs"; +import { + BudgetRowsFields, + getBudgetRowsErrors, + type BudgetRow +} from "@app/components/BudgetsEditor"; export type CreatedVirtualApiKey = { virtualApiKeyId: string; @@ -93,6 +99,10 @@ export default function CreateVirtualApiKeyForm({ const [selectedResources, setSelectedResources] = useState< SelectedResource[] >([]); + const [pendingBudgetRows, setPendingBudgetRows] = useState( + [] + ); + const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); const formSchema = z.object({ name: z.string().min(1), @@ -113,9 +123,29 @@ export default function CreateVirtualApiKeyForm({ setAllResources(false); setSelectedUser(null); setSelectedResources([]); + setPendingBudgetRows([]); + setAttemptedBudgetsSave(false); form.reset(); } + function handleFormSubmit(values: z.infer) { + const { conflictingKeys, invalidAmountKeys } = + getBudgetRowsErrors(pendingBudgetRows); + if (conflictingKeys.size > 0 || invalidAmountKeys.size > 0) { + setAttemptedBudgetsSave(true); + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: conflictingKeys.size + ? t("aiBudgetConflictError") + : t("aiBudgetInvalidAmountError") + }); + return; + } + + return onSubmit(values); + } + async function onSubmit(values: z.infer) { setLoading(true); @@ -155,6 +185,33 @@ export default function CreateVirtualApiKeyForm({ ); } + const pendingBudgets = pendingBudgetRows.filter( + (budget) => budget.amount.trim() !== "" + ); + if (pendingBudgets.length > 0) { + try { + await Promise.all( + pendingBudgets.map((budget) => + api.put(`/org/${org.org.orgId}/ai-budget`, { + virtualApiKeyId: key.virtualApiKeyId, + amount: Number(budget.amount), + unit: budget.unit, + period: budget.period + }) + ) + ); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: formatAxiosError( + e, + t("aiBudgetErrorSave") + ) + }); + } + } + const resourceLookup = new Map( selectedResources.map((r) => [ r.resourceId, @@ -219,10 +276,26 @@ export default function CreateVirtualApiKeyForm({ {!credential && (
+ +
)}
+
+ +
+

+ {t( + "virtualApiKeysInferenceBudgetDescription" + )} +

+ +
+ )} diff --git a/src/components/EditVirtualApiKeyForm.tsx b/src/components/EditVirtualApiKeyForm.tsx index 0ef5e936b..1d3e78dbd 100644 --- a/src/components/EditVirtualApiKeyForm.tsx +++ b/src/components/EditVirtualApiKeyForm.tsx @@ -50,6 +50,16 @@ import { getUserDisplayName } from "@app/lib/getUserDisplayName"; import CopyTextBox from "@app/components/CopyTextBox"; import type { CreatedVirtualApiKey } from "@app/components/CreateVirtualApiKeyForm"; import type { GetVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; +import { HorizontalTabs } from "@app/components/HorizontalTabs"; +import { + BudgetRowsFields, + getBudgetRowsErrors, + rowsFromBudgets, + saveBudgetRows, + type BudgetRow +} from "@app/components/BudgetsEditor"; +import { aiBudgetQueries } from "@app/lib/queries"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; type FormProps = { open: boolean; @@ -93,6 +103,7 @@ export default function EditVirtualApiKeyForm({ const { env } = useEnvContext(); const api = createApiClient({ env }); const t = useTranslations(); + const queryClient = useQueryClient(); const [loading, setLoading] = useState(false); const [selectedUser, setSelectedUser] = useState(null); @@ -101,6 +112,19 @@ export default function EditVirtualApiKeyForm({ >([]); const [credential, setCredential] = useState(null); const [credentialLoading, setCredentialLoading] = useState(false); + const [pendingBudgetRows, setPendingBudgetRows] = useState( + [] + ); + const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); + + const budgetScope = { + type: "virtualApiKey" as const, + id: virtualApiKey?.virtualApiKeyId ?? "" + }; + const budgetsQuery = useQuery({ + ...aiBudgetQueries.scoped({ scope: budgetScope }), + enabled: open && !!virtualApiKey + }); const formSchema = z .object({ @@ -191,6 +215,32 @@ export default function EditVirtualApiKeyForm({ }; }, [open, virtualApiKey, form]); + useEffect(() => { + if (!open || !budgetsQuery.data) { + return; + } + setPendingBudgetRows(rowsFromBudgets(budgetsQuery.data)); + setAttemptedBudgetsSave(false); + }, [open, budgetsQuery.data]); + + function handleFormSubmit(values: z.infer) { + const { conflictingKeys, invalidAmountKeys } = + getBudgetRowsErrors(pendingBudgetRows); + if (conflictingKeys.size > 0 || invalidAmountKeys.size > 0) { + setAttemptedBudgetsSave(true); + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: conflictingKeys.size + ? t("aiBudgetConflictError") + : t("aiBudgetInvalidAmountError") + }); + return; + } + + return onSubmit(values); + } + async function onSubmit(values: z.infer) { if (!virtualApiKey) { return; @@ -223,6 +273,26 @@ export default function EditVirtualApiKeyForm({ if (res?.data.data.virtualApiKey) { const key = res.data.data.virtualApiKey; + + try { + await saveBudgetRows({ + api, + orgId: virtualApiKey.orgId, + scope: budgetScope, + existingBudgets: budgetsQuery.data ?? [], + rows: pendingBudgetRows + }); + await queryClient.invalidateQueries( + aiBudgetQueries.scoped({ scope: budgetScope }) + ); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiBudgetErrorSave"), + description: formatAxiosError(e, t("aiBudgetErrorSave")) + }); + } + const resourceLookup = new Map( selectedResources.map((r) => [ r.resourceId, @@ -280,10 +350,26 @@ export default function EditVirtualApiKeyForm({
+ +
)}
+
+ +
+

+ {t( + "virtualApiKeysInferenceBudgetDescription" + )} +

+ +
+
diff --git a/src/lib/aiBudgetScope.ts b/src/lib/aiBudgetScope.ts index beabc85a3..333f63411 100644 --- a/src/lib/aiBudgetScope.ts +++ b/src/lib/aiBudgetScope.ts @@ -5,11 +5,12 @@ export type AiBudgetScopeType = | "model" | "resource" | "siteResource" - | "role"; + | "role" + | "virtualApiKey"; export type AiBudgetScope = { type: AiBudgetScopeType; - id: number; + id: number | string; }; export type AiBudgetScopeBodyField = @@ -17,11 +18,15 @@ export type AiBudgetScopeBodyField = | "modelId" | "resourceId" | "siteResourceId" - | "roleId"; + | "roleId" + | "virtualApiKeyId"; const scopeConfig: Record< AiBudgetScopeType, - { listPath: (id: number) => string; bodyField: AiBudgetScopeBodyField } + { + listPath: (id: number | string) => string; + bodyField: AiBudgetScopeBodyField; + } > = { provider: { listPath: (id) => `/ai-provider/${id}/ai-budgets`, @@ -42,6 +47,10 @@ const scopeConfig: Record< role: { listPath: (id) => `/role/${id}/ai-budgets`, bodyField: "roleId" + }, + virtualApiKey: { + listPath: (id) => `/virtual-api-key/${id}/ai-budgets`, + bodyField: "virtualApiKeyId" } }; From 49b4fcf0634e74c55e520b2bb53e50411585acf5 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 12 Aug 2026 21:36:40 +0200 Subject: [PATCH 145/296] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/alert-rule-editor/AlertRuleGraphEditor.tsx | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx b/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx index a3e5e89cb..9114df3ed 100644 --- a/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx +++ b/src/components/alert-rule-editor/AlertRuleGraphEditor.tsx @@ -171,15 +171,10 @@ export default function AlertRuleGraphEditor({ variant: "destructive" }); } - // const submit = form.handleSubmit(async (values) => { - - // }); - - // await submit(); }; const testAlert = async () => { - const isValid = await form.trigger(); + const isValid = await form.trigger("actions"); const values = form.getValues(); if (!isValid) { From 653dd920ad25ef86ff8123963d5acfd326ca8f5d Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 12 Aug 2026 15:54:22 -0400 Subject: [PATCH 146/296] Allow remote header overrides and fix display issues with log --- messages/en-US.json | 1 + server/lib/readConfigFile.ts | 22 ++ server/lib/traefik/TraefikConfigManager.ts | 26 ++- server/lib/virtualApiKey.ts | 19 +- server/routers/aiGateway/pipeline.ts | 30 ++- .../routers/traefik/traefikConfigProvider.ts | 22 +- .../virtualApiKey/listMyVirtualApiKeys.ts | 18 +- src/app/[orgId]/settings/logs/ai/page.tsx | 3 - .../[orgId]/settings/logs/request/page.tsx | 2 + src/components/AiUsageAnalyticsData.tsx | 203 ++++++++---------- 10 files changed, 213 insertions(+), 133 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 0abe669ea..7630612ca 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -3437,6 +3437,7 @@ "validPassword": "Valid Password", "validEmail": "Valid email", "validSSO": "Valid SSO", + "validVirtualAPIKey": "Valid Virtual API Key", "view": "View", "configManaged": "Config Managed", "connectedClient": "Connected Client", diff --git a/server/lib/readConfigFile.ts b/server/lib/readConfigFile.ts index d5c52f13a..58d7b4006 100644 --- a/server/lib/readConfigFile.ts +++ b/server/lib/readConfigFile.ts @@ -111,6 +111,20 @@ export const configSchema = z }) .optional() .prefault({}), + remote_headers: z + .object({ + user_id: z.string().optional().default("Remote-User-Id"), + virtual_api_key_id: z + .string() + .optional() + .default("Remote-Virtual-Api-Key-Id"), + user: z.string().optional().default("Remote-User"), + email: z.string().optional().default("Remote-Email"), + name: z.string().optional().default("Remote-Name"), + role: z.string().optional().default("Remote-Role") + }) + .optional() + .prefault({}), resource_session_request_param: z .string() .optional() @@ -154,6 +168,14 @@ export const configSchema = z id: "P-Access-Token-Id", token: "P-Access-Token" }, + remote_headers: { + user_id: "Remote-User-Id", + virtual_api_key_id: "Remote-Virtual-Api-Key-Id", + user: "Remote-User", + email: "Remote-Email", + name: "Remote-Name", + role: "Remote-Role" + }, resource_session_request_param: "resource_session_request_param", dashboard_session_length_hours: 720, diff --git a/server/lib/traefik/TraefikConfigManager.ts b/server/lib/traefik/TraefikConfigManager.ts index a52c5232a..91bc37249 100644 --- a/server/lib/traefik/TraefikConfigManager.ts +++ b/server/lib/traefik/TraefikConfigManager.ts @@ -605,7 +605,31 @@ export class TraefikConfigManager { resourceSessionRequestParam: config.getRawConfig().server - .resource_session_request_param + .resource_session_request_param, + + remoteUserIdHeader: + config.getRawConfig().server.remote_headers + .user_id, + + remoteVirtualApiKeyIdHeader: + config.getRawConfig().server.remote_headers + .virtual_api_key_id, + + remoteUserHeader: + config.getRawConfig().server.remote_headers + .user, + + remoteEmailHeader: + config.getRawConfig().server.remote_headers + .email, + + remoteNameHeader: + config.getRawConfig().server.remote_headers + .name, + + remoteRoleHeader: + config.getRawConfig().server.remote_headers + .role } } }; diff --git a/server/lib/virtualApiKey.ts b/server/lib/virtualApiKey.ts index e4f7480db..471be89fd 100644 --- a/server/lib/virtualApiKey.ts +++ b/server/lib/virtualApiKey.ts @@ -5,6 +5,7 @@ import { import { db, resources, + User, virtualApiKeyResources, virtualApiKeys, type Transaction, @@ -148,12 +149,12 @@ async function selectUserVirtualApiKey( export async function getOrCreateUserVirtualApiKey(params: { orgId: string; - userId: string; + user: User; createdByUserId?: string | null; }): Promise<{ key: VirtualApiKey; secret: string }> { - const { orgId, userId, createdByUserId } = params; + const { orgId, user, createdByUserId } = params; - const existing = await selectUserVirtualApiKey(orgId, userId); + const existing = await selectUserVirtualApiKey(orgId, user.userId); if (existing) { return { key: existing, @@ -171,8 +172,8 @@ export async function getOrCreateUserVirtualApiKey(params: { virtualApiKeyId: minted.virtualApiKeyId, orgId, kind: "user", - userId, - name: null, + userId: user.userId, + name: `${user.name ?? user.username}'s API Key`, description: null, token: encryptVirtualApiKeyToken(minted.secret), lastChars: minted.lastChars, @@ -186,7 +187,7 @@ export async function getOrCreateUserVirtualApiKey(params: { return { key: created, secret: minted.secret }; } catch { - const raced = await selectUserVirtualApiKey(orgId, userId); + const raced = await selectUserVirtualApiKey(orgId, user.userId); if (raced) { return { key: raced, @@ -199,11 +200,11 @@ export async function getOrCreateUserVirtualApiKey(params: { export async function rotateUserVirtualApiKey(params: { orgId: string; - userId: string; + user: User; createdByUserId?: string | null; }): Promise<{ key: VirtualApiKey; secret: string }> { - const { orgId, userId, createdByUserId } = params; - const existing = await selectUserVirtualApiKey(orgId, userId); + const { orgId, user, createdByUserId } = params; + const existing = await selectUserVirtualApiKey(orgId, user.userId); if (!existing) { return getOrCreateUserVirtualApiKey(params); diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 2d916a97a..06ca9cdae 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -180,15 +180,16 @@ export function applyRequestUserHeaders( if (!requestUser) { return; } - headers["Remote-User"] = requestUser.username; + const remoteHeaders = config.getRawConfig().server.remote_headers; + headers[remoteHeaders.user] = requestUser.username; if (requestUser.email) { - headers["Remote-Email"] = requestUser.email; + headers[remoteHeaders.email] = requestUser.email; } if (requestUser.name) { - headers["Remote-Name"] = requestUser.name; + headers[remoteHeaders.name] = requestUser.name; } if (requestUser.role) { - headers["Remote-Role"] = requestUser.role; + headers[remoteHeaders.role] = requestUser.role; } } @@ -236,14 +237,20 @@ async function resolveRequestUser( // Public inference: identity comes from Badger via Remote-* only when the // Traefik trust header proves the request passed verify-session (VAK). if (isAiGatewayTrustHeaderValid(req.headers as Record)) { + const remoteHeaders = config.getRawConfig().server.remote_headers; const virtualApiKeyId = - getRequestHeader(req, "remote-virtual-api-key-id") || null; - const userId = getRequestHeader(req, "remote-user-id"); + getRequestHeader(req, remoteHeaders.virtual_api_key_id) || null; + const userId = getRequestHeader(req, remoteHeaders.user_id); + logger.debug("+++++++AI gateway request identity from trust header", { + virtualApiKeyId, + userId + }); if (userId) { - const username = getRequestHeader(req, "remote-user") || userId; - const email = getRequestHeader(req, "remote-email"); - const name = getRequestHeader(req, "remote-name"); - const role = getRequestHeader(req, "remote-role"); + const username = + getRequestHeader(req, remoteHeaders.user) || userId; + const email = getRequestHeader(req, remoteHeaders.email); + const name = getRequestHeader(req, remoteHeaders.name); + const role = getRequestHeader(req, remoteHeaders.role); const orgRoles = orgId ? await getUserOrgRoles(userId, orgId) : []; return { @@ -742,6 +749,9 @@ export async function handleAiGatewayProxy( } logger.info(`AI gateway ${capability} request for host: ${host}`); + logger.debug("AI gateway request headers", { + headers: req.headers + }); const target = await resolveTarget(host); if (!target) { diff --git a/server/routers/traefik/traefikConfigProvider.ts b/server/routers/traefik/traefikConfigProvider.ts index e1847d2ed..02e05f5e0 100644 --- a/server/routers/traefik/traefikConfigProvider.ts +++ b/server/routers/traefik/traefikConfigProvider.ts @@ -68,7 +68,27 @@ export async function traefikConfigProvider( resourceSessionRequestParam: config.getRawConfig().server - .resource_session_request_param + .resource_session_request_param, + + remoteUserIdHeader: + config.getRawConfig().server.remote_headers + .user_id, + + remoteVirtualApiKeyIdHeader: + config.getRawConfig().server.remote_headers + .virtual_api_key_id, + + remoteUserHeader: + config.getRawConfig().server.remote_headers.user, + + remoteEmailHeader: + config.getRawConfig().server.remote_headers.email, + + remoteNameHeader: + config.getRawConfig().server.remote_headers.name, + + remoteRoleHeader: + config.getRawConfig().server.remote_headers.role } } }; diff --git a/server/routers/virtualApiKey/listMyVirtualApiKeys.ts b/server/routers/virtualApiKey/listMyVirtualApiKeys.ts index 5177a975a..46531560a 100644 --- a/server/routers/virtualApiKey/listMyVirtualApiKeys.ts +++ b/server/routers/virtualApiKey/listMyVirtualApiKeys.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { db, resources, + users, virtualApiKeyResources, virtualApiKeys, type VirtualApiKey @@ -152,9 +153,24 @@ export async function listMyVirtualApiKeys( resourceName = resource.name; } + const [user] = await db + .select() + .from(users) + .where(eq(users.userId, userId)) + .limit(1); + + if (!user) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `User with ID ${userId} not found` + ) + ); + } + const { key: userKeyRow } = await getOrCreateUserVirtualApiKey({ orgId, - userId, + user, createdByUserId: userId }); diff --git a/src/app/[orgId]/settings/logs/ai/page.tsx b/src/app/[orgId]/settings/logs/ai/page.tsx index 15c573337..8c077929b 100644 --- a/src/app/[orgId]/settings/logs/ai/page.tsx +++ b/src/app/[orgId]/settings/logs/ai/page.tsx @@ -577,9 +577,6 @@ export default function AiSessionLogsPage() {
{row.usage && (
-
- {t("tokenUsage")} -
{t("promptTokens")} diff --git a/src/app/[orgId]/settings/logs/request/page.tsx b/src/app/[orgId]/settings/logs/request/page.tsx index a4d1dec37..32e4f4b86 100644 --- a/src/app/[orgId]/settings/logs/request/page.tsx +++ b/src/app/[orgId]/settings/logs/request/page.tsx @@ -247,6 +247,7 @@ export default function GeneralPage() { // 106 - Valid email // 107 - Valid SSO // 108 - Connected Client + // 109 - Valid Virtual API Key // 201 - Resource Not Found // 202 - Resource Blocked @@ -265,6 +266,7 @@ export default function GeneralPage() { 106: t("validEmail"), 107: t("validSSO"), 108: t("connectedClient"), + 109: t("validVirtualAPIKey"), 201: t("resourceNotFound"), 202: t("resourceBlocked"), 203: t("droppedByRule"), diff --git a/src/components/AiUsageAnalyticsData.tsx b/src/components/AiUsageAnalyticsData.tsx index a5d3db2b8..3cb35198e 100644 --- a/src/components/AiUsageAnalyticsData.tsx +++ b/src/components/AiUsageAnalyticsData.tsx @@ -161,120 +161,107 @@ export function AiUsageAnalyticsData(props: AiUsageAnalyticsDataProps) { return (
- -
- + + - + -
- - setFilter("providerId", v) - } - /> - setFilter("model", v)} - /> - - setFilter("resourceId", v) - } - /> - setFilter("roleId", v)} - /> - setFilter("userId", v)} - /> - - setFilter("virtualApiKeyId", v) - } - /> + setFilter("providerId", v)} + /> + setFilter("model", v)} + /> + setFilter("resourceId", v)} + /> + setFilter("roleId", v)} + /> + setFilter("userId", v)} + /> + setFilter("virtualApiKeyId", v)} + /> - {!isEmptySearchParams && ( - - )} -
-
-
+ {!isEmptySearchParams && ( -
+ )} + + @@ -301,7 +288,7 @@ type FilterSelectProps = { function FilterSelect(props: FilterSelectProps) { return ( -
+
- - - - )} - /> - - { + const previousType = + field.value; field.onChange( value ); @@ -293,6 +277,30 @@ export default function CreateAiProviderPage() { value !== "custom" ) { + const currentName = + form.getValues( + "name" + ); + const previousLabel = + t( + aiProviderTypeLabelMap[ + previousType + ] + ); + if ( + !currentName.trim() || + currentName === + previousLabel + ) { + form.setValue( + "name", + t( + aiProviderTypeLabelMap[ + value + ] + ) + ); + } form.setValue( "routingMode", "url" @@ -309,6 +317,27 @@ export default function CreateAiProviderPage() { /> + + ( + + + {t("name")} + + + + + + + )} + /> + + aiProviderTypeValues.map((type) => ({ type, - title: t(typeLabelMap[type]), + title: t(aiProviderTypeLabelMap[type]), description: t(typeDescriptionMap[type]) })), [t] From e100645a00e48a339a2c9cd9dfd31da8b8e1589c Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 13 Aug 2026 09:37:56 -0400 Subject: [PATCH 154/296] Use the right subnet --- server/routers/aiGateway/pipeline.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/routers/aiGateway/pipeline.ts b/server/routers/aiGateway/pipeline.ts index 3cc94e30e..50d5e98eb 100644 --- a/server/routers/aiGateway/pipeline.ts +++ b/server/routers/aiGateway/pipeline.ts @@ -123,7 +123,12 @@ async function findClientByIp(ip: string): Promise { const [client] = await db .select({ clientId: clients.clientId, userId: clients.userId }) .from(clients) - .where(eq(clients.exitNodeSubnet, `${ip}/32`)) + .where( + eq( + clients.exitNodeSubnet, + `${ip}/${config.getRawConfig().gerbil.site_block_size}` + ) + ) .limit(1); const result: CachedClient = client || null; @@ -323,6 +328,8 @@ async function resolveRequestUser( return { user: null, virtualApiKeyId: null }; } + logger.debug(`AI gateway request from IP ${ip} (site-resource)`); + const exitNodeRanges = await getExitNodeRanges(); const inExitNodeRange = exitNodeRanges.some((range) => isIpInCidr(ip, range) From c35ac493ed6bf98596ece8172f71b6b2071ddaf8 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 13 Aug 2026 09:38:22 -0400 Subject: [PATCH 155/296] Rebalance the data in the expand column --- src/app/[orgId]/settings/logs/ai/page.tsx | 30 +++-------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/src/app/[orgId]/settings/logs/ai/page.tsx b/src/app/[orgId]/settings/logs/ai/page.tsx index 8c077929b..512278a6c 100644 --- a/src/app/[orgId]/settings/logs/ai/page.tsx +++ b/src/app/[orgId]/settings/logs/ai/page.tsx @@ -551,27 +551,9 @@ export default function AiSessionLogsPage() {

- {t("virtualApiKey")} -

- {row.virtualApiKeyId ? ( - <> - {row.virtualApiKeyName ?? - t("aiUsageUnnamedVirtualApiKey")} - {row.virtualApiKeyLastChars && ( - <> - {" "} - ( - {formatVirtualApiKeyPreview( - row.virtualApiKeyId, - row.virtualApiKeyLastChars - )} - ) - - )} - - ) : ( - t("noVirtualApiKey") - )} + {t("totalTokens")} +

+ {row.usage.totalTokens.toLocaleString()}

@@ -608,12 +590,6 @@ export default function AiSessionLogsPage() { {row.usage.reasoningTokens.toLocaleString()}

-
- {t("totalTokens")} -

- {row.usage.totalTokens.toLocaleString()} -

-
)} From 9a551c91e53c6e18e87c9a0f88ccaa75b0fb6fc4 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Thu, 13 Aug 2026 10:07:01 -0400 Subject: [PATCH 156/296] always require upstream url --- messages/en-US.json | 2 +- .../[providerId]/network/page.tsx | 35 ++++--------------- .../settings/ai-providers/create/page.tsx | 23 +++--------- src/lib/aiProviderDefaults.ts | 9 ++--- src/lib/aiProviderFormSchema.ts | 11 ------ 5 files changed, 13 insertions(+), 67 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 660ffa5c9..e7b7f3c12 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1777,7 +1777,7 @@ "aiProviderTypeMicrosoftFoundryDescription": "Microsoft Foundry; upstream URL required", "aiProviderTypeOpenRouterDescription": "OpenRouter API", "aiProviderTypeVercelAiGatewayDescription": "Vercel AI Gateway", - "aiProviderTypeCustomDescription": "Bring your own OpenAI-compatible endpoint or route via Pangolin targets", + "aiProviderTypeCustomDescription": "Bring your own endpoint or route via site targets", "aiProviderUpstreamUrl": "Upstream URL", "aiProviderUpstreamUrlDescription": "Base URL for the provider API", "aiProviderUpstreamUrlOptionalDescription": "Leave blank to use the default upstream URL for this provider", diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx index e685bc560..fafbdf49f 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx @@ -41,7 +41,6 @@ import { createAiProviderFormSchema, showsUpstreamUrlField, toAiProviderNetworkPayload, - upstreamUrlRequired, type AiProviderFormValues } from "@app/lib/aiProviderFormSchema"; import { aiProviderQueries } from "@app/lib/queries"; @@ -91,7 +90,6 @@ export default function AiProviderNetworkPage() { const providerType = form.watch("type"); const routingMode = form.watch("routingMode"); const showUpstream = showsUpstreamUrlField(providerType, routingMode); - const requireUpstream = upstreamUrlRequired(providerType, routingMode); const showRoutingMode = providerType === "custom"; const isTargetModeSelected = routingMode === "target"; const isTargetModeSaved = @@ -218,22 +216,9 @@ export default function AiProviderNetworkPage() { field.value ?? "url" } - onChange={( - value - ) => { - field.onChange( - value - ); - if ( - value === - "target" - ) { - form.setValue( - "upstreamUrl", - "" - ); - } - }} + onChange={ + field.onChange + } /> @@ -284,14 +269,10 @@ export default function AiProviderNetworkPage() { {t( "aiProviderUpstreamUrl" )} - {requireUpstream - ? "" - : " (optional)"} - {requireUpstream - ? t( - "aiProviderUpstreamUrlDescription" - ) - : t( - "aiProviderUpstreamUrlOptionalDescription" - )} + {t( + "aiProviderUpstreamUrlDescription" + )} diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 6b47146a5..c9125659d 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -49,7 +49,6 @@ import { emptyUpstreamForType, showsUpstreamUrlField, toAiProviderCreatePayload, - upstreamUrlRequired, type AiProviderFormValues } from "@app/lib/aiProviderFormSchema"; import { zodResolver } from "@hookform/resolvers/zod"; @@ -95,7 +94,6 @@ export default function CreateAiProviderPage() { const authType = form.watch("authType"); const showUpstream = showsUpstreamUrlField(providerType, routingMode); - const requireUpstream = upstreamUrlRequired(providerType, routingMode); const showRoutingMode = providerType === "custom"; const showTargets = providerType === "custom" && routingMode === "target"; const showApiKey = authTypeRequiresApiKey(authType ?? "bearer"); @@ -436,14 +434,9 @@ export default function CreateAiProviderPage() { value ); if ( - value === + value !== "target" ) { - form.setValue( - "upstreamUrl", - "" - ); - } else { targetsRef.current = []; } @@ -498,14 +491,10 @@ export default function CreateAiProviderPage() { {t( "aiProviderUpstreamUrl" )} - {requireUpstream - ? "" - : " (optional)"} - {requireUpstream - ? t( - "aiProviderUpstreamUrlDescription" - ) - : t( - "aiProviderUpstreamUrlOptionalDescription" - )} + {t( + "aiProviderUpstreamUrlDescription" + )} diff --git a/src/lib/aiProviderDefaults.ts b/src/lib/aiProviderDefaults.ts index da4c43739..96fbb5e18 100644 --- a/src/lib/aiProviderDefaults.ts +++ b/src/lib/aiProviderDefaults.ts @@ -85,13 +85,8 @@ export function providerRequiresUpstreamUrl( type: AiProviderType, routingMode: AiProviderRoutingMode = "url" ): boolean { - if (routingMode === "target") { - return false; - } - if (type === "custom") { - return true; - } - return AI_PROVIDER_DEFAULTS[type].upstreamUrl === null; + const mode = type === "custom" ? routingMode : "url"; + return mode !== "target"; } export function defaultsForProviderType( diff --git a/src/lib/aiProviderFormSchema.ts b/src/lib/aiProviderFormSchema.ts index e033c0185..a425fb0e3 100644 --- a/src/lib/aiProviderFormSchema.ts +++ b/src/lib/aiProviderFormSchema.ts @@ -148,17 +148,6 @@ export function emptyUpstreamForType(type: AiProviderType): string { export function showsUpstreamUrlField( type: AiProviderType, routingMode: "url" | "target" | undefined -): boolean { - const mode = type === "custom" ? (routingMode ?? "url") : "url"; - if (mode === "target") { - return false; - } - return true; -} - -export function upstreamUrlRequired( - type: AiProviderType, - routingMode: "url" | "target" | undefined ): boolean { const mode = type === "custom" ? (routingMode ?? "url") : "url"; return providerRequiresUpstreamUrl(type, mode); From 77e3422e88c38baa17db22b8dd00cf6aed6b19fe Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Thu, 13 Aug 2026 10:13:32 -0400 Subject: [PATCH 157/296] change provider descriptions and dont set default url for bedrock --- messages/en-US.json | 14 +++++++------- src/lib/aiProviderDefaults.ts | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index e7b7f3c12..25a94eb32 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1769,14 +1769,14 @@ "aiProviderTypeOpenRouter": "OpenRouter", "aiProviderTypeVercelAiGateway": "Vercel AI Gateway", "aiProviderTypeCustom": "Custom", - "aiProviderTypeOpenaiDescription": "OpenAI API with default upstream URL", - "aiProviderTypeAnthropicDescription": "Anthropic API with default upstream URL", - "aiProviderTypeGoogleGeminiDescription": "Google Gemini generateContent API", - "aiProviderTypeVertexAiDescription": "Google Vertex AI; upstream URL required", - "aiProviderTypeBedrockDescription": "Amazon Bedrock Runtime", - "aiProviderTypeMicrosoftFoundryDescription": "Microsoft Foundry; upstream URL required", + "aiProviderTypeOpenaiDescription": "OpenAI API", + "aiProviderTypeAnthropicDescription": "Anthropic API", + "aiProviderTypeGoogleGeminiDescription": "Google Gemini API", + "aiProviderTypeVertexAiDescription": "Google Vertex AI API", + "aiProviderTypeBedrockDescription": "Amazon Bedrock Runtime API", + "aiProviderTypeMicrosoftFoundryDescription": "Microsoft Foundry API", "aiProviderTypeOpenRouterDescription": "OpenRouter API", - "aiProviderTypeVercelAiGatewayDescription": "Vercel AI Gateway", + "aiProviderTypeVercelAiGatewayDescription": "Vercel AI Gateway API", "aiProviderTypeCustomDescription": "Bring your own endpoint or route via site targets", "aiProviderUpstreamUrl": "Upstream URL", "aiProviderUpstreamUrlDescription": "Base URL for the provider API", diff --git a/src/lib/aiProviderDefaults.ts b/src/lib/aiProviderDefaults.ts index 96fbb5e18..31499a19d 100644 --- a/src/lib/aiProviderDefaults.ts +++ b/src/lib/aiProviderDefaults.ts @@ -56,7 +56,7 @@ export const AI_PROVIDER_DEFAULTS: Record< capabilities: ["google_generate_content", "google_raw_predict"] }, bedrock: { - upstreamUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", + upstreamUrl: null, authType: "bearer", capabilities: ["bedrock_converse"] }, From a2f5d830d60a422faace722c57f9b2de32badba6 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 13 Aug 2026 10:20:23 -0400 Subject: [PATCH 158/296] Remove not null constraint where setting null --- server/db/pg/schema/schema.ts | 14 ++++++++------ server/db/sqlite/schema/schema.ts | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 8efebc619..50fa3753b 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1826,9 +1826,10 @@ export const aiUsageRecords = pgTable( orgId: varchar("orgId") .notNull() .references(() => orgs.orgId, { onDelete: "cascade" }), - providerId: integer("providerId") - .notNull() - .references(() => aiProviders.providerId, { onDelete: "set null" }), + providerId: integer("providerId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ), resourceId: integer("resourceId").references( () => resources.resourceId, { onDelete: "set null" } @@ -1943,9 +1944,10 @@ export const aiSessionLog = pgTable( orgId: varchar("orgId").references(() => orgs.orgId, { onDelete: "cascade" }), - providerId: integer("providerId") - .notNull() - .references(() => aiProviders.providerId, { onDelete: "set null" }), + providerId: integer("providerId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ), capability: varchar("capability").notNull(), resourceId: integer("resourceId").references( () => resources.resourceId, diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index e46aef8d2..628563ee3 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1814,9 +1814,10 @@ export const aiUsageRecords = sqliteTable( orgId: text("orgId") .notNull() .references(() => orgs.orgId, { onDelete: "cascade" }), - providerId: integer("providerId") - .notNull() - .references(() => aiProviders.providerId, { onDelete: "set null" }), + providerId: integer("providerId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ), resourceId: integer("resourceId").references( () => resources.resourceId, { onDelete: "set null" } @@ -1933,9 +1934,10 @@ export const aiSessionLog = sqliteTable( orgId: text("orgId").references(() => orgs.orgId, { onDelete: "cascade" }), - providerId: integer("providerId") - .notNull() - .references(() => aiProviders.providerId, { onDelete: "set null" }), + providerId: integer("providerId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ), capability: text("capability").notNull(), resourceId: integer("resourceId").references( () => resources.resourceId, From e92bb9043b7f1fd424f5f5c5ddbe0b48bfd7557f Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 13 Aug 2026 11:04:11 -0400 Subject: [PATCH 159/296] Fix issues with possible null providerId --- server/routers/auditLogs/queryAiSessionLog.ts | 13 +++++++-- .../auditLogs/queryAiUsageProviders.ts | 27 ++++++++++++++----- server/routers/auditLogs/types.ts | 2 +- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/server/routers/auditLogs/queryAiSessionLog.ts b/server/routers/auditLogs/queryAiSessionLog.ts index 437d74550..ac2d99b65 100644 --- a/server/routers/auditLogs/queryAiSessionLog.ts +++ b/server/routers/auditLogs/queryAiSessionLog.ts @@ -208,7 +208,14 @@ async function enrichWithDetails( type: aiProviders.type }) .from(aiProviders) - .where(inArray(aiProviders.providerId, providerIds)); + .where( + inArray( + aiProviders.providerId, + providerIds.filter( + (id): id is number => id !== null && id !== undefined + ) + ) + ); for (const p of providerDetails) { providerMap.set(p.providerId, { name: p.name, type: p.type }); @@ -336,7 +343,9 @@ async function enrichWithDetails( } return logs.map((log) => { - const provider = providerMap.get(log.providerId); + const provider = log.providerId + ? providerMap.get(log.providerId) + : null; let resourceId = log.resourceId; let resourceName: string | null = null; diff --git a/server/routers/auditLogs/queryAiUsageProviders.ts b/server/routers/auditLogs/queryAiUsageProviders.ts index 77df6330d..8536ad79f 100644 --- a/server/routers/auditLogs/queryAiUsageProviders.ts +++ b/server/routers/auditLogs/queryAiUsageProviders.ts @@ -93,9 +93,17 @@ async function query(data: Q) { const nameMap = new Map(); if (providerIds.length > 0) { const providerDetails = await db - .select({ providerId: aiProviders.providerId, name: aiProviders.name }) + .select({ + providerId: aiProviders.providerId, + name: aiProviders.name + }) .from(aiProviders) - .where(inArray(aiProviders.providerId, providerIds)); + .where( + inArray( + aiProviders.providerId, + providerIds.filter((id): id is number => id !== null) + ) + ); for (const p of providerDetails) { nameMap.set(p.providerId, p.name); } @@ -103,7 +111,7 @@ async function query(data: Q) { const topProviders = topProvidersRaw.map((r) => ({ providerId: r.providerId, - name: nameMap.get(r.providerId) ?? null, + name: r.providerId ? (nameMap.get(r.providerId) ?? null) : null, requests: r.requests, totalTokens: r.totalTokens, costUsd: r.costUsd @@ -119,7 +127,8 @@ async function query(data: Q) { registry.registerPath({ method: "get", path: "/org/{orgId}/logs/ai/usage/providers", - description: "Query the AI usage analytics provider breakdown for an organization", + description: + "Query the AI usage analytics provider breakdown for an organization", tags: [OpenAPITags.Logs], request: { query: aiUsageAnalyticsFiltersQuery, @@ -154,14 +163,20 @@ export async function queryAiUsageProviders( const parsedQuery = aiUsageAnalyticsFiltersQuery.safeParse(req.query); if (!parsedQuery.success) { return next( - createHttpError(HttpCode.BAD_REQUEST, fromError(parsedQuery.error)) + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error) + ) ); } const parsedParams = aiUsageAnalyticsParams.safeParse(req.params); if (!parsedParams.success) { return next( - createHttpError(HttpCode.BAD_REQUEST, fromError(parsedParams.error)) + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error) + ) ); } diff --git a/server/routers/auditLogs/types.ts b/server/routers/auditLogs/types.ts index dc8d4015f..21663f6ce 100644 --- a/server/routers/auditLogs/types.ts +++ b/server/routers/auditLogs/types.ts @@ -99,7 +99,7 @@ export type QueryAiSessionLogResponse = { id: number; sessionId: string; orgId: string | null; - providerId: number; + providerId: number | null; providerName: string | null; providerType: string | null; capability: string; From 1143404a65832ea5b84dbdabb056adbac89fab56 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 13 Aug 2026 19:15:23 +0200 Subject: [PATCH 160/296] =?UTF-8?q?=F0=9F=92=84alert=20rule=20popover?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../alert-rule-editor/AlertRuleFields.tsx | 67 +++++++++++++------ 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/src/components/alert-rule-editor/AlertRuleFields.tsx b/src/components/alert-rule-editor/AlertRuleFields.tsx index 7ae6d0a04..8780ea102 100644 --- a/src/components/alert-rule-editor/AlertRuleFields.tsx +++ b/src/components/alert-rule-editor/AlertRuleFields.tsx @@ -45,7 +45,14 @@ import { import { getUserDisplayName } from "@app/lib/getUserDisplayName"; import { orgQueries } from "@app/lib/queries"; import { useQuery } from "@tanstack/react-query"; -import { Bell, ChevronsUpDown, Globe, Plus, Trash2 } from "lucide-react"; +import { + Bell, + ChevronRightIcon, + ChevronsUpDown, + Globe, + Plus, + Trash2 +} from "lucide-react"; import { useTranslations } from "next-intl"; import { useEffect, useMemo, useRef, useState } from "react"; import type { Control, UseFormReturn } from "react-hook-form"; @@ -95,6 +102,7 @@ export function AddActionPanel({ const EXTERNAL_IDS = EXTERNAL_INTEGRATIONS.map((i) => i.id); const [selected, setSelected] = useState("notify"); + const [isPopoverOpen, setPopoverOpen] = useState(false); const isPremiumSelected = selected !== null && EXTERNAL_IDS.includes(selected as any); @@ -131,27 +139,48 @@ export function AddActionPanel({ if (!isBuiltInSelected) return; onAdd(selected as AlertRuleFormAction["type"]); setSelected(null); + setPopoverOpen(false); }; return ( -
- setSelected(v)} - /> - {isPremiumSelected && } - {!isPremiumSelected && ( - - )} +
+

Add new action

+ + + + + + setSelected(v)} + /> + + {isPremiumSelected && } + {!isPremiumSelected && ( + + )} + + + {/* */}
); } From 2878d5690cf10046960cc30b714b0572762de6d8 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 13 Aug 2026 19:19:55 +0200 Subject: [PATCH 161/296] =?UTF-8?q?=F0=9F=92=AC=20update=20texts=20for=20h?= =?UTF-8?q?eading=20&=20trigger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- messages/en-US.json | 2 ++ .../alert-rule-editor/AlertRuleFields.tsx | 14 +++----------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 05b40d27d..7b0f0b4b3 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1807,6 +1807,8 @@ "alertingHealthChecksBannerTitle": "Monitor Health & Resources", "alertingHealthChecksBannerDescription": "Health checks are HTTP or TCP monitors you define once. You can then use them as sources in alert rules so you get notified when a target becomes healthy or unhealthy. Health checks on resources also appear here.", "alertingTestRule": "Test Alert Rule", + "alertingAddActionHeading": "Add New Action", + "alertingSelectActionType": "Choose an action type", "alertingNoActionsTitle": "No actions configured", "alertingNoActionsSaveDescription": "Add at least one action so this rule can notify someone when it fires.", "alertingNoActionsTestDescription": "Add at least one action before you can test this rule.", diff --git a/src/components/alert-rule-editor/AlertRuleFields.tsx b/src/components/alert-rule-editor/AlertRuleFields.tsx index 8780ea102..53fa9cdc1 100644 --- a/src/components/alert-rule-editor/AlertRuleFields.tsx +++ b/src/components/alert-rule-editor/AlertRuleFields.tsx @@ -144,18 +144,11 @@ export function AddActionPanel({ return (
-

Add new action

+

{t("alertingAddActionHeading")}

- @@ -180,7 +173,6 @@ export function AddActionPanel({ )} - {/* */}
); } From 9bb413bb1e39cd8e9b9ed37e9827145e3aaa5bf0 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 13 Aug 2026 19:23:39 +0200 Subject: [PATCH 162/296] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/alert-rule-editor/AlertRuleFields.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/components/alert-rule-editor/AlertRuleFields.tsx b/src/components/alert-rule-editor/AlertRuleFields.tsx index 53fa9cdc1..147369a9e 100644 --- a/src/components/alert-rule-editor/AlertRuleFields.tsx +++ b/src/components/alert-rule-editor/AlertRuleFields.tsx @@ -160,8 +160,9 @@ export function AddActionPanel({ onChange={(v) => setSelected(v)} /> - {isPremiumSelected && } - {!isPremiumSelected && ( + {isPremiumSelected ? ( + + ) : ( +
+ ) : null} + {hasOverflow && listExpanded ? ( +
+ +
) : null}
- {isCollapsed ? ( -
- -
- ) : null} - {hasOverflow && listExpanded ? ( -
- -
- ) : null} -
+ )}
( - [] - ); + const [pendingBudgetRows, setPendingBudgetRows] = useState([]); const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); const [savingBudgets, setSavingBudgets] = useState(false); @@ -929,9 +923,7 @@ function EditModelCredenza({ items={[ { title: t("general"), href: "#" }, { - title: t( - "aiProviderModelsBudgetTab" - ), + title: t("aiProviderModelsBudgetTab"), href: "#" } ]} @@ -970,9 +962,7 @@ function EditModelCredenza({

) : (

- {t( - "aiProviderModelsBudgetUnsaved" - )} + {t("aiProviderModelsBudgetUnsaved")}

)}
diff --git a/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx b/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx index 51b718e88..f6d18b7cf 100644 --- a/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx +++ b/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx @@ -573,7 +573,6 @@ export function PolicyAuthStackSectionEdit({ className="text-primary hover:underline" > {inferenceResourceUrl} - ) } diff --git a/src/components/resource-target-address-item.tsx b/src/components/resource-target-address-item.tsx index 58ef3b33b..79f72369c 100644 --- a/src/components/resource-target-address-item.tsx +++ b/src/components/resource-target-address-item.tsx @@ -142,6 +142,8 @@ export function ResourceTargetAddressItem({ isHttp, allowedMethods = ["http", "https", "h2c"] }: ResourceTargetAddressItemProps) { + "use no memo"; + return (
From 9b10292e025a4d00c68b062799f4c9d7b065328d Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Thu, 13 Aug 2026 15:08:10 -0400 Subject: [PATCH 166/296] add model picker to create provider wizard --- messages/en-US.json | 1 + server/lib/aiModelCatalog.ts | 30 +++++ server/routers/aiProvider/index.ts | 1 + .../routers/aiProvider/listCatalogModels.ts | 35 +----- .../aiProvider/listCatalogModelsByType.ts | 81 +++++++++++++ server/routers/external.ts | 7 ++ .../ai-providers/[providerId]/models/page.tsx | 60 ++-------- .../settings/ai-providers/create/page.tsx | 107 ++++++++++++++++++ src/components/AiProviderModelsLists.tsx | 80 +++++++++++++ src/lib/queries.ts | 20 ++++ 10 files changed, 341 insertions(+), 81 deletions(-) create mode 100644 server/routers/aiProvider/listCatalogModelsByType.ts create mode 100644 src/components/AiProviderModelsLists.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 137440d42..a3bfe73b8 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1864,6 +1864,7 @@ "aiProviderErrorNoUpdate": "AI provider is not available to update", "aiProviderModels": "Models", "aiProviderModelsDescription": "Define allow and block lists for this provider. Requests must match an allow entry and must not match a block entry.", + "aiProviderCreateModelsDescription": "Choose which models this provider can serve. An empty allow list denies all traffic. You can add or change models later.", "aiProviderModelsPlaceholder": "Search models or type a custom key", "aiProviderModelsAllow": "Allow List", "aiProviderModelsAllowDescription": "Models that may be used through this provider. Empty means deny all.", diff --git a/server/lib/aiModelCatalog.ts b/server/lib/aiModelCatalog.ts index a74378e6b..5f2f7fb94 100644 --- a/server/lib/aiModelCatalog.ts +++ b/server/lib/aiModelCatalog.ts @@ -284,6 +284,36 @@ export class AiModelCatalog { export const aiModelCatalog = new AiModelCatalog(); +export function listCatalogModelsForType( + type: AiProviderType, + query?: string +): { model: string }[] { + const catalogProvider = getCatalogProviderForType(type); + + let models = catalogProvider + ? aiModelCatalog.list(catalogProvider).map((entry) => ({ + model: entry.model + })) + : []; + + if (query) { + const q = query.toLowerCase(); + models = models.filter((m) => m.model.toLowerCase().includes(q)); + } + + const seen = new Set(); + models = models.filter((m) => { + if (seen.has(m.model)) { + return false; + } + seen.add(m.model); + return true; + }); + + models.sort((a, b) => a.model.localeCompare(b.model)); + return models; +} + /** * Loads the AI model pricing catalog into memory and schedules periodic * background refreshes. Call once at server startup. diff --git a/server/routers/aiProvider/index.ts b/server/routers/aiProvider/index.ts index 835991fbd..32378d3d9 100644 --- a/server/routers/aiProvider/index.ts +++ b/server/routers/aiProvider/index.ts @@ -6,6 +6,7 @@ export * from "./deleteAiProvider"; export * from "./createAiModel"; export * from "./listAiModels"; export * from "./listCatalogModels"; +export * from "./listCatalogModelsByType"; export * from "./getAiModel"; export * from "./updateAiModel"; export * from "./deleteAiModel"; diff --git a/server/routers/aiProvider/listCatalogModels.ts b/server/routers/aiProvider/listCatalogModels.ts index 9a3241276..5320ada8a 100644 --- a/server/routers/aiProvider/listCatalogModels.ts +++ b/server/routers/aiProvider/listCatalogModels.ts @@ -8,10 +8,7 @@ import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; import { eq } from "drizzle-orm"; -import { - aiModelCatalog, - getCatalogProviderForType -} from "@server/lib/aiModelCatalog"; +import { listCatalogModelsForType } from "@server/lib/aiModelCatalog"; import type { AiProviderType } from "@server/lib/aiProviderDefaults"; import type { ListCatalogModelsResponse } from "@server/routers/aiProvider/types"; @@ -86,35 +83,11 @@ export async function listCatalogModels( ); } - const catalogProvider = getCatalogProviderForType( - provider.type as AiProviderType + const models = listCatalogModelsForType( + provider.type as AiProviderType, + parsedQuery.data.query ); - let models = catalogProvider - ? aiModelCatalog.list(catalogProvider).map((entry) => ({ - model: entry.model - })) - : []; - - const { query } = parsedQuery.data; - if (query) { - const q = query.toLowerCase(); - models = models.filter((m) => m.model.toLowerCase().includes(q)); - } - - // Deduplicate model keys (catalog may have duplicates after provider - // normalization, e.g. bedrock + bedrock_converse). - const seen = new Set(); - models = models.filter((m) => { - if (seen.has(m.model)) { - return false; - } - seen.add(m.model); - return true; - }); - - models.sort((a, b) => a.model.localeCompare(b.model)); - return response(res, { data: { models }, success: true, diff --git a/server/routers/aiProvider/listCatalogModelsByType.ts b/server/routers/aiProvider/listCatalogModelsByType.ts new file mode 100644 index 000000000..4fbca54e9 --- /dev/null +++ b/server/routers/aiProvider/listCatalogModelsByType.ts @@ -0,0 +1,81 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { listCatalogModelsForType } from "@server/lib/aiModelCatalog"; +import type { ListCatalogModelsResponse } from "@server/routers/aiProvider/types"; +import { aiProviderTypeSchema } from "@server/routers/aiProvider/validation"; + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +const querySchema = z.strictObject({ + type: aiProviderTypeSchema, + query: z.string().optional() +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/ai-catalog-models", + description: + "List known catalog models for an AI provider type. Used for model key suggestions before a provider exists.", + tags: [OpenAPITags.AiModel], + request: { + params: paramsSchema, + query: querySchema + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function listCatalogModelsByType( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedQuery = querySchema.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error).toString() + ) + ); + } + + const { type, query } = parsedQuery.data; + const models = listCatalogModelsForType(type, query); + + return response(res, { + data: { models }, + success: true, + error: false, + message: "Catalog models retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/external.ts b/server/routers/external.ts index c391665b9..17cf0d921 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1645,6 +1645,13 @@ authenticated.get( aiProvider.listCatalogModels ); +authenticated.get( + "/org/:orgId/ai-catalog-models", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.listAiModels), + aiProvider.listCatalogModelsByType +); + authenticated.get( "/ai-model/:modelId", verifyAiModelAccess, diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx index f8ace5cb2..b85c0076c 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx @@ -11,12 +11,11 @@ import { SettingsSectionTitle } from "@app/components/Settings"; import { - AiProviderModelListEditor, type AiProviderModelListItem, type ModelListType } from "@app/components/AiProviderModelListEditor"; +import { AiProviderModelsLists } from "@app/components/AiProviderModelsLists"; import { Button } from "@app/components/ui/button"; -import { Label } from "@app/components/ui/label"; import { useAiProviderContext } from "@app/hooks/useAiProviderContext"; import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; @@ -65,15 +64,6 @@ export default function AiProviderModelsPage() { [catalogQuery.data] ); - const allowExcludeKeys = useMemo( - () => new Set(blockItems.map((item) => item.modelKey)), - [blockItems] - ); - const blockExcludeKeys = useMemo( - () => new Set(allowItems.map((item) => item.modelKey)), - [allowItems] - ); - useEffect(() => { if (!modelsQuery.data) return; setAllowItems( @@ -232,45 +222,15 @@ export default function AiProviderModelsPage() { -
- - -

- {t("aiProviderModelsAllowDescription")} -

-
- -
- - -

- {t("aiProviderModelsBlockDescription")} -

-
+ diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 722874028..7684b2d48 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -21,6 +21,11 @@ import { import HeaderTitle from "@app/components/SettingsSectionTitle"; import { AiProviderAuthTypeSelect } from "@app/components/AiProviderAuthTypeSelect"; import { AiProviderCapabilitiesSelect } from "@app/components/AiProviderCapabilitiesSelect"; +import { + type AiProviderModelListItem, + type ModelListType +} from "@app/components/AiProviderModelListEditor"; +import { AiProviderModelsLists } from "@app/components/AiProviderModelsLists"; import { AiProviderTypeSelect, aiProviderTypeLabelMap @@ -53,7 +58,9 @@ import { } from "@app/lib/aiProviderFormSchema"; import { zodResolver } from "@hookform/resolvers/zod"; import { authTypeRequiresApiKey } from "@app/lib/aiProviderDefaults"; +import { aiProviderQueries } from "@app/lib/queries"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; +import { useQuery } from "@tanstack/react-query"; import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; import { useParams, useRouter } from "next/navigation"; @@ -69,6 +76,8 @@ export default function CreateAiProviderPage() { const t = useTranslations(); const [loading, setLoading] = useState(false); const [headersValid, setHeadersValid] = useState(true); + const [allowItems, setAllowItems] = useState([]); + const [blockItems, setBlockItems] = useState([]); const targetsRef = useRef([]); const formSchema = useMemo(() => createAiProviderCreateFormSchema(t), [t]); @@ -98,6 +107,17 @@ export default function CreateAiProviderPage() { const showTargets = providerType === "custom" && routingMode === "target"; const showApiKey = authTypeRequiresApiKey(authType ?? "bearer"); + const catalogQuery = useQuery( + aiProviderQueries.catalogModelsByType({ + orgId, + type: providerType + }) + ); + const catalogModels = useMemo( + () => (catalogQuery.data ?? []).map((entry) => entry.model), + [catalogQuery.data] + ); + async function createTargets( providerId: number, localTargets: LocalTarget[] @@ -135,6 +155,19 @@ export default function CreateAiProviderPage() { } } + async function createModels( + providerId: number, + items: { modelKey: string; listType: ModelListType }[] + ) { + for (const item of items) { + await api.put(`/ai-provider/${providerId}/model`, { + modelKey: item.modelKey, + name: item.modelKey, + listType: item.listType + }); + } + } + async function onSubmit(values: AiProviderFormValues) { const targets = targetsRef.current; @@ -157,6 +190,31 @@ export default function CreateAiProviderPage() { } } + const nextAllow = new Set( + allowItems.map((item) => item.modelKey.trim()).filter(Boolean) + ); + const nextBlock = new Set( + blockItems.map((item) => item.modelKey.trim()).filter(Boolean) + ); + const overlap = [...nextAllow].filter((key) => nextBlock.has(key)); + if (overlap.length > 0) { + toast({ + variant: "destructive", + title: t("aiProviderErrorCreate"), + description: t("aiProviderModelsOverlapError", { + keys: overlap.join(", ") + }) + }); + return; + } + + const modelItems = [...allowItems, ...blockItems] + .map((item) => ({ + modelKey: item.modelKey.trim(), + listType: item.listType + })) + .filter((item) => item.modelKey); + setLoading(true); try { const res = await api.put< @@ -184,6 +242,25 @@ export default function CreateAiProviderPage() { } } + if (modelItems.length > 0) { + try { + await createModels(providerId, modelItems); + } catch (e) { + toast({ + variant: "destructive", + title: t("aiProviderErrorCreate"), + description: formatAxiosError( + e, + t("aiProviderErrorCreate") + ) + }); + router.push( + `/${orgId}/settings/ai-providers/${providerId}/models` + ); + return; + } + } + toast({ title: t("success"), description: t("aiProviderCreated") @@ -253,6 +330,12 @@ export default function CreateAiProviderPage() { field.onChange( value ); + setAllowItems( + [] + ); + setBlockItems( + [] + ); form.setValue( "upstreamUrl", emptyUpstreamForType( @@ -689,6 +772,30 @@ export default function CreateAiProviderPage() { + + + + + {t("aiProviderModels")} + + + {t("aiProviderCreateModelsDescription")} + + + + + + + + +
diff --git a/src/components/AiProviderModelsLists.tsx b/src/components/AiProviderModelsLists.tsx new file mode 100644 index 000000000..8ed7f6fef --- /dev/null +++ b/src/components/AiProviderModelsLists.tsx @@ -0,0 +1,80 @@ +"use client"; + +import { + AiProviderModelListEditor, + type AiProviderModelListItem +} from "@app/components/AiProviderModelListEditor"; +import { Label } from "@app/components/ui/label"; +import { useTranslations } from "next-intl"; +import { useMemo } from "react"; + +export type AiProviderModelsListsProps = { + orgId: string; + allowItems: AiProviderModelListItem[]; + onAllowChange: (items: AiProviderModelListItem[]) => void; + blockItems: AiProviderModelListItem[]; + onBlockChange: (items: AiProviderModelListItem[]) => void; + catalogModels: string[]; + disabled?: boolean; +}; + +export function AiProviderModelsLists({ + orgId, + allowItems, + onAllowChange, + blockItems, + onBlockChange, + catalogModels, + disabled +}: AiProviderModelsListsProps) { + const t = useTranslations(); + + const allowExcludeKeys = useMemo( + () => new Set(blockItems.map((item) => item.modelKey)), + [blockItems] + ); + const blockExcludeKeys = useMemo( + () => new Set(allowItems.map((item) => item.modelKey)), + [allowItems] + ); + + return ( +
+
+ + +

+ {t("aiProviderModelsAllowDescription")} +

+
+ +
+ + +

+ {t("aiProviderModelsBlockDescription")} +

+
+
+ ); +} diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 56a41cee8..c58c6c567 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -75,6 +75,7 @@ import type { ListAiProvidersResponse, ListCatalogModelsResponse } from "@server/routers/aiProvider/types"; +import type { AiProviderType } from "@app/lib/aiProviderDefaults"; import type { ListAiBudgetsByScopeResponse } from "@server/routers/aiBudget/types"; import { getAiBudgetScopeListPath, @@ -1495,6 +1496,25 @@ export const aiProviderQueries = { return res.data.data.models; } }), + catalogModelsByType: ({ + orgId, + type + }: { + orgId: string; + type: AiProviderType; + }) => + queryOptions({ + queryKey: ["AI_PROVIDERS", orgId, "CATALOG_MODELS", type] as const, + queryFn: async ({ signal, meta }) => { + const res = await meta!.api.get< + AxiosResponse + >(`/org/${orgId}/ai-catalog-models`, { + params: { type }, + signal + }); + return res.data.data.models; + } + }), orgProviders: ({ orgId, query }: { orgId: string; query?: string }) => queryOptions({ queryKey: ["AI_PROVIDERS", orgId, "LIST", query ?? ""] as const, From 4a9b0117a2b34528dfb3f937cd3cda60da93ddc5 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Thu, 13 Aug 2026 15:16:39 -0400 Subject: [PATCH 167/296] copy key without clicking reveal first --- src/components/CopyTextBox.tsx | 34 ++++-- src/components/CopyToClipboard.tsx | 37 ++++-- src/components/UserVirtualApiKeys.tsx | 89 +++------------ .../LauncherInferenceApiKeysSection.tsx | 76 ++----------- src/hooks/useMyVirtualApiKeySecret.ts | 107 ++++++++++++++++++ 5 files changed, 183 insertions(+), 160 deletions(-) create mode 100644 src/hooks/useMyVirtualApiKeySecret.ts diff --git a/src/components/CopyTextBox.tsx b/src/components/CopyTextBox.tsx index e516bf0d6..5f12a7f39 100644 --- a/src/components/CopyTextBox.tsx +++ b/src/components/CopyTextBox.tsx @@ -8,29 +8,40 @@ import { useTranslations } from "next-intl"; type CopyTextBoxProps = { text?: string; displayText?: string; + getCopyText?: () => Promise; wrapText?: boolean; outline?: boolean; + centered?: boolean; }; export default function CopyTextBox({ text = "", displayText, + getCopyText, wrapText = false, - outline = true + outline = true, + centered = false }: CopyTextBoxProps) { const [isCopied, setIsCopied] = useState(false); + const [isCopying, setIsCopying] = useState(false); const textRef = useRef(null); const t = useTranslations(); const copyToClipboard = async () => { - if (textRef.current) { - try { - await navigator.clipboard.writeText(text); - setIsCopied(true); - setTimeout(() => setIsCopied(false), 2000); - } catch (err) { - console.error(t("copyTextFailed"), err); - } + if (!textRef.current || isCopying) { + return; + } + + setIsCopying(true); + try { + const value = getCopyText ? await getCopyText() : text; + await navigator.clipboard.writeText(value); + setIsCopied(true); + setTimeout(() => setIsCopied(false), 2000); + } catch (err) { + console.error(t("copyTextFailed"), err); + } finally { + setIsCopying(false); } }; @@ -40,7 +51,9 @@ export default function CopyTextBox({ > 
                 {isCopied ? (
diff --git a/src/components/CopyToClipboard.tsx b/src/components/CopyToClipboard.tsx
index 147972e78..344a2996a 100644
--- a/src/components/CopyToClipboard.tsx
+++ b/src/components/CopyToClipboard.tsx
@@ -1,5 +1,5 @@
 import { cn } from "@app/lib/cn";
-import { Check, Copy } from "lucide-react";
+import { Check, Copy, Loader2 } from "lucide-react";
 import Link from "next/link";
 import { useState } from "react";
 import { useTranslations } from "next-intl";
@@ -7,6 +7,7 @@ import { useTranslations } from "next-intl";
 type CopyToClipboardProps = {
     text: string;
     displayText?: string;
+    getCopyText?: () => Promise;
     isLink?: boolean;
     className?: string;
 };
@@ -14,18 +15,31 @@ type CopyToClipboardProps = {
 const CopyToClipboard = ({
     text,
     displayText,
+    getCopyText,
     isLink,
     className
 }: CopyToClipboardProps) => {
     const [copied, setCopied] = useState(false);
+    const [copying, setCopying] = useState(false);
 
-    const handleCopy = () => {
-        navigator.clipboard.writeText(text);
-        setCopied(true);
+    const handleCopy = async () => {
+        if (copying) {
+            return;
+        }
 
-        setTimeout(() => {
-            setCopied(false);
-        }, 2000);
+        setCopying(true);
+        try {
+            const value = getCopyText ? await getCopyText() : text;
+            await navigator.clipboard.writeText(value);
+            setCopied(true);
+            setTimeout(() => {
+                setCopied(false);
+            }, 2000);
+        } catch {
+            // Fetch errors are toasted by the caller; clipboard failures stay silent.
+        } finally {
+            setCopying(false);
+        }
     };
 
     const displayValue = displayText ?? text;
@@ -38,11 +52,14 @@ const CopyToClipboard = ({
                 type="button"
                 className="h-4 w-4 p-0 flex items-center justify-center cursor-pointer flex-shrink-0"
                 onClick={handleCopy}
+                disabled={copying}
             >
-                {!copied ? (
-                    
-                ) : (
+                {copying ? (
+                    
+                ) : copied ? (
                     
+                ) : (
+                    
                 )}
                 {t("copyText")}
             
diff --git a/src/components/UserVirtualApiKeys.tsx b/src/components/UserVirtualApiKeys.tsx
index 578f007b7..1ab2ae92d 100644
--- a/src/components/UserVirtualApiKeys.tsx
+++ b/src/components/UserVirtualApiKeys.tsx
@@ -1,8 +1,6 @@
 "use client";
 
-import { useState } from "react";
 import { useTranslations } from "next-intl";
-import { AxiosResponse } from "axios";
 import moment from "moment";
 import { Button } from "@app/components/ui/button";
 import CopyTextBox from "@app/components/CopyTextBox";
@@ -17,73 +15,18 @@ import {
     SettingsSectionHeader,
     SettingsSectionTitle as SectionTitle
 } from "@app/components/Settings";
-import { createApiClient, formatAxiosError } from "@app/lib/api";
-import { useEnvContext } from "@app/hooks/useEnvContext";
-import { toast } from "@app/hooks/useToast";
+import { useMyVirtualApiKeySecret } from "@app/hooks/useMyVirtualApiKeySecret";
 import type {
-    GetMyVirtualApiKeyResponse,
     ListMyVirtualApiKeysResponse,
     VirtualApiKeyWithResources
 } from "@server/routers/virtualApiKey/types";
-import {
-    formatVirtualApiKeyCredential,
-    formatVirtualApiKeyPreview
-} from "@app/lib/virtualApiKeyFormat";
+import { formatVirtualApiKeyPreview } from "@app/lib/virtualApiKeyFormat";
 
 type UserVirtualApiKeysProps = {
     orgId: string;
     initialData: ListMyVirtualApiKeysResponse;
 };
 
-function useRevealSecret(orgId: string, virtualApiKeyId: string) {
-    const t = useTranslations();
-    const api = createApiClient(useEnvContext());
-    const [credential, setCredential] = useState(null);
-    const [loading, setLoading] = useState(false);
-
-    const revealSecret = () => {
-        if (credential || loading) {
-            return;
-        }
-
-        setLoading(true);
-        api.get>(
-            `/org/${orgId}/my-virtual-api-keys/${virtualApiKeyId}`
-        )
-            .then((res) => {
-                const secret = res.data.data.virtualApiKey.secret;
-                if (secret) {
-                    setCredential(
-                        formatVirtualApiKeyCredential(virtualApiKeyId, secret)
-                    );
-                } else {
-                    toast({
-                        variant: "destructive",
-                        title: t("virtualApiKeysErrorFetchSecret"),
-                        description: t(
-                            "virtualApiKeysErrorFetchSecretDescription"
-                        )
-                    });
-                }
-            })
-            .catch((e) => {
-                toast({
-                    variant: "destructive",
-                    title: t("virtualApiKeysErrorFetchSecret"),
-                    description: formatAxiosError(
-                        e,
-                        t("virtualApiKeysErrorFetchSecretDescription")
-                    )
-                });
-            })
-            .finally(() => {
-                setLoading(false);
-            });
-    };
-
-    return { credential, loading, revealSecret };
-}
-
 function OwnedKeySecret({
     orgId,
     virtualApiKeyId,
@@ -95,11 +38,9 @@ function OwnedKeySecret({
 }) {
     const t = useTranslations();
     const preview = formatVirtualApiKeyPreview(virtualApiKeyId, lastChars);
-    const { credential, loading, revealSecret } = useRevealSecret(
-        orgId,
-        virtualApiKeyId
-    );
-    const displayValue = credential ?? preview;
+    const { credential, revealed, loading, getCopyText, revealSecret } =
+        useMyVirtualApiKeySecret(orgId, virtualApiKeyId);
+    const displayValue = revealed && credential ? credential : preview;
 
     return (
         

@@ -107,9 +48,10 @@ function OwnedKeySecret({
                 
             

-            {!credential ? (
+            {!revealed ? (
                 
+                
+                
+                    
+                        
+                        
+                            {emptyMessage}
+                            
+                                {options.map((option) => (
+                                     {
+                                            onChange(option.value);
+                                            setOpen(false);
+                                        }}
+                                    >
+                                        
+                                        

+                                            
+                                                {option.title}
+                                            
+                                            
+                                                {option.description}
+                                            
+                                        

+                                    
+                                ))}
+                            
+                        
+                    
+                
+            
+
+ ); +} diff --git a/src/components/DomainPicker.tsx b/src/components/DomainPicker.tsx index bf25fcfce..957d759eb 100644 --- a/src/components/DomainPicker.tsx +++ b/src/components/DomainPicker.tsx @@ -516,7 +516,9 @@ export default function DomainPicker({ return ( - {t("domainPickerNoDomainsAvailableTitle")} + + {t("domainPickerNoDomainsAvailableTitle")} +

{t("domainPickerNoDomainsAvailableDescription")}

+
+ ); + })} +
)} + {errorMessage} {rows.length > 0 && addRowButton}
); @@ -474,12 +519,16 @@ export function BudgetsEditor({ const body = ( <> - + + + + + diff --git a/src/components/CreateVirtualApiKeyForm.tsx b/src/components/CreateVirtualApiKeyForm.tsx index c443ad4b9..4a97d358a 100644 --- a/src/components/CreateVirtualApiKeyForm.tsx +++ b/src/components/CreateVirtualApiKeyForm.tsx @@ -99,9 +99,7 @@ export default function CreateVirtualApiKeyForm({ const [selectedResources, setSelectedResources] = useState< SelectedResource[] >([]); - const [pendingBudgetRows, setPendingBudgetRows] = useState( - [] - ); + const [pendingBudgetRows, setPendingBudgetRows] = useState([]); const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); const formSchema = z.object({ @@ -204,10 +202,7 @@ export default function CreateVirtualApiKeyForm({ toast({ variant: "destructive", title: t("aiBudgetErrorSave"), - description: formatAxiosError( - e, - t("aiBudgetErrorSave") - ) + description: formatAxiosError(e, t("aiBudgetErrorSave")) }); } } @@ -295,120 +290,47 @@ export default function CreateVirtualApiKeyForm({ } ]} > -
- ( - - - {t("virtualApiKeysName")} - - - - - - - )} - /> - - ( - - - {t( - "virtualApiKeysDescriptionOptional" - )} - - - - - - - )} - /> - -
- - {t( - "virtualApiKeysAssociateUserOptional" - )} - - - - - - - - - -

- {t( - "virtualApiKeysAssociateUserDescription" - )} -

-
- -
-
- { - setAllResources( - val as boolean - ); - if (val) { - setSelectedResources( - [] - ); - } - }} - className="mt-0.5" +
+ ( + + + {t( + "virtualApiKeysName" + )} + + + + + + + )} + /> + + ( + + + {t( + "virtualApiKeysDescriptionOptional" + )} + + + + + + + )} /> -
- -

- {t( - "virtualApiKeysAllResourcesDescription" - )} -

-
-
- {!allResources && (
{t( - "virtualApiKeysSelectResources" + "virtualApiKeysAssociateUserOptional" )} @@ -418,69 +340,155 @@ export default function CreateVirtualApiKeyForm({ role="combobox" className={cn( "w-full justify-between", - selectedResources.length === - 0 && + !selectedUser && "text-muted-foreground" )} > - - {formatMultiResourcesSelectorLabel( - selectedResources, - t, - "virtualApiKeysSelectResourcesPlaceholder" - )} - + {selectedUser?.text + ? selectedUser.text + : t( + "userSelect" + )} - - + - 0 - } - onClear={() => - setSelectedResources( - [] - ) + onSelectUser={ + setSelectedUser } /> - +

{t( - "virtualApiKeysSelectResourcesDescription" + "virtualApiKeysAssociateUserDescription" )} - +

- )} -
-
-
-

- {t( - "virtualApiKeysInferenceBudgetDescription" - )} -

- -
+
+
+ { + setAllResources( + val as boolean + ); + if (val) { + setSelectedResources( + [] + ); + } + }} + className="mt-0.5" + /> +
+ +

+ {t( + "virtualApiKeysAllResourcesDescription" + )} +

+
+
+ + {!allResources && ( +
+ + {t( + "virtualApiKeysSelectResources" + )} + + + + + + + + 0 + } + onClear={() => + setSelectedResources( + [] + ) + } + /> + + + + {t( + "virtualApiKeysSelectResourcesDescription" + )} + +
+ )} +
+
+ +
+

+ {t( + "virtualApiKeysInferenceBudgetDescription" + )} +

+ +
diff --git a/src/components/EditVirtualApiKeyForm.tsx b/src/components/EditVirtualApiKeyForm.tsx index 1d3e78dbd..b8fbcac98 100644 --- a/src/components/EditVirtualApiKeyForm.tsx +++ b/src/components/EditVirtualApiKeyForm.tsx @@ -112,9 +112,7 @@ export default function EditVirtualApiKeyForm({ >([]); const [credential, setCredential] = useState(null); const [credentialLoading, setCredentialLoading] = useState(false); - const [pendingBudgetRows, setPendingBudgetRows] = useState( - [] - ); + const [pendingBudgetRows, setPendingBudgetRows] = useState([]); const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); const budgetScope = { @@ -350,9 +348,7 @@ export default function EditVirtualApiKeyForm({
@@ -369,99 +365,11 @@ export default function EditVirtualApiKeyForm({ } ]} > -
-
- - - - - - - - - -

- {t( - "virtualApiKeysAssociateUserDescription" - )} -

-
- -
- ( - -
- - { - field.onChange( - val as boolean - ); - if (val) { - setSelectedResources( - [] - ); - } - }} - className="mt-0.5" - /> - -
- -

- {t( - "virtualApiKeysAllResourcesDescription" - )} -

-
-
- - - )} - /> - - {!allResources && ( +
@@ -471,66 +379,163 @@ export default function EditVirtualApiKeyForm({ role="combobox" className={cn( "w-full justify-between", - selectedResources.length === - 0 && + !selectedUser && "text-muted-foreground" )} > - - {formatMultiResourcesSelectorLabel( - selectedResources, - t, - "virtualApiKeysSelectResourcesPlaceholder" - )} - + {selectedUser?.text + ? selectedUser.text + : t("userSelect")} - - + - 0 - } - onClear={() => - setSelectedResources( - [] - ) + onSelectUser={ + setSelectedUser } /> - +

{t( - "virtualApiKeysSelectResourcesRequired" + "virtualApiKeysAssociateUserDescription" )} - +

- )} -
-
-
-

- {t( - "virtualApiKeysInferenceBudgetDescription" - )} -

- -
+
+ ( + +
+ + { + field.onChange( + val as boolean + ); + if ( + val + ) { + setSelectedResources( + [] + ); + } + }} + className="mt-0.5" + /> + +
+ +

+ {t( + "virtualApiKeysAllResourcesDescription" + )} +

+
+
+ + + )} + /> + + {!allResources && ( +
+ + + + + + + + 0 + } + onClear={() => + setSelectedResources( + [] + ) + } + /> + + + + {t( + "virtualApiKeysSelectResourcesRequired" + )} + +
+ )} +
+
+ +
+

+ {t( + "virtualApiKeysInferenceBudgetDescription" + )} +

+ +
diff --git a/src/components/RoleForm.tsx b/src/components/RoleForm.tsx index 4e8f457cb..186da2a00 100644 --- a/src/components/RoleForm.tsx +++ b/src/components/RoleForm.tsx @@ -219,9 +219,7 @@ export function RoleForm({ useState(null); const [dragOverField, setDragOverField] = useState(null); - const [pendingBudgetRows, setPendingBudgetRows] = useState( - [] - ); + const [pendingBudgetRows, setPendingBudgetRows] = useState([]); const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); const budgetsQuery = useQuery({ @@ -702,7 +700,6 @@ export function RoleForm({
)} - {/* Inference Budget tab */}

{t("accessRoleInferenceBudgetDescription")} @@ -711,8 +708,7 @@ export function RoleForm({ rows={pendingBudgetRows} onChange={setPendingBudgetRows} disabled={ - variant === "edit" && - budgetsQuery.isLoading + variant === "edit" && budgetsQuery.isLoading } attemptedSave={attemptedBudgetsSave} /> From ee75a09f8cea537e80201ca301cf1df9630c2f10 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 14 Aug 2026 10:33:44 -0400 Subject: [PATCH 179/296] improve sign in url style --- messages/en-US.json | 4 +-- .../resource-policy/PolicyAuthSsoSection.tsx | 34 ++++++++++++++++--- .../PolicyAuthStackSectionEdit.tsx | 34 ++----------------- 3 files changed, 33 insertions(+), 39 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 363a2a99d..c3a6a235c 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -878,9 +878,9 @@ "policyAuthMethodOff": "Off", "policyAuthSsoTitle": "Platform SSO", "policyAuthSsoDescription": "Require sign-in through your organization's identity provider", - "policyAuthInferenceSsoDescription": "Selected users and roles can authenticate to the gateway using their identity API key.", + "policyAuthInferenceIdentityKeySignInUrl": "Sign-in URL", "policyAuthInferenceIdentityKeyHelpNoUrl": "Every user already has an identity API key, so you only need to create virtual API keys for non-user clients or shared access. Users can retrieve their key by signing in with their identity provider at this resource's URL, where it will be shown after login.", - "policyAuthInferenceIdentityKeyHelp": "Every user already has an identity API key, so you only need to create virtual API keys for non-user clients or shared access. Users can retrieve their key by signing in with their identity provider at , where it will be shown after login.", + "policyAuthInferenceIdentityKeyHelp": "Every user already has an identity API key, so you only need to create virtual API keys for non-user clients or shared access. Users retrieve their key here after signing in with their identity provider.", "policyAuthSsoSummary": "{idp} · {users} users, {roles} roles", "policyAuthSsoDefaultIdp": "Default provider", "policyAuthAddDefaultIdentityProvider": "Add Default Identity Provider", diff --git a/src/components/resource-policy/PolicyAuthSsoSection.tsx b/src/components/resource-policy/PolicyAuthSsoSection.tsx index 3cc2e54c4..c8ce238bf 100644 --- a/src/components/resource-policy/PolicyAuthSsoSection.tsx +++ b/src/components/resource-policy/PolicyAuthSsoSection.tsx @@ -1,5 +1,6 @@ "use client"; +import CopyToClipboard from "@app/components/CopyToClipboard"; import { SettingsFormCell, SettingsFormGrid } from "@app/components/Settings"; import { SwitchInput } from "@app/components/SwitchInput"; import { Button } from "@app/components/ui/button"; @@ -32,6 +33,7 @@ export type PolicyAuthSsoSectionProps = { ssoLocked?: boolean; title?: string; description?: string; + identityKeyUrl?: string | null; }; export function PolicyAuthSsoSection({ @@ -46,7 +48,8 @@ export function PolicyAuthSsoSection({ idpDisabled, ssoLocked, title, - description + description, + identityKeyUrl }: PolicyAuthSsoSectionProps) { const t = useTranslations(); const [showIdpSelect, setShowIdpSelect] = useState(skipToIdpId != null); @@ -79,11 +82,32 @@ export function PolicyAuthSsoSection({ {ssoActive && ( <> - {ssoLocked && ssoDescription && ( + {ssoLocked && identityKeyUrl !== undefined && ( -

- {ssoDescription} -

+ {identityKeyUrl ? ( + + + {t( + "policyAuthInferenceIdentityKeySignInUrl" + )} + + + + {t( + "policyAuthInferenceIdentityKeyHelp" + )} + + + ) : ( +

+ {t( + "policyAuthInferenceIdentityKeyHelpNoUrl" + )} +

+ )} )} diff --git a/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx b/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx index f6d18b7cf..4a4734200 100644 --- a/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx +++ b/src/components/resource-policy/PolicyAuthStackSectionEdit.tsx @@ -15,7 +15,6 @@ import { } from "@app/components/roles-selector"; import { UsersSelector } from "@app/components/users-selector"; import { Button } from "@app/components/ui/button"; -import { Alert, AlertDescription } from "@app/components/ui/alert"; import { Form, FormField } from "@app/components/ui/form"; import { toast } from "@app/hooks/useToast"; import { useEnvContext } from "@app/hooks/useEnvContext"; @@ -29,7 +28,6 @@ import type { GetResourcePolicyResponse } from "@server/routers/policy"; import { UserType } from "@server/types/UserTypes"; import { useQuery } from "@tanstack/react-query"; import type { AxiosResponse } from "axios"; -import { ExternalLink, InfoIcon } from "lucide-react"; import { useTranslations } from "next-intl"; import { useRouter } from "next/navigation"; import { toUnicode } from "punycode"; @@ -555,34 +553,6 @@ export function PolicyAuthStackSectionEdit({ {isResourceOverlay && ( )} - {isInferenceResource && ( - - - - {inferenceResourceUrl - ? t.rich( - "policyAuthInferenceIdentityKeyHelp", - { - resourceLink: () => ( - - {inferenceResourceUrl} - - ) - } - ) - : t( - "policyAuthInferenceIdentityKeyHelpNoUrl" - )} - - - )} Date: Fri, 14 Aug 2026 09:30:44 -0400 Subject: [PATCH 180/296] add niceId to provider --- server/db/names.ts | 27 +++++ server/db/pg/schema/schema.ts | 97 +++++++++--------- server/db/sqlite/schema/schema.ts | 99 ++++++++++--------- server/routers/aiProvider/createAiProvider.ts | 4 + server/routers/aiProvider/updateAiProvider.ts | 37 ++++++- 5 files changed, 171 insertions(+), 93 deletions(-) diff --git a/server/db/names.ts b/server/db/names.ts index ebe38573d..37e851379 100644 --- a/server/db/names.ts +++ b/server/db/names.ts @@ -1,6 +1,7 @@ import { join } from "path"; import { readFileSync } from "fs"; import { + aiProviders, clients, db, resourcePolicies, @@ -113,6 +114,32 @@ export async function getUniqueResourceName(orgId: string): Promise { } } +export async function getUniqueProviderName(orgId: string): Promise { + let loops = 0; + while (true) { + if (loops > 100) { + throw new Error("Could not generate a unique name"); + } + + const name = generateName(); + + const aiProviderCount = await db + .select({ + niceId: aiProviders.niceId, + orgId: aiProviders.orgId + }) + .from(aiProviders) + .where( + and(eq(aiProviders.niceId, name), eq(aiProviders.orgId, orgId)) + ); + + if (aiProviderCount.length === 0) { + return name; + } + loops++; + } +} + export async function getUniqueResourcePolicyName( orgId: string ): Promise { diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 50fa3753b..432afbbc1 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1684,52 +1684,57 @@ export const statusHistory = pgTable( ] ); -export const aiProviders = pgTable("aiProviders", { - providerId: serial("providerId").primaryKey(), - orgId: varchar("orgId") - .notNull() - .references(() => orgs.orgId, { onDelete: "cascade" }), - name: varchar("name").notNull(), - type: varchar("type") - .$type< - | "openai" - | "anthropic" - | "googleGemini" - | "vertexAi" - | "bedrock" - | "microsoftFoundry" - | "openRouter" - | "vercelAiGateway" - | "custom" - >() - .notNull(), - upstreamUrl: text("upstreamUrl"), - apiKey: text("apiKey"), - apiKeyLastChars: varchar("apiKeyLastChars"), - authType: varchar("authType") - .$type< - | "bearer" - | "x-api-key" - | "x-goog-api-key" - | "hec" - | "cf-aig-authorization" - | "none" - | "passthrough" - >() - .notNull(), - routingMode: varchar("routingMode") - .$type<"url" | "target">() - .notNull() - .default("url"), - capabilities: text("capabilities").notNull().default("[]"), - headers: text("headers"), // JSON array of { name, value } - skipTlsVerification: boolean("skipTlsVerification") - .notNull() - .default(false), - enabled: boolean("enabled").notNull().default(true), - createdAt: bigint("createdAt", { mode: "number" }).notNull(), - updatedAt: bigint("updatedAt", { mode: "number" }).notNull() -}); +export const aiProviders = pgTable( + "aiProviders", + { + providerId: serial("providerId").primaryKey(), + orgId: varchar("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + name: varchar("name").notNull(), + niceId: varchar("niceId").notNull(), + type: varchar("type") + .$type< + | "openai" + | "anthropic" + | "googleGemini" + | "vertexAi" + | "bedrock" + | "microsoftFoundry" + | "openRouter" + | "vercelAiGateway" + | "custom" + >() + .notNull(), + upstreamUrl: text("upstreamUrl"), + apiKey: text("apiKey"), + apiKeyLastChars: varchar("apiKeyLastChars"), + authType: varchar("authType") + .$type< + | "bearer" + | "x-api-key" + | "x-goog-api-key" + | "hec" + | "cf-aig-authorization" + | "none" + | "passthrough" + >() + .notNull(), + routingMode: varchar("routingMode") + .$type<"url" | "target">() + .notNull() + .default("url"), + capabilities: text("capabilities").notNull().default("[]"), + headers: text("headers"), // JSON array of { name, value } + skipTlsVerification: boolean("skipTlsVerification") + .notNull() + .default(false), + enabled: boolean("enabled").notNull().default(true), + createdAt: bigint("createdAt", { mode: "number" }).notNull(), + updatedAt: bigint("updatedAt", { mode: "number" }).notNull() + }, + (t) => [index("idx_aiProviders_orgId_niceId").on(t.orgId, t.niceId)] +); export const aiModels = pgTable( "aiModels", diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 628563ee3..c70424152 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1668,52 +1668,59 @@ export const statusHistory = sqliteTable( ] ); -export const aiProviders = sqliteTable("aiProviders", { - providerId: integer("providerId").primaryKey({ autoIncrement: true }), - orgId: text("orgId") - .notNull() - .references(() => orgs.orgId, { onDelete: "cascade" }), - name: text("name").notNull(), - type: text("type") - .$type< - | "openai" - | "anthropic" - | "googleGemini" - | "vertexAi" - | "bedrock" - | "microsoftFoundry" - | "openRouter" - | "vercelAiGateway" - | "custom" - >() - .notNull(), - upstreamUrl: text("upstreamUrl"), - apiKey: text("apiKey"), - apiKeyLastChars: text("apiKeyLastChars"), - authType: text("authType") - .$type< - | "bearer" - | "x-api-key" - | "x-goog-api-key" - | "hec" - | "cf-aig-authorization" - | "none" - | "passthrough" - >() - .notNull(), - routingMode: text("routingMode") - .$type<"url" | "target">() - .notNull() - .default("url"), - capabilities: text("capabilities").notNull().default("[]"), - headers: text("headers"), // JSON array of { name, value } - skipTlsVerification: integer("skipTlsVerification", { mode: "boolean" }) - .notNull() - .default(false), - enabled: integer("enabled", { mode: "boolean" }).notNull().default(true), - createdAt: integer("createdAt").notNull(), - updatedAt: integer("updatedAt").notNull() -}); +export const aiProviders = sqliteTable( + "aiProviders", + { + providerId: integer("providerId").primaryKey({ autoIncrement: true }), + orgId: text("orgId") + .notNull() + .references(() => orgs.orgId, { onDelete: "cascade" }), + name: text("name").notNull(), + niceId: text("niceId").notNull(), + type: text("type") + .$type< + | "openai" + | "anthropic" + | "googleGemini" + | "vertexAi" + | "bedrock" + | "microsoftFoundry" + | "openRouter" + | "vercelAiGateway" + | "custom" + >() + .notNull(), + upstreamUrl: text("upstreamUrl"), + apiKey: text("apiKey"), + apiKeyLastChars: text("apiKeyLastChars"), + authType: text("authType") + .$type< + | "bearer" + | "x-api-key" + | "x-goog-api-key" + | "hec" + | "cf-aig-authorization" + | "none" + | "passthrough" + >() + .notNull(), + routingMode: text("routingMode") + .$type<"url" | "target">() + .notNull() + .default("url"), + capabilities: text("capabilities").notNull().default("[]"), + headers: text("headers"), // JSON array of { name, value } + skipTlsVerification: integer("skipTlsVerification", { mode: "boolean" }) + .notNull() + .default(false), + enabled: integer("enabled", { mode: "boolean" }) + .notNull() + .default(true), + createdAt: integer("createdAt").notNull(), + updatedAt: integer("updatedAt").notNull() + }, + (t) => [index("idx_aiProviders_orgId_niceId").on(t.orgId, t.niceId)] +); export const aiModels = sqliteTable( "aiModels", diff --git a/server/routers/aiProvider/createAiProvider.ts b/server/routers/aiProvider/createAiProvider.ts index f115d674a..8a6ad9e4d 100644 --- a/server/routers/aiProvider/createAiProvider.ts +++ b/server/routers/aiProvider/createAiProvider.ts @@ -25,6 +25,7 @@ import { refineProviderUpstreamFields } from "@server/routers/aiProvider/validation"; import { serializeCapabilities } from "@server/lib/aiCapabilities"; +import { getUniqueProviderName, getUniqueResourceName } from "@server/db/names"; const paramsSchema = z.strictObject({ orgId: z.string().nonempty() @@ -133,11 +134,14 @@ export async function createAiProvider( ); } + const niceId = await getUniqueProviderName(orgId); + const [provider] = await db .insert(aiProviders) .values({ orgId, name, + niceId, type, upstreamUrl: resolved.upstreamUrl, apiKey: encryptedApiKey, diff --git a/server/routers/aiProvider/updateAiProvider.ts b/server/routers/aiProvider/updateAiProvider.ts index 1f22dc6ef..19555dda9 100644 --- a/server/routers/aiProvider/updateAiProvider.ts +++ b/server/routers/aiProvider/updateAiProvider.ts @@ -7,7 +7,7 @@ import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { eq } from "drizzle-orm"; +import { eq, ne, and } from "drizzle-orm"; import { encrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; @@ -37,6 +37,15 @@ const paramsSchema = z.strictObject({ const bodySchema = z.strictObject({ name: z.string().nonempty().optional(), + niceId: z + .string() + .min(1) + .max(255) + .regex( + /^[a-zA-Z0-9-]+$/, + "niceId can only contain letters, numbers, and dashes" + ) + .optional(), upstreamUrl: z.url().optional().nullable(), apiKey: z.string().optional(), authType: aiAuthTypeSchema.optional(), @@ -170,6 +179,9 @@ export async function updateAiProvider( if (body.name !== undefined) { updateData.name = body.name; } + if (body.niceId !== undefined) { + updateData.niceId = body.niceId; + } if (body.skipTlsVerification !== undefined) { updateData.skipTlsVerification = body.skipTlsVerification; } @@ -199,6 +211,29 @@ export async function updateAiProvider( updateData.headers = serializeAiProviderHeaders(body.headers, key); } + if (updateData.niceId) { + const [existingAiProvider] = await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.niceId, updateData.niceId), + eq(aiProviders.orgId, existing.orgId), + ne(aiProviders.providerId, existing.providerId) // exclude the current provider from the search + ) + ) + .limit(1); + + if (existingAiProvider) { + return next( + createHttpError( + HttpCode.CONFLICT, + `A resource with niceId "${updateData.niceId}" already exists` + ) + ); + } + } + const [provider] = await db .update(aiProviders) .set(updateData) From 5d37f44241e38ae765fd04ce8ec76bce735a7b15 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 09:43:03 -0400 Subject: [PATCH 181/296] Add niceid to ui and api for providers --- .../verifyApiKeyAiProviderAccess.ts | 46 +++++++---- server/middlewares/verifyAiProviderAccess.ts | 46 +++++++---- server/routers/aiProvider/getAiProvider.ts | 78 +++++++++++++++---- server/routers/external.ts | 7 ++ server/routers/integration.ts | 7 ++ .../authentication/page.tsx | 0 .../budget/page.tsx | 0 .../[niceId]/configuration/page.tsx | 12 +++ .../general/page.tsx | 35 +++++++++ .../{[providerId] => [niceId]}/layout.tsx | 16 ++-- .../models/page.tsx | 0 .../network/page.tsx | 0 .../settings/ai-providers/[niceId]/page.tsx | 10 +++ .../[providerId]/configuration/page.tsx | 12 --- .../ai-providers/[providerId]/page.tsx | 10 --- .../settings/ai-providers/create/page.tsx | 7 +- .../[orgId]/settings/ai-providers/page.tsx | 1 + src/components/AiProvidersTable.tsx | 7 +- 18 files changed, 219 insertions(+), 75 deletions(-) rename src/app/[orgId]/settings/ai-providers/{[providerId] => [niceId]}/authentication/page.tsx (100%) rename src/app/[orgId]/settings/ai-providers/{[providerId] => [niceId]}/budget/page.tsx (100%) create mode 100644 src/app/[orgId]/settings/ai-providers/[niceId]/configuration/page.tsx rename src/app/[orgId]/settings/ai-providers/{[providerId] => [niceId]}/general/page.tsx (85%) rename src/app/[orgId]/settings/ai-providers/{[providerId] => [niceId]}/layout.tsx (83%) rename src/app/[orgId]/settings/ai-providers/{[providerId] => [niceId]}/models/page.tsx (100%) rename src/app/[orgId]/settings/ai-providers/{[providerId] => [niceId]}/network/page.tsx (100%) create mode 100644 src/app/[orgId]/settings/ai-providers/[niceId]/page.tsx delete mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx delete mode 100644 src/app/[orgId]/settings/ai-providers/[providerId]/page.tsx diff --git a/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts b/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts index 1668b5727..91cfa47e4 100644 --- a/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts +++ b/server/middlewares/integration/verifyApiKeyAiProviderAccess.ts @@ -1,5 +1,5 @@ import { Request, Response, NextFunction } from "express"; -import { aiProviders, apiKeyOrg, db } from "@server/db"; +import { AiProvider, aiProviders, apiKeyOrg, db } from "@server/db"; import { and, eq } from "drizzle-orm"; import createHttpError from "http-errors"; import HttpCode from "@server/types/HttpCode"; @@ -13,7 +13,8 @@ export async function verifyApiKeyAiProviderAccess( try { const apiKey = req.apiKey; const providerIdRaw = getFirstString(req.params.providerId); - const providerId = Number.parseInt(providerIdRaw ?? "", 10); + const niceId = getFirstString(req.params.niceId); + const orgIdParam = getFirstString(req.params.orgId); if (!apiKey) { return next( @@ -21,23 +22,42 @@ export async function verifyApiKeyAiProviderAccess( ); } - if (Number.isNaN(providerId)) { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") - ); - } + let provider: AiProvider | undefined; - const [provider] = await db - .select() - .from(aiProviders) - .where(eq(aiProviders.providerId, providerId)) - .limit(1); + if (niceId && orgIdParam) { + const [providerRes] = await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.niceId, niceId), + eq(aiProviders.orgId, orgIdParam) + ) + ) + .limit(1); + provider = providerRes; + } else { + const providerId = Number.parseInt(providerIdRaw ?? "", 10); + + if (Number.isNaN(providerId)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") + ); + } + + const [providerRes] = await db + .select() + .from(aiProviders) + .where(eq(aiProviders.providerId, providerId)) + .limit(1); + provider = providerRes; + } if (!provider) { return next( createHttpError( HttpCode.NOT_FOUND, - `AI provider with ID ${providerId} not found` + `AI provider with ID ${providerIdRaw || niceId} not found` ) ); } diff --git a/server/middlewares/verifyAiProviderAccess.ts b/server/middlewares/verifyAiProviderAccess.ts index 22edd6f86..057497cc3 100644 --- a/server/middlewares/verifyAiProviderAccess.ts +++ b/server/middlewares/verifyAiProviderAccess.ts @@ -1,5 +1,5 @@ import { Request, Response, NextFunction } from "express"; -import { aiProviders, db, userOrgs } from "@server/db"; +import { AiProvider, aiProviders, db, userOrgs } from "@server/db"; import { and, eq } from "drizzle-orm"; import createHttpError from "http-errors"; import HttpCode from "@server/types/HttpCode"; @@ -15,7 +15,8 @@ export async function verifyAiProviderAccess( try { const userId = req.user!.userId; const providerIdRaw = getFirstString(req.params.providerId); - const providerId = Number.parseInt(providerIdRaw ?? "", 10); + const niceId = getFirstString(req.params.niceId); + const orgIdParam = getFirstString(req.params.orgId); if (!userId) { return next( @@ -23,23 +24,42 @@ export async function verifyAiProviderAccess( ); } - if (Number.isNaN(providerId)) { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") - ); - } + let provider: AiProvider | undefined; - const [provider] = await db - .select() - .from(aiProviders) - .where(eq(aiProviders.providerId, providerId)) - .limit(1); + if (niceId && orgIdParam) { + const [providerRes] = await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.niceId, niceId), + eq(aiProviders.orgId, orgIdParam) + ) + ) + .limit(1); + provider = providerRes; + } else { + const providerId = Number.parseInt(providerIdRaw ?? "", 10); + + if (Number.isNaN(providerId)) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Invalid provider ID") + ); + } + + const [providerRes] = await db + .select() + .from(aiProviders) + .where(eq(aiProviders.providerId, providerId)) + .limit(1); + provider = providerRes; + } if (!provider) { return next( createHttpError( HttpCode.NOT_FOUND, - `AI provider with ID ${providerId} not found` + `AI provider with ID ${providerIdRaw || niceId} not found` ) ); } diff --git a/server/routers/aiProvider/getAiProvider.ts b/server/routers/aiProvider/getAiProvider.ts index 28a0152e1..53a3e099a 100644 --- a/server/routers/aiProvider/getAiProvider.ts +++ b/server/routers/aiProvider/getAiProvider.ts @@ -2,26 +2,78 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; import { aiProviders, db } from "@server/db"; import response from "@server/lib/response"; +import stoi from "@server/lib/stoi"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { OpenAPITags, registry } from "@server/openApi"; -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import type { GetAiProviderResponse } from "@server/routers/aiProvider/types"; import { toPublicAiProvider } from "@server/routers/aiProvider/types"; const paramsSchema = z.strictObject({ - providerId: z.coerce.number().int().positive() + providerId: z + .string() + .optional() + .transform(stoi) + .pipe(z.int().positive().optional()) + .optional(), + niceId: z.string().optional(), + orgId: z.string().optional() }); +async function query(providerId?: number, niceId?: string, orgId?: string) { + if (providerId) { + const [res] = await db + .select() + .from(aiProviders) + .where(eq(aiProviders.providerId, providerId)) + .limit(1); + return res; + } else if (niceId && orgId) { + const [res] = await db + .select() + .from(aiProviders) + .where( + and( + eq(aiProviders.niceId, niceId), + eq(aiProviders.orgId, orgId) + ) + ) + .limit(1); + return res; + } +} + registry.registerPath({ method: "get", path: "/ai-provider/{providerId}", description: "Get an AI provider by ID.", tags: [OpenAPITags.AiProvider], request: { - params: paramsSchema + params: z.object({ + providerId: z.string() + }) + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +registry.registerPath({ + method: "get", + path: "/org/{orgId}/ai-provider/{niceId}", + description: + "Get an AI provider by orgId and niceId. NiceId is a readable ID for the provider and unique on a per org basis.", + tags: [OpenAPITags.AiProvider], + request: { + params: z.object({ + orgId: z.string(), + niceId: z.string() + }) }, responses: { 200: { @@ -46,22 +98,22 @@ export async function getAiProvider( ); } - const { providerId } = parsedParams.data; + const { providerId, niceId, orgId } = parsedParams.data; - const [provider] = - req.aiProvider && req.aiProvider.providerId === providerId - ? [req.aiProvider] - : await db - .select() - .from(aiProviders) - .where(eq(aiProviders.providerId, providerId)) - .limit(1); + const provider = + req.aiProvider && + (req.aiProvider.providerId === providerId || + (niceId && + req.aiProvider.niceId === niceId && + req.aiProvider.orgId === orgId)) + ? req.aiProvider + : await query(providerId, niceId, orgId); if (!provider) { return next( createHttpError( HttpCode.NOT_FOUND, - `AI provider with ID ${providerId} not found` + `AI provider with ID ${providerId || niceId} not found` ) ); } diff --git a/server/routers/external.ts b/server/routers/external.ts index 17cf0d921..e182d2b44 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1589,6 +1589,13 @@ authenticated.get( verifyUserHasAction(ActionsEnum.getAiProvider), aiProvider.getAiProvider ); +authenticated.get( + "/org/:orgId/ai-provider/:niceId", + verifyOrgAccess, + verifyAiProviderAccess, + verifyUserHasAction(ActionsEnum.getAiProvider), + aiProvider.getAiProvider +); authenticated.put( "/ai-provider/:providerId/target", diff --git a/server/routers/integration.ts b/server/routers/integration.ts index 5dc9315e9..f609443d4 100644 --- a/server/routers/integration.ts +++ b/server/routers/integration.ts @@ -1624,6 +1624,13 @@ authenticated.get( verifyApiKeyHasAction(ActionsEnum.getAiProvider), aiProvider.getAiProvider ); +authenticated.get( + "/org/:orgId/ai-provider/:niceId", + verifyApiKeyOrgAccess, + verifyApiKeyAiProviderAccess, + verifyApiKeyHasAction(ActionsEnum.getAiProvider), + aiProvider.getAiProvider +); authenticated.put( "/ai-provider/:providerId/target", diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx b/src/app/[orgId]/settings/ai-providers/[niceId]/authentication/page.tsx similarity index 100% rename from src/app/[orgId]/settings/ai-providers/[providerId]/authentication/page.tsx rename to src/app/[orgId]/settings/ai-providers/[niceId]/authentication/page.tsx diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/budget/page.tsx b/src/app/[orgId]/settings/ai-providers/[niceId]/budget/page.tsx similarity index 100% rename from src/app/[orgId]/settings/ai-providers/[providerId]/budget/page.tsx rename to src/app/[orgId]/settings/ai-providers/[niceId]/budget/page.tsx diff --git a/src/app/[orgId]/settings/ai-providers/[niceId]/configuration/page.tsx b/src/app/[orgId]/settings/ai-providers/[niceId]/configuration/page.tsx new file mode 100644 index 000000000..ee3f676ae --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[niceId]/configuration/page.tsx @@ -0,0 +1,12 @@ +import { redirect } from "next/navigation"; + +type Props = { + params: Promise<{ orgId: string; niceId: string }>; +}; + +export default async function AiProviderConfigurationRedirect({ + params +}: Props) { + const { orgId, niceId } = await params; + redirect(`/${orgId}/settings/ai-providers/${niceId}/network`); +} diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx b/src/app/[orgId]/settings/ai-providers/[niceId]/general/page.tsx similarity index 85% rename from src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx rename to src/app/[orgId]/settings/ai-providers/[niceId]/general/page.tsx index 9e41ff9c3..b65b1984a 100644 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/general/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[niceId]/general/page.tsx @@ -55,6 +55,7 @@ export default function AiProviderGeneralPage() { .string() .trim() .min(1, { message: t("nameRequired") }), + niceId: z.string().min(1).max(255).optional(), enabled: z.boolean(), capabilities: z.array(z.enum(AI_CAPABILITIES)).optional() }) @@ -76,6 +77,7 @@ export default function AiProviderGeneralPage() { resolver: zodResolver(generalSchema), defaultValues: { name: provider.name, + niceId: provider.niceId, enabled: provider.enabled, capabilities: provider.capabilities ?? [] } @@ -86,10 +88,12 @@ export default function AiProviderGeneralPage() { try { const body: { name: string; + niceId?: string; enabled: boolean; capabilities?: AiCapability[]; } = { name: values.name.trim(), + niceId: values.niceId, enabled: values.enabled, capabilities: values.capabilities ?? [] }; @@ -101,6 +105,7 @@ export default function AiProviderGeneralPage() { updateProvider(updated); form.reset({ name: updated.name, + niceId: updated.niceId, enabled: updated.enabled, capabilities: updated.capabilities ?? [] }); @@ -108,6 +113,13 @@ export default function AiProviderGeneralPage() { title: t("success"), description: t("aiProviderUpdated") }); + + if (values.niceId && values.niceId !== provider.niceId) { + router.replace( + `/${provider.orgId}/settings/ai-providers/${values.niceId}/general` + ); + } + router.refresh(); } catch (e) { toast({ @@ -190,6 +202,29 @@ export default function AiProviderGeneralPage() { /> + + ( + + + {t("identifier")} + + + + + + + )} + /> + + ; + params: Promise<{ orgId: string; niceId: string }>; }; export default async function AiProviderLayout({ children, params }: Props) { - const { orgId, providerId } = await params; + const { orgId, niceId } = await params; const t = await getTranslations(); let provider = null; try { const res = await internal.get>( - `/ai-provider/${providerId}`, + `/org/${orgId}/ai-provider/${niceId}`, await authCookieHeader() ); provider = res.data.data.provider; @@ -63,23 +63,23 @@ export default async function AiProviderLayout({ children, params }: Props) { const navItems = [ { title: t("general"), - href: "/{orgId}/settings/ai-providers/{providerId}/general" + href: "/{orgId}/settings/ai-providers/{niceId}/general" }, { title: t("aiProviderNetworkSettings"), - href: "/{orgId}/settings/ai-providers/{providerId}/network" + href: "/{orgId}/settings/ai-providers/{niceId}/network" }, { title: t("aiProviderModels"), - href: "/{orgId}/settings/ai-providers/{providerId}/models" + href: "/{orgId}/settings/ai-providers/{niceId}/models" }, { title: t("aiProviderAuthSettings"), - href: "/{orgId}/settings/ai-providers/{providerId}/authentication" + href: "/{orgId}/settings/ai-providers/{niceId}/authentication" }, { title: t("aiProviderBudgetSettings"), - href: "/{orgId}/settings/ai-providers/{providerId}/budget" + href: "/{orgId}/settings/ai-providers/{niceId}/budget" } ]; diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx b/src/app/[orgId]/settings/ai-providers/[niceId]/models/page.tsx similarity index 100% rename from src/app/[orgId]/settings/ai-providers/[providerId]/models/page.tsx rename to src/app/[orgId]/settings/ai-providers/[niceId]/models/page.tsx diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx b/src/app/[orgId]/settings/ai-providers/[niceId]/network/page.tsx similarity index 100% rename from src/app/[orgId]/settings/ai-providers/[providerId]/network/page.tsx rename to src/app/[orgId]/settings/ai-providers/[niceId]/network/page.tsx diff --git a/src/app/[orgId]/settings/ai-providers/[niceId]/page.tsx b/src/app/[orgId]/settings/ai-providers/[niceId]/page.tsx new file mode 100644 index 000000000..664127d17 --- /dev/null +++ b/src/app/[orgId]/settings/ai-providers/[niceId]/page.tsx @@ -0,0 +1,10 @@ +import { redirect } from "next/navigation"; + +type Props = { + params: Promise<{ orgId: string; niceId: string }>; +}; + +export default async function AiProviderPage({ params }: Props) { + const { orgId, niceId } = await params; + redirect(`/${orgId}/settings/ai-providers/${niceId}/general`); +} diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx deleted file mode 100644 index bf003b96e..000000000 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/configuration/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import { redirect } from "next/navigation"; - -type Props = { - params: Promise<{ orgId: string; providerId: string }>; -}; - -export default async function AiProviderConfigurationRedirect({ - params -}: Props) { - const { orgId, providerId } = await params; - redirect(`/${orgId}/settings/ai-providers/${providerId}/network`); -} diff --git a/src/app/[orgId]/settings/ai-providers/[providerId]/page.tsx b/src/app/[orgId]/settings/ai-providers/[providerId]/page.tsx deleted file mode 100644 index 2daca1227..000000000 --- a/src/app/[orgId]/settings/ai-providers/[providerId]/page.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import { redirect } from "next/navigation"; - -type Props = { - params: Promise<{ orgId: string; providerId: string }>; -}; - -export default async function AiProviderPage({ params }: Props) { - const { orgId, providerId } = await params; - redirect(`/${orgId}/settings/ai-providers/${providerId}/general`); -} diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 7684b2d48..29f005e8a 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -222,6 +222,7 @@ export default function CreateAiProviderPage() { >(`/org/${orgId}/ai-provider`, toAiProviderCreatePayload(values)); const providerId = res.data.data.provider.providerId; + const niceId = res.data.data.provider.niceId; if (showTargets && targets.length > 0) { try { @@ -236,7 +237,7 @@ export default function CreateAiProviderPage() { ) }); router.push( - `/${orgId}/settings/ai-providers/${providerId}/network` + `/${orgId}/settings/ai-providers/${niceId}/network` ); return; } @@ -255,7 +256,7 @@ export default function CreateAiProviderPage() { ) }); router.push( - `/${orgId}/settings/ai-providers/${providerId}/models` + `/${orgId}/settings/ai-providers/${niceId}/models` ); return; } @@ -266,7 +267,7 @@ export default function CreateAiProviderPage() { description: t("aiProviderCreated") }); - router.push(`/${orgId}/settings/ai-providers/${providerId}`); + router.push(`/${orgId}/settings/ai-providers/${niceId}`); } catch (e) { toast({ variant: "destructive", diff --git a/src/app/[orgId]/settings/ai-providers/page.tsx b/src/app/[orgId]/settings/ai-providers/page.tsx index 9858641e2..024170590 100644 --- a/src/app/[orgId]/settings/ai-providers/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/page.tsx @@ -53,6 +53,7 @@ export default async function AiProvidersPage({ params, searchParams }: Props) { orgId={orgId} providers={providers.map((provider) => ({ providerId: provider.providerId, + niceId: provider.niceId, name: provider.name, type: provider.type, routingMode: provider.routingMode, diff --git a/src/components/AiProvidersTable.tsx b/src/components/AiProvidersTable.tsx index ba7fbb00c..82bf23cfe 100644 --- a/src/components/AiProvidersTable.tsx +++ b/src/components/AiProvidersTable.tsx @@ -27,6 +27,7 @@ import { useDebouncedCallback } from "use-debounce"; export type AiProviderRow = { providerId: number; + niceId: string; name: string; type: string; routingMode: string; @@ -176,7 +177,7 @@ export default function AiProvidersTable({ header: () => {t("name")}, cell: ({ row }) => ( {row.original.name} @@ -238,7 +239,7 @@ export default function AiProvidersTable({ {t("edit")} @@ -256,7 +257,7 @@ export default function AiProvidersTable({ + )} + - )} - - +
@@ -287,29 +292,93 @@ type FilterSelectProps = { }; function FilterSelect(props: FilterSelectProps) { + const t = useTranslations(); + const [open, setOpen] = useState(false); + const selected = props.options.find( + (option) => option.value === props.value + ); + return ( -
+
- + + + + + + + + + + {t("aiUsageFilterNotFound")} + + + { + props.onValueChange(undefined); + setOpen(false); + }} + > + + {props.placeholder} + + {props.options.map((option) => ( + { + props.onValueChange( + option.value === props.value + ? undefined + : option.value + ); + setOpen(false); + }} + > + + + {option.label} + + + ))} + + + + +
); } From c568251d8d75d9e46722afe09d54cfb3a5d315b7 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 14 Aug 2026 11:38:11 -0400 Subject: [PATCH 185/296] improve model picker --- messages/en-US.json | 2 - .../ai-providers/[niceId]/models/page.tsx | 30 ++- .../settings/ai-providers/create/page.tsx | 25 ++- src/components/AiProviderModelListEditor.tsx | 183 ++++++------------ 4 files changed, 95 insertions(+), 145 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index 950952e91..52458fcbe 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1926,8 +1926,6 @@ "aiProviderModelsEditTitle": "Edit Model", "aiProviderModelsEditDescription": "Update the model key or configure its budget.", "aiProviderModelsBudgetTab": "Budget", - "aiProviderModelsBudgetDescription": "Configure how this model restricts usage based on spending or token limits", - "aiProviderModelsBudgetUnsaved": "Save this model first to configure its budget.", "aiProviderModelsKeyLabel": "Model Key", "aiProviderModelsKeyRequired": "Enter a model key", "aiProviderModelsKeyDuplicate": "This model key is already on a list", diff --git a/src/app/[orgId]/settings/ai-providers/[niceId]/models/page.tsx b/src/app/[orgId]/settings/ai-providers/[niceId]/models/page.tsx index b85c0076c..db882fac0 100644 --- a/src/app/[orgId]/settings/ai-providers/[niceId]/models/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/[niceId]/models/page.tsx @@ -11,6 +11,7 @@ import { SettingsSectionTitle } from "@app/components/Settings"; import { + persistPendingModelBudgets, type AiProviderModelListItem, type ModelListType } from "@app/components/AiProviderModelListEditor"; @@ -21,7 +22,9 @@ import { useEnvContext } from "@app/hooks/useEnvContext"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { aiProviderQueries } from "@app/lib/queries"; +import type { CreateOrEditAiModelResponse } from "@server/routers/aiProvider/types"; import { useQuery, useQueryClient } from "@tanstack/react-query"; +import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; import { useEffect, useMemo, useState } from "react"; @@ -113,8 +116,7 @@ export default function AiProviderModelsPage() { return; } - const toCreate: { modelKey: string; listType: ModelListType }[] = - []; + const toCreate: AiProviderModelListItem[] = []; const toUpdate: { modelId: number; modelKey: string; @@ -159,7 +161,7 @@ export default function AiProviderModelsPage() { continue; } - toCreate.push({ modelKey, listType }); + toCreate.push(item); } const toDelete = existing @@ -167,13 +169,21 @@ export default function AiProviderModelsPage() { .map((model) => model.modelId); await Promise.all([ - ...toCreate.map(({ modelKey, listType }) => - api.put(`/ai-provider/${provider.providerId}/model`, { - modelKey, - name: modelKey, - listType - }) - ), + ...toCreate.map(async (item) => { + const res = await api.put< + AxiosResponse + >(`/ai-provider/${provider.providerId}/model`, { + modelKey: item.modelKey, + name: item.modelKey, + listType: item.listType + }); + await persistPendingModelBudgets({ + api, + orgId: provider.orgId, + modelId: res.data.data.model.modelId, + pendingBudgets: item.pendingBudgets + }); + }), ...toUpdate.map(({ modelId, modelKey, listType }) => api.post(`/ai-model/${modelId}`, { modelKey, diff --git a/src/app/[orgId]/settings/ai-providers/create/page.tsx b/src/app/[orgId]/settings/ai-providers/create/page.tsx index 29f005e8a..5872594ef 100644 --- a/src/app/[orgId]/settings/ai-providers/create/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/create/page.tsx @@ -22,8 +22,8 @@ import HeaderTitle from "@app/components/SettingsSectionTitle"; import { AiProviderAuthTypeSelect } from "@app/components/AiProviderAuthTypeSelect"; import { AiProviderCapabilitiesSelect } from "@app/components/AiProviderCapabilitiesSelect"; import { - type AiProviderModelListItem, - type ModelListType + persistPendingModelBudgets, + type AiProviderModelListItem } from "@app/components/AiProviderModelListEditor"; import { AiProviderModelsLists } from "@app/components/AiProviderModelsLists"; import { @@ -59,7 +59,10 @@ import { import { zodResolver } from "@hookform/resolvers/zod"; import { authTypeRequiresApiKey } from "@app/lib/aiProviderDefaults"; import { aiProviderQueries } from "@app/lib/queries"; -import type { CreateOrEditAiProviderResponse } from "@server/routers/aiProvider/types"; +import type { + CreateOrEditAiModelResponse, + CreateOrEditAiProviderResponse +} from "@server/routers/aiProvider/types"; import { useQuery } from "@tanstack/react-query"; import type { AxiosResponse } from "axios"; import { useTranslations } from "next-intl"; @@ -157,14 +160,22 @@ export default function CreateAiProviderPage() { async function createModels( providerId: number, - items: { modelKey: string; listType: ModelListType }[] + items: AiProviderModelListItem[] ) { for (const item of items) { - await api.put(`/ai-provider/${providerId}/model`, { + const res = await api.put< + AxiosResponse + >(`/ai-provider/${providerId}/model`, { modelKey: item.modelKey, name: item.modelKey, listType: item.listType }); + await persistPendingModelBudgets({ + api, + orgId, + modelId: res.data.data.model.modelId, + pendingBudgets: item.pendingBudgets + }); } } @@ -210,8 +221,8 @@ export default function CreateAiProviderPage() { const modelItems = [...allowItems, ...blockItems] .map((item) => ({ - modelKey: item.modelKey.trim(), - listType: item.listType + ...item, + modelKey: item.modelKey.trim() })) .filter((item) => item.modelKey); diff --git a/src/components/AiProviderModelListEditor.tsx b/src/components/AiProviderModelListEditor.tsx index 4da6fb10c..032db8f2b 100644 --- a/src/components/AiProviderModelListEditor.tsx +++ b/src/components/AiProviderModelListEditor.tsx @@ -34,12 +34,6 @@ import { PopoverContent, PopoverTrigger } from "@app/components/ui/popover"; -import { - Tooltip, - TooltipContent, - TooltipProvider, - TooltipTrigger -} from "@app/components/ui/tooltip"; import { cn } from "@app/lib/cn"; import { isModelKeyPattern } from "@server/lib/aiModelKeyMatch"; import { HorizontalTabs } from "@app/components/HorizontalTabs"; @@ -56,16 +50,8 @@ import { createApiClient, formatAxiosError } from "@app/lib/api"; import { aiBudgetQueries } from "@app/lib/queries"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { zodResolver } from "@hookform/resolvers/zod"; -import { - Asterisk, - BookMarked, - Check, - Globe, - Pencil, - Plus, - Wallet, - XIcon -} from "lucide-react"; +import type { AxiosInstance } from "axios"; +import { Globe, Plus, XIcon } from "lucide-react"; import { useTranslations } from "next-intl"; import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { useForm } from "react-hook-form"; @@ -78,6 +64,7 @@ export type ModelSource = "catalog" | "custom" | "pattern" | "all"; export const ALL_MODELS_KEY = "*"; const COLLAPSED_ROWS = 5; +const GRID_COLUMNS = 2; export type AiProviderModelListItem = { clientId: string; @@ -85,8 +72,32 @@ export type AiProviderModelListItem = { modelKey: string; listType: ModelListType; hasBudget?: boolean; + pendingBudgets?: BudgetRow[]; }; +export async function persistPendingModelBudgets({ + api, + orgId, + modelId, + pendingBudgets +}: { + api: AxiosInstance; + orgId: string; + modelId: number; + pendingBudgets?: BudgetRow[]; +}): Promise { + if (!pendingBudgets || pendingBudgets.length === 0) { + return; + } + await saveBudgetRows({ + api, + orgId, + scope: { type: "model", id: modelId }, + existingBudgets: [], + rows: pendingBudgets + }); +} + export type AiProviderModelListEditorProps = { orgId: string; listType: ModelListType; @@ -138,27 +149,6 @@ function parseBulkKeys(raw: string): string[] { return keys; } -function useModelGridColumns(): number { - const [columns, setColumns] = useState(1); - - useEffect(() => { - const sm = window.matchMedia("(min-width: 640px)"); - const xl = window.matchMedia("(min-width: 1280px)"); - const update = () => { - setColumns(xl.matches ? 3 : sm.matches ? 2 : 1); - }; - update(); - sm.addEventListener("change", update); - xl.addEventListener("change", update); - return () => { - sm.removeEventListener("change", update); - xl.removeEventListener("change", update); - }; - }, []); - - return columns; -} - export function AiProviderModelListEditor({ orgId, listType, @@ -178,8 +168,7 @@ export function AiProviderModelListEditor({ const [listExpanded, setListExpanded] = useState(false); const [clipHeight, setClipHeight] = useState(null); const gridRef = useRef(null); - const columns = useModelGridColumns(); - const collapsedLimit = columns * COLLAPSED_ROWS; + const collapsedLimit = GRID_COLUMNS * COLLAPSED_ROWS; const hasOverflow = items.length > collapsedLimit; const isCollapsed = hasOverflow && !listExpanded; @@ -358,7 +347,7 @@ export function AiProviderModelListEditor({
{source === "all" ? ( - <> - - {t("aiProviderModelsAllLabel")} - - + + {t("aiProviderModelsAllLabel")} + ) : ( {item.modelKey} )}
- -
- - - e.stopPropagation()} - > - - - - {sourceLabel} - - {item.hasBudget ? ( - - - e.stopPropagation()} - > - - - - - {t("aiProviderModelsBudgetConfigured")} - - - ) : null} -
-
- {budgetScope ? ( - <> -

- {t( - "aiProviderModelsBudgetDescription" - )} -

- - - ) : ( -

- {t("aiProviderModelsBudgetUnsaved")} -

- )} +
From 4989d1e31a249d28816e3e0d9b8683edf3ff647f Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 11:31:56 -0400 Subject: [PATCH 186/296] Allow overlapping domains on public inference resources --- server/lib/blueprints/publicResources.ts | 11 +++++++++++ server/routers/resource/createResource.ts | 17 ++++++++++++++--- server/routers/resource/updateResource.ts | 15 ++++++++++++++- .../siteResource/listAllSiteResourcesByOrg.ts | 4 ++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/server/lib/blueprints/publicResources.ts b/server/lib/blueprints/publicResources.ts index 9c9242984..806d97e9c 100644 --- a/server/lib/blueprints/publicResources.ts +++ b/server/lib/blueprints/publicResources.ts @@ -306,6 +306,7 @@ export async function updatePublicResources( existingResource.resourceId, resourceData["full-domain"]!, orgId, + resourceData.mode === "inference", trx ); @@ -1098,6 +1099,7 @@ export async function updatePublicResources( undefined, resourceData["full-domain"]!, orgId, + resourceData.mode === "inference", trx ); @@ -2113,6 +2115,7 @@ export async function getDomain( resourceId: number | undefined, fullDomain: string, orgId: string, + isInference: boolean, trx: Transaction ) { const [fullDomainExists] = await trx @@ -2122,6 +2125,14 @@ export async function getDomain( and( eq(resources.fullDomain, fullDomain), eq(resources.orgId, orgId), + // Inference resources route through the central AI gateway + // rather than normal target-based proxying, so they're + // allowed to share a full-domain with a non-inference + // resource (and vice versa) - only conflicts within the + // same routing category are rejected. + isInference + ? eq(resources.mode, "inference") + : ne(resources.mode, "inference"), resourceId ? ne(resources.resourceId, resourceId) : isNotNull(resources.resourceId) diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index 45e66c2c3..897d32f6a 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -18,7 +18,7 @@ import { import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; -import { eq, and } from "drizzle-orm"; +import { eq, and, ne } from "drizzle-orm"; import { fromError } from "zod-validation-error"; import logger from "@server/logger"; import { subdomainSchema, wildcardSubdomainSchema } from "@server/lib/schemas"; @@ -484,11 +484,22 @@ async function createHttpResource( logger.debug(`Full domain: ${fullDomain}`); - // make sure the full domain is unique + // make sure the full domain is unique. Inference resources are routed + // through the central AI gateway rather than normal target-based + // proxying, so they're allowed to share a full-domain with a + // non-inference resource (and vice versa) - only conflicts within the + // same routing category are rejected. const existingResource = await db .select() .from(resources) - .where(eq(resources.fullDomain, fullDomain)); + .where( + and( + eq(resources.fullDomain, fullDomain), + effectiveMode === "inference" + ? eq(resources.mode, "inference") + : ne(resources.mode, "inference") + ) + ); if (existingResource.length > 0) { return next( diff --git a/server/routers/resource/updateResource.ts b/server/routers/resource/updateResource.ts index 0d6f8f8aa..3ed26c9cb 100644 --- a/server/routers/resource/updateResource.ts +++ b/server/routers/resource/updateResource.ts @@ -596,10 +596,23 @@ async function updateHttpResource( logger.debug(`Full domain: ${fullDomain}`); if (fullDomain) { + // Inference resources route through the central AI gateway + // rather than normal target-based proxying, so they're allowed + // to share a full-domain with a non-inference resource (and + // vice versa) - only conflicts within the same routing category + // are rejected. mode isn't updatable here, so `resource.mode` + // reflects the resource's actual (unchanging) routing category. const [existingDomain] = await db .select() .from(resources) - .where(eq(resources.fullDomain, fullDomain)); + .where( + and( + eq(resources.fullDomain, fullDomain), + resource.mode === "inference" + ? eq(resources.mode, "inference") + : ne(resources.mode, "inference") + ) + ); if ( existingDomain && diff --git a/server/routers/siteResource/listAllSiteResourcesByOrg.ts b/server/routers/siteResource/listAllSiteResourcesByOrg.ts index 50bbca6b4..7eb54c829 100644 --- a/server/routers/siteResource/listAllSiteResourcesByOrg.ts +++ b/server/routers/siteResource/listAllSiteResourcesByOrg.ts @@ -55,12 +55,12 @@ const listAllSiteResourcesByOrgQuerySchema = z.strictObject({ }), query: z.string().optional(), mode: z - .enum(["host", "cidr", "http"]) + .enum(["host", "cidr", "http", "ssh", "inference"]) .optional() .catch(undefined) .openapi({ type: "string", - enum: ["host", "cidr", "http"], + enum: ["host", "cidr", "http", "ssh", "inference"], description: "Filter site resources by mode" }), sort_by: z From c8f170d19771932f3eddb3438a7feea133d7def7 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 11:42:33 -0400 Subject: [PATCH 187/296] Allow budgets to be set on the resources --- server/lib/blueprints/aiBudgets.ts | 89 +++++++++++++++++++++++ server/lib/blueprints/privateResources.ts | 17 +++++ server/lib/blueprints/publicResources.ts | 17 +++++ server/lib/blueprints/types.ts | 39 +++++++++- 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 server/lib/blueprints/aiBudgets.ts diff --git a/server/lib/blueprints/aiBudgets.ts b/server/lib/blueprints/aiBudgets.ts new file mode 100644 index 000000000..0ae2fbdd4 --- /dev/null +++ b/server/lib/blueprints/aiBudgets.ts @@ -0,0 +1,89 @@ +import { eq } from "drizzle-orm"; +import { aiBudgets, Transaction } from "@server/db"; + +export type BlueprintAiBudgetInput = { + amount: number; + unit: "usd" | "tokens"; + period: + | "monthly" + | "yearly" + | "lifetime" + | "daily" + | "hourly" + | "weekly"; + enforcement: "hard" | "soft"; + enabled: boolean; +}; + +type SyncAiBudgetsInput = { + orgId: string; + trx: Transaction; + budgets: BlueprintAiBudgetInput[]; +} & ( + | { scope: "public"; resourceId: number } + | { scope: "site"; siteResourceId: number } +); + +/** + * Fully declarative: makes the resource's/site resource's AI budgets match + * exactly what the blueprint declares (omitted unit/period budgets are removed). + */ +export async function syncAiBudgets(input: SyncAiBudgetsInput): Promise { + const { orgId, trx, budgets } = input; + + const existing = await trx + .select() + .from(aiBudgets) + .where( + input.scope === "public" + ? eq(aiBudgets.resourceId, input.resourceId) + : eq(aiBudgets.siteResourceId, input.siteResourceId) + ); + + const existingByKey = new Map( + existing.map((b) => [`${b.unit}::${b.period}`, b]) + ); + + const seenKeys = new Set(); + const now = Date.now(); + + for (const budget of budgets) { + const key = `${budget.unit}::${budget.period}`; + seenKeys.add(key); + const existingBudget = existingByKey.get(key); + + if (existingBudget) { + await trx + .update(aiBudgets) + .set({ + amount: budget.amount, + enforcement: budget.enforcement, + enabled: budget.enabled, + updatedAt: now + }) + .where(eq(aiBudgets.budgetId, existingBudget.budgetId)); + } else { + await trx.insert(aiBudgets).values({ + orgId, + resourceId: input.scope === "public" ? input.resourceId : null, + siteResourceId: + input.scope === "site" ? input.siteResourceId : null, + amount: budget.amount, + unit: budget.unit, + period: budget.period, + enforcement: budget.enforcement, + enabled: budget.enabled, + createdAt: now, + updatedAt: now + }); + } + } + + for (const [key, existingBudget] of existingByKey) { + if (!seenKeys.has(key)) { + await trx + .delete(aiBudgets) + .where(eq(aiBudgets.budgetId, existingBudget.budgetId)); + } + } +} diff --git a/server/lib/blueprints/privateResources.ts b/server/lib/blueprints/privateResources.ts index 070ac68b4..bcbee7ee6 100644 --- a/server/lib/blueprints/privateResources.ts +++ b/server/lib/blueprints/privateResources.ts @@ -30,6 +30,7 @@ import { build } from "@server/build"; import { LimitId } from "../billing"; import { usageService } from "../billing/usageService"; import { syncInferenceAiConfig } from "./aiProviders"; +import { syncAiBudgets } from "./aiBudgets"; async function getDomainForSiteResource( siteResourceId: number | undefined, @@ -367,6 +368,14 @@ export async function updatePrivateResources( })) }); + await syncAiBudgets({ + orgId, + trx, + scope: "site", + siteResourceId, + budgets: resourceData["ai-budget"] + }); + await trx .delete(clientSiteResources) .where(eq(clientSiteResources.siteResourceId, siteResourceId)); @@ -668,6 +677,14 @@ export async function updatePrivateResources( })) }); + await syncAiBudgets({ + orgId, + trx, + scope: "site", + siteResourceId, + budgets: resourceData["ai-budget"] + }); + const [adminRole] = await trx .select() .from(roles) diff --git a/server/lib/blueprints/publicResources.ts b/server/lib/blueprints/publicResources.ts index 806d97e9c..4adc981ce 100644 --- a/server/lib/blueprints/publicResources.ts +++ b/server/lib/blueprints/publicResources.ts @@ -57,6 +57,7 @@ import next from "next"; import { LimitId } from "../billing"; import { usageService } from "../billing/usageService"; import { syncInferenceAiConfig } from "./aiProviders"; +import { syncAiBudgets } from "./aiBudgets"; export type PublicResourcesResults = { proxyResource: Resource; @@ -696,6 +697,14 @@ export async function updatePublicResources( }) ) }); + + await syncAiBudgets({ + orgId, + trx, + scope: "public", + resourceId: existingResource.resourceId, + budgets: resourceData["ai-budget"] || [] + }); } const existingResourceTargets = await trx @@ -1258,6 +1267,14 @@ export async function updatePublicResources( })) }); + await syncAiBudgets({ + orgId, + trx, + scope: "public", + resourceId: newResource.resourceId, + budgets: resourceData["ai-budget"] || [] + }); + await trx.insert(roleResources).values({ roleId: adminRole.roleId, resourceId: newResource.resourceId diff --git a/server/lib/blueprints/types.ts b/server/lib/blueprints/types.ts index 099656ddc..d9e9415d0 100644 --- a/server/lib/blueprints/types.ts +++ b/server/lib/blueprints/types.ts @@ -5,6 +5,11 @@ import { MaintenanceSchema } from "#dynamic/lib/blueprints/MaintenanceSchema"; import { isValidRegionId } from "@server/db/regions"; import { wildcardSubdomainSchema } from "@server/lib/schemas"; import config from "@server/lib/config"; +import { + aiBudgetEnforcementSchema, + aiBudgetPeriodSchema, + aiBudgetUnitSchema +} from "@server/routers/aiBudget/validation"; const maxmindDbPath = config.getRawConfig().server.maxmind_db_path; const maxmindAsnPath = config.getRawConfig().server.maxmind_asn_path; @@ -211,6 +216,33 @@ export const AiProviderAttachmentSchema = z } ); +export const AiBudgetSchema = z.object({ + amount: z.number().positive(), + unit: aiBudgetUnitSchema, + period: aiBudgetPeriodSchema.optional().default("monthly"), + enforcement: aiBudgetEnforcementSchema.optional().default("hard"), + enabled: z.boolean().optional().default(true) +}); + +const aiBudgetArraySchema = z.array(AiBudgetSchema).refine( + (budgets) => { + const keys = budgets.map((b) => `${b.unit}::${b.period}`); + return keys.length === new Set(keys).size; + }, + { + message: + "'ai-budget' entries must not overlap: only one budget per unit/period combination is allowed" + } +); + +// No default here: an object with only 'targets' set must remain +// recognized as a targets-only resource by isTargetsOnlyResource(). +export const AiBudgetListSchema = aiBudgetArraySchema.optional(); + +export const AiBudgetListSchemaWithDefault = aiBudgetArraySchema + .optional() + .default([]); + export const AuthDaemonSchema = z .object({ pam: z.enum(["passthrough", "push"]).optional().default("passthrough"), @@ -257,7 +289,8 @@ export const PublicResourceSchema = z "proxy-protocol": z.boolean().optional(), "proxy-protocol-version": z.int().min(1).optional(), labels: z.array(z.string().min(1)).optional(), - "ai-providers": z.array(AiProviderAttachmentSchema).optional() + "ai-providers": z.array(AiProviderAttachmentSchema).optional(), + "ai-budget": AiBudgetListSchema }) .refine( (resource) => { @@ -565,7 +598,8 @@ export const PrivateResourceSchema = z machines: z.array(z.string()).optional().default([]), labels: z.array(z.string().min(1)).optional().default([]), "auth-daemon": AuthDaemonSchema.optional(), - "ai-providers": z.array(AiProviderAttachmentSchema).optional().default([]) + "ai-providers": z.array(AiProviderAttachmentSchema).optional().default([]), + "ai-budget": AiBudgetListSchemaWithDefault }) .refine( (data) => { @@ -921,3 +955,4 @@ export type Target = z.infer; export type Resource = z.infer; export type Config = z.infer; export type BlueprintResourcePolicy = z.infer; +export type BlueprintAiBudget = z.infer; From 7375b9efae5ed5caf3a4a7abcecdbe8009c6dd82 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 14 Aug 2026 11:38:11 -0400 Subject: [PATCH 188/296] improve model picker --- src/app/[orgId]/settings/layout.tsx | 3 +- src/app/navigation.tsx | 22 ++++ src/components/CreateVirtualApiKeyForm.tsx | 5 - src/components/EditVirtualApiKeyForm.tsx | 5 - src/components/LayoutMobileMenu.tsx | 66 +---------- src/components/LayoutSidebar.tsx | 121 +-------------------- src/components/SidebarNav.tsx | 20 +++- 7 files changed, 45 insertions(+), 197 deletions(-) diff --git a/src/app/[orgId]/settings/layout.tsx b/src/app/[orgId]/settings/layout.tsx index cac644be3..598df4f74 100644 --- a/src/app/[orgId]/settings/layout.tsx +++ b/src/app/[orgId]/settings/layout.tsx @@ -80,7 +80,8 @@ export default async function SettingsLayout(props: SettingsLayoutProps) { orgId={params.orgId} orgs={orgs} navItems={orgNavSections(env, { - isPrimaryOrg: primaryOrg + isPrimaryOrg: primaryOrg, + isServerAdmin: user.serverAdmin })} commandNavItems={commandBarNavSections(env, { isPrimaryOrg: primaryOrg diff --git a/src/app/navigation.tsx b/src/app/navigation.tsx index 9bd932f22..2f00744c0 100644 --- a/src/app/navigation.tsx +++ b/src/app/navigation.tsx @@ -47,6 +47,7 @@ export type SidebarNavSection = { export type OrgNavSectionsOptions = { isPrimaryOrg?: boolean; + isServerAdmin?: boolean; }; // Merged from 'user-management-and-resources' branch @@ -67,6 +68,27 @@ export const orgNavSections = ( env?: Env, options?: OrgNavSectionsOptions ): SidebarNavSection[] => [ + { + heading: "sidebarOverview", + items: [ + { + title: "resourceSidebarLauncherTitle", + href: "/{orgId}", + icon: , + exact: true + }, + ...(options?.isServerAdmin + ? [ + { + title: "serverAdmin", + href: "/admin", + icon: , + exact: true + } + ] + : []) + ] + }, { heading: "network", items: [ diff --git a/src/components/CreateVirtualApiKeyForm.tsx b/src/components/CreateVirtualApiKeyForm.tsx index 4a97d358a..5c2e75d0f 100644 --- a/src/components/CreateVirtualApiKeyForm.tsx +++ b/src/components/CreateVirtualApiKeyForm.tsx @@ -476,11 +476,6 @@ export default function CreateVirtualApiKeyForm({
-

- {t( - "virtualApiKeysInferenceBudgetDescription" - )} -

-

- {t( - "virtualApiKeysInferenceBudgetDescription" - )} -

org.orgId === orgId); - const isSettingsPage = Boolean( - orgId && pathname?.includes(`/${orgId}/settings`) - ); - const canViewResourceLauncher = Boolean( - currentOrg?.isAdmin || currentOrg?.isOwner - ); const mobileNavLinkClassName = cn( "flex items-center rounded transition-colors text-muted-foreground hover:text-foreground text-sm w-full hover:bg-secondary/50 dark:hover:bg-secondary/20 rounded-md px-3 py-1.5" @@ -183,58 +171,6 @@ export function LayoutMobileMenu({
- {!isAdminPage && - isSettingsPage && - canViewResourceLauncher && - orgId && ( -
- - setIsMobileMenuOpen( - false - ) - } - > - - - - - {t( - "resourceSidebarLauncherTitle" - )} - - -
- )} - {!isAdminPage && - user.serverAdmin && ( -
- - setIsMobileMenuOpen( - false - ) - } - > - - - - - {t( - "serverAdmin" - )} - - -
- )} diff --git a/src/components/LayoutSidebar.tsx b/src/components/LayoutSidebar.tsx index df1eb45d3..bf5f7bb4c 100644 --- a/src/components/LayoutSidebar.tsx +++ b/src/components/LayoutSidebar.tsx @@ -18,13 +18,7 @@ import { approvalQueries } from "@app/lib/queries"; import { build } from "@server/build"; import { useQuery } from "@tanstack/react-query"; import { ListUserOrgsResponse } from "@server/routers/org"; -import { - ArrowRight, - ExternalLink, - LayoutGrid, - PanelRightOpen, - Server -} from "lucide-react"; +import { ExternalLink, PanelRightOpen } from "lucide-react"; import { useTranslations } from "next-intl"; import dynamic from "next/dynamic"; import Link from "next/link"; @@ -136,13 +130,6 @@ export function LayoutSidebar({ const showTrial = build === "saas" && Boolean(orgId) && subscriptionContext?.isTrial; - const isSettingsPage = Boolean( - orgId && pathname?.includes(`/${orgId}/settings`) - ); - const canViewResourceLauncher = Boolean( - currentOrg?.isAdmin || currentOrg?.isOwner - ); - return (
- {!isAdminPage && - isSettingsPage && - canViewResourceLauncher && - orgId && ( -
- {isSidebarCollapsed ? ( - - - - - - - - - - -

- {t( - "resourceSidebarLauncherTitle" - )} -

- - - - ) : ( - - - - - - {t("resourceSidebarLauncherTitle")} - - - )} -
- )} - {!isAdminPage && user.serverAdmin && ( -
- {isSidebarCollapsed ? ( - - - - - - - - - - -

{t("serverAdmin")}

- - - - ) : ( - - - - - - {t("serverAdmin")} - - - )} -
- )} {canShowProductUpdates ? ( diff --git a/src/components/SidebarNav.tsx b/src/components/SidebarNav.tsx index eb8386ffc..258d43a56 100644 --- a/src/components/SidebarNav.tsx +++ b/src/components/SidebarNav.tsx @@ -34,6 +34,7 @@ export type SidebarNavItem = { showEE?: boolean; isBeta?: boolean; items?: SidebarNavItem[]; + exact?: boolean; }; export type SidebarNavSection = { @@ -49,7 +50,14 @@ export interface SidebarNavProps extends React.HTMLAttributes { notificationCounts?: Record; } -function isPathActive(pathname: string, href: string): boolean { +function isPathActive( + pathname: string, + href: string, + exact?: boolean +): boolean { + if (exact) { + return pathname === href; + } return pathname === href || pathname.startsWith(`${href}/`); } @@ -289,7 +297,11 @@ function CollapsedNavItemWithPopover({ childItem.href ); const childIsActive = childHydratedHref - ? isPathActive(pathname, childHydratedHref) + ? isPathActive( + pathname, + childHydratedHref, + childItem.exact + ) : false; const childIsEE = build === "enterprise" && @@ -396,7 +408,7 @@ export function SidebarNav({ function isItemOrChildActive(item: SidebarNavItem): boolean { const hydratedHref = hydrateHref(item.href); - if (hydratedHref && isPathActive(pathname, hydratedHref)) { + if (hydratedHref && isPathActive(pathname, hydratedHref, item.exact)) { return true; } if (item.items) { @@ -412,7 +424,7 @@ export function SidebarNav({ const hydratedHref = hydrateHref(item.href); const hasNestedItems = item.items && item.items.length > 0; const isActive = hydratedHref - ? isPathActive(pathname, hydratedHref) + ? isPathActive(pathname, hydratedHref, item.exact) : false; const isChildActive = hasNestedItems ? isItemOrChildActive(item) From b34a8d116b6cafd8d61224756ccbc45def733b61 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 14 Aug 2026 12:18:55 -0400 Subject: [PATCH 189/296] improve org selector --- src/components/OrgPicker.tsx | 137 ++++++++++++ src/components/OrgSelector.tsx | 203 ++++-------------- .../resource-launcher/LauncherOrgSelector.tsx | 140 ++---------- 3 files changed, 193 insertions(+), 287 deletions(-) create mode 100644 src/components/OrgPicker.tsx diff --git a/src/components/OrgPicker.tsx b/src/components/OrgPicker.tsx new file mode 100644 index 000000000..3113198fc --- /dev/null +++ b/src/components/OrgPicker.tsx @@ -0,0 +1,137 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList +} from "@app/components/ui/command"; +import { + Popover, + PopoverContent, + PopoverTrigger +} from "@app/components/ui/popover"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { useUserContext } from "@app/hooks/useUserContext"; +import { cn } from "@app/lib/cn"; +import { build } from "@server/build"; +import { ListUserOrgsResponse } from "@server/routers/org"; +import { CheckIcon, Plus } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { usePathname, useRouter } from "next/navigation"; +import { useMemo, useState, type ReactNode } from "react"; + +export type OrgPickerProps = { + orgId?: string; + orgs?: ListUserOrgsResponse["orgs"]; + contentClassName?: string; + sideOffset?: number; + children: ReactNode; +}; + +export function OrgPicker({ + orgId, + orgs, + contentClassName, + sideOffset = 0, + children +}: OrgPickerProps) { + const [open, setOpen] = useState(false); + const router = useRouter(); + const pathname = usePathname(); + const t = useTranslations(); + const { env } = useEnvContext(); + const { user } = useUserContext(); + + let canCreateOrg = !env.flags.disableUserCreateOrg || user.serverAdmin; + if (build === "saas" && user.type !== "internal") { + canCreateOrg = false; + } + + const sortedOrgs = useMemo(() => { + if (!orgs?.length) { + return orgs ?? []; + } + return [...orgs].sort((a, b) => { + const aPrimary = Boolean(a.isPrimaryOrg); + const bPrimary = Boolean(b.isPrimaryOrg); + if (aPrimary && !bPrimary) return -1; + if (!aPrimary && bPrimary) return 1; + return 0; + }); + }, [orgs]); + + function selectOrg(nextOrgId: string) { + setOpen(false); + const newPath = pathname.includes("/settings/") + ? pathname.replace(/^\/[^/]+/, `/${nextOrgId}`) + : `/${nextOrgId}`; + router.push(newPath); + } + + return ( + + {children} + + + + + {t("orgNotFound2")} + + {sortedOrgs.map((org) => ( + selectOrg(org.orgId)} + > + +
+ + {org.name} + + + {org.orgId} + {org.isPrimaryOrg + ? ` · ${t("primary")}` + : ""} + +
+ + ))} + + + + {canCreateOrg && ( +
+ +
+ )} + + + ); +} diff --git a/src/components/OrgSelector.tsx b/src/components/OrgSelector.tsx index 79f2fad7d..9742fdb90 100644 --- a/src/components/OrgSelector.tsx +++ b/src/components/OrgSelector.tsx @@ -1,199 +1,76 @@ "use client"; -import { - Command, - CommandEmpty, - CommandGroup, - CommandInput, - CommandItem, - CommandList -} from "@app/components/ui/command"; -import { - Popover, - PopoverContent, - PopoverTrigger -} from "@app/components/ui/popover"; +import { OrgPicker } from "@app/components/OrgPicker"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@app/components/ui/tooltip"; -import { Badge } from "@app/components/ui/badge"; -import { useEnvContext } from "@app/hooks/useEnvContext"; import { cn } from "@app/lib/cn"; import { ListUserOrgsResponse } from "@server/routers/org"; -import { Check, ChevronsUpDown, Plus, Building2, Users } from "lucide-react"; -import { Button } from "@app/components/ui/button"; -import { usePathname, useRouter } from "next/navigation"; -import { useMemo, useState } from "react"; -import { useUserContext } from "@app/hooks/useUserContext"; +import { Building2, ChevronsUpDown } from "lucide-react"; import { useTranslations } from "next-intl"; -import { build } from "@server/build"; -interface OrgSelectorProps { +type OrgSelectorProps = { orgId?: string; orgs?: ListUserOrgsResponse["orgs"]; isCollapsed?: boolean; -} +}; export function OrgSelector({ orgId, orgs, isCollapsed = false }: OrgSelectorProps) { - const { user } = useUserContext(); - const [open, setOpen] = useState(false); - const router = useRouter(); - const pathname = usePathname(); - const { env } = useEnvContext(); const t = useTranslations(); - const selectedOrg = orgs?.find((org) => org.orgId === orgId); - let canCreateOrg = !env.flags.disableUserCreateOrg || user.serverAdmin; - if (build === "saas" && user.type !== "internal") { - canCreateOrg = false; - } - - const sortedOrgs = useMemo(() => { - if (!orgs?.length) return orgs ?? []; - return [...orgs].sort((a, b) => { - const aPrimary = Boolean(a.isPrimaryOrg); - const bPrimary = Boolean(b.isPrimaryOrg); - if (aPrimary && !bPrimary) return -1; - if (!aPrimary && bPrimary) return 1; - return 0; - }); - }, [orgs]); - - const orgSelectorContent = ( - - -
- {isCollapsed ? ( - - ) : ( -
-
-
- - {t("org")} - - - {selectedOrg?.name || t("noneSelected")} - -
-
- -
- )} -
- - +
- - - - -
- {t("orgNotFound2")} + {isCollapsed ? ( + + ) : ( +
+
+
+ + {t("org")} + + + {selectedOrg?.name || t("noneSelected")} +
- - - {sortedOrgs.map((org) => ( - { - setOpen(false); - const newPath = pathname.includes( - "/settings/" - ) - ? pathname.replace( - /^\/[^/]+/, - `/${org.orgId}` - ) - : `/${org.orgId}`; - router.push(newPath); - }} - className="mx-1 rounded-md py-1.5 h-auto min-h-0" - > -
- -
-
- - {org.name} - -
- - {org.orgId} - - {org.isPrimaryOrg && ( - - {t("primary")} - - )} -
-
- - - ))} - - - - {canCreateOrg && ( -
- +
+
)} - - +
+ ); if (isCollapsed) { return ( - - {orgSelectorContent} - + {picker}

@@ -209,5 +86,5 @@ export function OrgSelector({ ); } - return orgSelectorContent; + return picker; } diff --git a/src/components/resource-launcher/LauncherOrgSelector.tsx b/src/components/resource-launcher/LauncherOrgSelector.tsx index 04773d2de..80510d77e 100644 --- a/src/components/resource-launcher/LauncherOrgSelector.tsx +++ b/src/components/resource-launcher/LauncherOrgSelector.tsx @@ -1,28 +1,10 @@ "use client"; -import { - Command, - CommandEmpty, - CommandGroup, - CommandInput, - CommandItem, - CommandList -} from "@app/components/ui/command"; -import { - Popover, - PopoverContent, - PopoverTrigger -} from "@app/components/ui/popover"; -import { cn } from "@app/lib/cn"; -import { ListUserOrgsResponse } from "@server/routers/org"; -import { Check, ChevronDown, Plus } from "lucide-react"; -import { usePathname, useRouter } from "next/navigation"; -import { useMemo, useState } from "react"; -import { useTranslations } from "next-intl"; +import { OrgPicker } from "@app/components/OrgPicker"; import { Button } from "@app/components/ui/button"; -import { useEnvContext } from "@app/hooks/useEnvContext"; -import { useUserContext } from "@app/hooks/useUserContext"; -import { build } from "@server/build"; +import { ListUserOrgsResponse } from "@server/routers/org"; +import { ChevronDown } from "lucide-react"; +import { useTranslations } from "next-intl"; type LauncherOrgSelectorProps = { orgId?: string; @@ -30,111 +12,21 @@ type LauncherOrgSelectorProps = { }; export function LauncherOrgSelector({ orgId, orgs }: LauncherOrgSelectorProps) { - const [open, setOpen] = useState(false); - const router = useRouter(); - const pathname = usePathname(); const t = useTranslations(); - const { env } = useEnvContext(); - const { user } = useUserContext(); - const selectedOrg = orgs?.find((org) => org.orgId === orgId); - let canCreateOrg = !env.flags.disableUserCreateOrg || user.serverAdmin; - if (build === "saas" && user.type !== "internal") { - canCreateOrg = false; - } - - const sortedOrgs = useMemo(() => { - if (!orgs?.length) { - return orgs ?? []; - } - return [...orgs].sort((a, b) => { - const aPrimary = Boolean(a.isPrimaryOrg); - const bPrimary = Boolean(b.isPrimaryOrg); - if (aPrimary && !bPrimary) { - return -1; - } - if (!aPrimary && bPrimary) { - return 1; - } - return 0; - }); - }, [orgs]); - return ( - - - - - - - - - {t("orgNotFound2")} - - {sortedOrgs.map((org) => ( - { - setOpen(false); - const newPath = pathname.includes( - "/settings/" - ) - ? pathname.replace( - /^\/[^/]+/, - `/${org.orgId}` - ) - : `/${org.orgId}`; - router.push(newPath); - }} - > -

- - {org.name} - - - {org.orgId} - -
- - - ))} - - - - {canCreateOrg && ( -
- -
- )} - - + + + ); } From 234819f8f22ccb7ecd5df39ecd0d5a2e0e02d0fb Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 11:53:10 -0400 Subject: [PATCH 190/296] Dont load private resource inference without a exit node loaded --- server/lib/traefik/getTraefikConfig.ts | 172 +++++++------- .../private/lib/traefik/getTraefikConfig.ts | 212 +++++++++--------- 2 files changed, 196 insertions(+), 188 deletions(-) diff --git a/server/lib/traefik/getTraefikConfig.ts b/server/lib/traefik/getTraefikConfig.ts index d0d553910..da9fb0810 100644 --- a/server/lib/traefik/getTraefikConfig.ts +++ b/server/lib/traefik/getTraefikConfig.ts @@ -767,15 +767,14 @@ export async function getTraefikConfig( if (!config_output.http.middlewares) { config_output.http.middlewares = {}; } - config_output.http.middlewares[aiGatewayTrustMiddlewareNameResource] = - { - headers: { - customRequestHeaders: { - [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken(), - [AI_GATEWAY_RESOURCE_TYPE_HEADER]: "resource" - } + config_output.http.middlewares[aiGatewayTrustMiddlewareNameResource] = { + headers: { + customRequestHeaders: { + [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken(), + [AI_GATEWAY_RESOURCE_TYPE_HEADER]: "resource" } - }; + } + }; config_output.http.middlewares[ aiGatewayTrustMiddlewareNameSiteResource ] = { @@ -798,15 +797,14 @@ export async function getTraefikConfig( const enableAiGatewayClientIpHeader = config.getRawConfig().server.enable_ai_gateway_client_ip_header; if (enableAiGatewayClientIpHeader) { - config_output.http.middlewares[aiGatewayClientIpMiddlewareName] = - { - plugin: { - badger: { - disableForwardAuth: true, - realIpHeader: AI_GATEWAY_CLIENT_IP_HEADER - } + config_output.http.middlewares[aiGatewayClientIpMiddlewareName] = { + plugin: { + badger: { + disableForwardAuth: true, + realIpHeader: AI_GATEWAY_CLIENT_IP_HEADER } - }; + } + }; } // Public inference resources: same TLS/cert-resolver handling as @@ -935,90 +933,94 @@ export async function getTraefikConfig( ) ); - for (const sr of siteResourcesInference) { - if (!sr.enabled || !sr.fullDomain) continue; + if (exitNode) { + for (const sr of siteResourcesInference) { + if (!sr.enabled || !sr.fullDomain) continue; - if (!config_output.http.routers) config_output.http.routers = {}; - if (!config_output.http.services) config_output.http.services = {}; + if (!config_output.http.routers) + config_output.http.routers = {}; + if (!config_output.http.services) + config_output.http.services = {}; - const fullDomain = sr.fullDomain; - const srKey = `inference-sr${sr.siteResourceId}`; - const routerName = `${srKey}-router`; - const serviceName = `${srKey}-service`; - const rule = `Host(\`${fullDomain}\`) && ClientIP(${exitNode.address})`; // restrict to coming from the exit node ip range that the client is connected to + const fullDomain = sr.fullDomain; + const srKey = `inference-sr${sr.siteResourceId}`; + const routerName = `${srKey}-router`; + const serviceName = `${srKey}-service`; + const rule = `Host(\`${fullDomain}\`) && ClientIP(${exitNode.address})`; // restrict to coming from the exit node ip range that the client is connected to - const domainParts = fullDomain.split("."); - const wildCard = - domainParts.length <= 2 - ? `*.${domainParts.join(".")}` - : `*.${domainParts.slice(1).join(".")}`; + const domainParts = fullDomain.split("."); + const wildCard = + domainParts.length <= 2 + ? `*.${domainParts.join(".")}` + : `*.${domainParts.slice(1).join(".")}`; - const globalDefaultResolver = - config.getRawConfig().traefik.cert_resolver; - const globalDefaultPreferWildcard = - config.getRawConfig().traefik.prefer_wildcard_cert; + const globalDefaultResolver = + config.getRawConfig().traefik.cert_resolver; + const globalDefaultPreferWildcard = + config.getRawConfig().traefik.prefer_wildcard_cert; - const tls = { - certResolver: globalDefaultResolver, - ...(globalDefaultPreferWildcard - ? { domains: [{ main: wildCard }] } - : {}) - }; + const tls = { + certResolver: globalDefaultResolver, + ...(globalDefaultPreferWildcard + ? { domains: [{ main: wildCard }] } + : {}) + }; - const srHeadersMiddlewareName = `${srKey}-headers-middleware`; - if (!config_output.http.middlewares) { - config_output.http.middlewares = {}; - } - config_output.http.middlewares[srHeadersMiddlewareName] = { - headers: { - customRequestHeaders: { - ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": fullDomain - } + const srHeadersMiddlewareName = `${srKey}-headers-middleware`; + if (!config_output.http.middlewares) { + config_output.http.middlewares = {}; } - }; + config_output.http.middlewares[srHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), + "p-host": fullDomain + } + } + }; - const additionalMiddlewares = - config.getRawConfig().traefik.additional_middlewares || []; - const routerMiddlewares = [ - ...(enableAiGatewayClientIpHeader - ? [aiGatewayClientIpMiddlewareName] - : []), - aiGatewayTrustMiddlewareNameSiteResource, - srHeadersMiddlewareName, - ...additionalMiddlewares - ]; + const additionalMiddlewares = + config.getRawConfig().traefik.additional_middlewares || []; + const routerMiddlewares = [ + ...(enableAiGatewayClientIpHeader + ? [aiGatewayClientIpMiddlewareName] + : []), + aiGatewayTrustMiddlewareNameSiteResource, + srHeadersMiddlewareName, + ...additionalMiddlewares + ]; - if (sr.ssl) { - config_output.http.routers[routerName + "-redirect"] = { + if (sr.ssl) { + config_output.http.routers[routerName + "-redirect"] = { + entryPoints: [ + config.getRawConfig().traefik.http_entrypoint + ], + middlewares: [redirectHttpsMiddlewareName], + service: serviceName, + rule, + priority: 200 // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. + }; + } + + config_output.http.routers[routerName] = { entryPoints: [ - config.getRawConfig().traefik.http_entrypoint + sr.ssl + ? config.getRawConfig().traefik.https_entrypoint + : config.getRawConfig().traefik.http_entrypoint ], - middlewares: [redirectHttpsMiddlewareName], + middlewares: routerMiddlewares, service: serviceName, rule, - priority: 200 // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. + priority: 200, // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. + ...(sr.ssl ? { tls } : {}) + }; + + config_output.http.services[serviceName] = { + loadBalancer: { + servers: [{ url: aiGatewayUrl }] + } }; } - - config_output.http.routers[routerName] = { - entryPoints: [ - sr.ssl - ? config.getRawConfig().traefik.https_entrypoint - : config.getRawConfig().traefik.http_entrypoint - ], - middlewares: routerMiddlewares, - service: serviceName, - rule, - priority: 200, // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. - ...(sr.ssl ? { tls } : {}) - }; - - config_output.http.services[serviceName] = { - loadBalancer: { - servers: [{ url: aiGatewayUrl }] - } - }; } } diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index f90305bd1..cc841fc1e 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -1581,15 +1581,14 @@ export async function getTraefikConfig( "ai-gateway-trust-headers-resource"; const aiGatewayTrustMiddlewareNameSiteResource = "ai-gateway-trust-headers-site-resource"; - config_output.http.middlewares[aiGatewayTrustMiddlewareNameResource] = - { - headers: { - customRequestHeaders: { - [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken(), - [AI_GATEWAY_RESOURCE_TYPE_HEADER]: "resource" - } + config_output.http.middlewares[aiGatewayTrustMiddlewareNameResource] = { + headers: { + customRequestHeaders: { + [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken(), + [AI_GATEWAY_RESOURCE_TYPE_HEADER]: "resource" } - }; + } + }; config_output.http.middlewares[ aiGatewayTrustMiddlewareNameSiteResource ] = { @@ -1612,15 +1611,14 @@ export async function getTraefikConfig( const enableAiGatewayClientIpHeader = config.getRawConfig().server.enable_ai_gateway_client_ip_header; if (enableAiGatewayClientIpHeader) { - config_output.http.middlewares[aiGatewayClientIpMiddlewareName] = - { - plugin: { - badger: { - disableForwardAuth: true, - realIpHeader: AI_GATEWAY_CLIENT_IP_HEADER - } + config_output.http.middlewares[aiGatewayClientIpMiddlewareName] = { + plugin: { + badger: { + disableForwardAuth: true, + realIpHeader: AI_GATEWAY_CLIENT_IP_HEADER } - }; + } + }; } // Public inference resources: same TLS/cert-resolver handling as @@ -1742,108 +1740,116 @@ export async function getTraefikConfig( }; } - // Private (siteResource) inference resources: routed by their alias - // instead of a public fullDomain, and deliberately WITHOUT the - // badger middleware - no per-user auth/policy stack exists for - // siteResources today (see plan doc), so gating here is - // reachability-only for now. - for (const sr of siteResourcesInference) { - if (!sr.enabled || !sr.fullDomain) continue; + if (exitNode) { + // Private (siteResource) inference resources: routed by their alias + // instead of a public fullDomain, and deliberately WITHOUT the + // badger middleware - no per-user auth/policy stack exists for + // siteResources today (see plan doc), so gating here is + // reachability-only for now. + for (const sr of siteResourcesInference) { + if (!sr.enabled || !sr.fullDomain) continue; - if (!config_output.http.routers) config_output.http.routers = {}; - if (!config_output.http.services) config_output.http.services = {}; + if (!config_output.http.routers) + config_output.http.routers = {}; + if (!config_output.http.services) + config_output.http.services = {}; - const fullDomain = sr.fullDomain; - const srKey = `inference-sr${sr.siteResourceId}`; - const routerName = `${srKey}-router`; - const serviceName = `${srKey}-service`; - const rule = `Host(\`${fullDomain}\`) && ClientIP(\`${exitNode.address}\`)`; // restrict to coming from the exit node ip range that the client is connected to + const fullDomain = sr.fullDomain; + const srKey = `inference-sr${sr.siteResourceId}`; + const routerName = `${srKey}-router`; + const serviceName = `${srKey}-service`; + const rule = `Host(\`${fullDomain}\`) && ClientIP(\`${exitNode.address}\`)`; // restrict to coming from the exit node ip range that the client is connected to - let tls: any = {}; - if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { - const domainParts = fullDomain.split("."); - const wildCard = - domainParts.length <= 2 - ? `*.${domainParts.join(".")}` - : `*.${domainParts.slice(1).join(".")}`; + let tls: any = {}; + if ( + !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns + ) { + const domainParts = fullDomain.split("."); + const wildCard = + domainParts.length <= 2 + ? `*.${domainParts.join(".")}` + : `*.${domainParts.slice(1).join(".")}`; - const globalDefaultResolver = - config.getRawConfig().traefik.cert_resolver; - const globalDefaultPreferWildcard = - config.getRawConfig().traefik.prefer_wildcard_cert; + const globalDefaultResolver = + config.getRawConfig().traefik.cert_resolver; + const globalDefaultPreferWildcard = + config.getRawConfig().traefik.prefer_wildcard_cert; - tls = { - certResolver: globalDefaultResolver, - ...(globalDefaultPreferWildcard - ? { domains: [{ main: wildCard }] } - : {}) - }; - } else { - const matchingCert = validCerts.find( - (cert) => cert.queriedDomain === fullDomain - ); - if (!matchingCert) { - logger.debug( - `No matching certificate found for inference siteResource fullDomain: ${fullDomain}` + tls = { + certResolver: globalDefaultResolver, + ...(globalDefaultPreferWildcard + ? { domains: [{ main: wildCard }] } + : {}) + }; + } else { + const matchingCert = validCerts.find( + (cert) => cert.queriedDomain === fullDomain ); - continue; - } - } - - const additionalMiddlewares = - config.getRawConfig().traefik.additional_middlewares || []; - const routerMiddlewares: string[] = [ - ...(enableAiGatewayClientIpHeader - ? [aiGatewayClientIpMiddlewareName] - : []), - aiGatewayTrustMiddlewareNameSiteResource - ]; - - if (aiGatewayOverride) { - const srHeadersMiddlewareName = `${srKey}-headers-middleware`; - config_output.http.middlewares[srHeadersMiddlewareName] = { - headers: { - customRequestHeaders: { - ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), - "p-host": fullDomain - } + if (!matchingCert) { + logger.debug( + `No matching certificate found for inference siteResource fullDomain: ${fullDomain}` + ); + continue; } - }; - routerMiddlewares.push(srHeadersMiddlewareName); - } + } - routerMiddlewares.push(...additionalMiddlewares); + const additionalMiddlewares = + config.getRawConfig().traefik.additional_middlewares || []; + const routerMiddlewares: string[] = [ + ...(enableAiGatewayClientIpHeader + ? [aiGatewayClientIpMiddlewareName] + : []), + aiGatewayTrustMiddlewareNameSiteResource + ]; - if (sr.ssl) { - config_output.http.routers[routerName + "-redirect"] = { + if (aiGatewayOverride) { + const srHeadersMiddlewareName = `${srKey}-headers-middleware`; + config_output.http.middlewares[srHeadersMiddlewareName] = { + headers: { + customRequestHeaders: { + ...(aiGatewayHost + ? { Host: aiGatewayHost } + : {}), + "p-host": fullDomain + } + } + }; + routerMiddlewares.push(srHeadersMiddlewareName); + } + + routerMiddlewares.push(...additionalMiddlewares); + + if (sr.ssl) { + config_output.http.routers[routerName + "-redirect"] = { + entryPoints: [ + config.getRawConfig().traefik.http_entrypoint + ], + middlewares: [redirectHttpsMiddlewareName], + service: serviceName, + rule, + priority: 200 // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. + }; + } + + config_output.http.routers[routerName] = { entryPoints: [ - config.getRawConfig().traefik.http_entrypoint + sr.ssl + ? config.getRawConfig().traefik.https_entrypoint + : config.getRawConfig().traefik.http_entrypoint ], - middlewares: [redirectHttpsMiddlewareName], + middlewares: routerMiddlewares, service: serviceName, rule, - priority: 200 // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. + priority: 200, // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. + ...(sr.ssl ? { tls } : {}) + }; + + config_output.http.services[serviceName] = { + loadBalancer: { + servers: [{ url: aiGatewayUrl }] + } }; } - - config_output.http.routers[routerName] = { - entryPoints: [ - sr.ssl - ? config.getRawConfig().traefik.https_entrypoint - : config.getRawConfig().traefik.http_entrypoint - ], - middlewares: routerMiddlewares, - service: serviceName, - rule, - priority: 200, // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. - ...(sr.ssl ? { tls } : {}) - }; - - config_output.http.services[serviceName] = { - loadBalancer: { - servers: [{ url: aiGatewayUrl }] - } - }; } } From 954dc8d1d9da5ba90d3422de412ed463e4db7265 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 12:04:04 -0400 Subject: [PATCH 191/296] Handle overlapping fullDomain properly --- server/lib/blueprints/privateResources.ts | 4 +- server/lib/blueprints/publicResources.ts | 72 +++++++++++++------ .../siteResource/createSiteResource.ts | 7 +- .../siteResource/updateSiteResource.ts | 7 +- 4 files changed, 59 insertions(+), 31 deletions(-) diff --git a/server/lib/blueprints/privateResources.ts b/server/lib/blueprints/privateResources.ts index bcbee7ee6..97903e722 100644 --- a/server/lib/blueprints/privateResources.ts +++ b/server/lib/blueprints/privateResources.ts @@ -48,7 +48,9 @@ async function getDomainForSiteResource( eq(siteResources.orgId, orgId), // exclude looking at the ones on exit nodes if this is an inference resource, // and vice versa, so inference and non-inference resources can share a full-domain - ne(siteResources.requiresExitNodeConnection, !isInference), + isInference + ? ne(siteResources.mode, "inference") + : eq(siteResources.mode, "inference"), siteResourceId ? ne(siteResources.siteResourceId, siteResourceId) : isNotNull(siteResources.siteResourceId) diff --git a/server/lib/blueprints/publicResources.ts b/server/lib/blueprints/publicResources.ts index 4adc981ce..b495b6e5e 100644 --- a/server/lib/blueprints/publicResources.ts +++ b/server/lib/blueprints/publicResources.ts @@ -289,7 +289,9 @@ export async function updatePublicResources( if (existingResource) { let domain; if ( - ["http", "ssh", "rdp", "vnc", "inference"].includes(resourceData.mode || "") + ["http", "ssh", "rdp", "vnc", "inference"].includes( + resourceData.mode || "" + ) ) { if (resourceData["full-domain"]?.startsWith("*.")) { const isLicensed = await isLicensedOrSubscribed( @@ -374,14 +376,22 @@ export async function updatePublicResources( name: resourceData.name || "Unnamed Resource", mode: resourceData.mode, - proxyPort: ["http", "ssh", "rdp", "vnc", "inference"].includes( - resourceData.mode || "" - ) + proxyPort: [ + "http", + "ssh", + "rdp", + "vnc", + "inference" + ].includes(resourceData.mode || "") ? null : resourceData["proxy-port"], - fullDomain: ["http", "ssh", "rdp", "vnc", "inference"].includes( - resourceData.mode || "" - ) + fullDomain: [ + "http", + "ssh", + "rdp", + "vnc", + "inference" + ].includes(resourceData.mode || "") ? resourceData["full-domain"] : null, subdomain: domain ? domain.subdomain : null, @@ -571,14 +581,22 @@ export async function updatePublicResources( .set({ name: resourceData.name || "Unnamed Resource", mode: resourceData.mode, - proxyPort: ["http", "ssh", "rdp", "vnc", "inference"].includes( - resourceData.mode || "" - ) + proxyPort: [ + "http", + "ssh", + "rdp", + "vnc", + "inference" + ].includes(resourceData.mode || "") ? null : resourceData["proxy-port"], - fullDomain: ["http", "ssh", "rdp", "vnc", "inference"].includes( - resourceData.mode || "" - ) + fullDomain: [ + "http", + "ssh", + "rdp", + "vnc", + "inference" + ].includes(resourceData.mode || "") ? resourceData["full-domain"] : null, subdomain: domain ? domain.subdomain : null, @@ -1090,7 +1108,9 @@ export async function updatePublicResources( let domain; if ( - ["http", "ssh", "rdp", "vnc", "inference"].includes(resourceData.mode || "") + ["http", "ssh", "rdp", "vnc", "inference"].includes( + resourceData.mode || "" + ) ) { if (resourceData["full-domain"]?.startsWith("*.")) { const isLicensed = await isLicensedOrSubscribed( @@ -1188,14 +1208,22 @@ export async function updatePublicResources( status: resourceStatusFromSite, name: resourceData.name || "Unnamed Resource", mode: resourceData.mode, - proxyPort: ["http", "ssh", "rdp", "vnc", "inference"].includes( - resourceData.mode || "" - ) + proxyPort: [ + "http", + "ssh", + "rdp", + "vnc", + "inference" + ].includes(resourceData.mode || "") ? null : resourceData["proxy-port"], - fullDomain: ["http", "ssh", "rdp", "vnc", "inference"].includes( - resourceData.mode || "" - ) + fullDomain: [ + "http", + "ssh", + "rdp", + "vnc", + "inference" + ].includes(resourceData.mode || "") ? resourceData["full-domain"] : null, subdomain: domain ? domain.subdomain : null, @@ -2148,8 +2176,8 @@ export async function getDomain( // resource (and vice versa) - only conflicts within the // same routing category are rejected. isInference - ? eq(resources.mode, "inference") - : ne(resources.mode, "inference"), + ? ne(resources.mode, "inference") + : eq(resources.mode, "inference"), resourceId ? ne(resources.resourceId, resourceId) : isNotNull(resources.resourceId) diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 2038f9ce7..5887aae52 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -518,10 +518,9 @@ export async function createSiteResource( .where( and( eq(siteResources.fullDomain, fullDomain), - ne( - siteResources.requiresExitNodeConnection, - mode == "inference" - ) + mode == "inference" + ? ne(siteResources.mode, "inference") + : eq(siteResources.mode, "inference") ) ); // exclude looking at the ones on exit nodes if this is an inference resource diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index 3ec8f7f65..e0740e8c7 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -484,10 +484,9 @@ export async function updateSiteResource( .where( and( eq(siteResources.fullDomain, fullDomain), - ne( - siteResources.requiresExitNodeConnection, - mode == "inference" - ) + mode == "inference" + ? ne(siteResources.mode, "inference") + : eq(siteResources.mode, "inference") ) ); // exclude looking at the ones on exit nodes if this is an inference resource From 193f6da8d1fa1bdcd8496a97f26aa6bf437e6873 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 12:23:07 -0400 Subject: [PATCH 192/296] Handle the list of models without allow/block --- server/lib/blueprints/aiProviders.ts | 28 +++++++++-------------- server/lib/blueprints/privateResources.ts | 10 ++------ server/lib/blueprints/publicResources.ts | 10 ++------ server/lib/blueprints/types.ts | 18 ++++++++------- 4 files changed, 25 insertions(+), 41 deletions(-) diff --git a/server/lib/blueprints/aiProviders.ts b/server/lib/blueprints/aiProviders.ts index 5dbf18bf0..14e9914a3 100644 --- a/server/lib/blueprints/aiProviders.ts +++ b/server/lib/blueprints/aiProviders.ts @@ -26,7 +26,7 @@ export type BlueprintAiProviderInput = { provider: string; accessMode: AccessMode; enabled: boolean; - models: BlueprintAiModelInput[]; + models: string[]; }; async function resolveProviderNiceIds( @@ -99,12 +99,12 @@ async function resolveModelKeys( for (const provider of providers) { const providerId = providerIdByNiceId.get(provider.provider)!; for (const m of provider.models) { - const modelId = byProviderAndKey.get(`${providerId}::${m.model}`); + const modelId = byProviderAndKey.get(`${providerId}::${m}`); if (modelId === undefined) { - missing.push(`${provider.provider}/${m.model}`); + missing.push(`${provider.provider}/${m}`); continue; } - modelIdByEntryKey.set(`${provider.provider}::${m.model}`, modelId); + modelIdByEntryKey.set(`${provider.provider}::${m}`, modelId); } } @@ -117,7 +117,7 @@ async function resolveModelKeys( async function validateModelEntries(input: { orgId: string; - entries: { modelId: number; listType: ModelListType }[]; + entries: { modelId: number }[]; selectProviderIds: number[]; trx: Transaction; }): Promise { @@ -157,13 +157,10 @@ async function validateModelEntries(input: { `Model ${entry.modelId} does not exist or does not belong to a select-mode attached provider` ); } - if (catalog.listType !== entry.listType) { - throw new Error( - `Model ${entry.modelId} must use list-type "${catalog.listType}" to match the provider catalog entry` - ); - } if (!catalog.enabled) { - throw new Error(`Model ${entry.modelId} is disabled on its provider`); + throw new Error( + `Model ${entry.modelId} is disabled on its provider` + ); } } } @@ -238,8 +235,7 @@ export async function syncInferenceAiConfig( const modelEntries = input.providers.flatMap((p) => p.models.map((m) => ({ - modelId: modelIdByEntryKey.get(`${p.provider}::${m.model}`)!, - listType: m.listType + modelId: modelIdByEntryKey.get(`${p.provider}::${m}`)! })) ); @@ -262,8 +258,7 @@ export async function syncInferenceAiConfig( await trx.insert(resourceAiModels).values( modelEntries.map((m) => ({ resourceId: input.resourceId, - modelId: m.modelId, - listType: m.listType + modelId: m.modelId })) ); } @@ -277,8 +272,7 @@ export async function syncInferenceAiConfig( await trx.insert(siteResourceAiModels).values( modelEntries.map((m) => ({ siteResourceId: input.siteResourceId, - modelId: m.modelId, - listType: m.listType + modelId: m.modelId })) ); } diff --git a/server/lib/blueprints/privateResources.ts b/server/lib/blueprints/privateResources.ts index 97903e722..40d058634 100644 --- a/server/lib/blueprints/privateResources.ts +++ b/server/lib/blueprints/privateResources.ts @@ -363,10 +363,7 @@ export async function updatePrivateResources( provider: p.provider, accessMode: p["access-mode"], enabled: p.enabled, - models: p.models.map((m) => ({ - model: m.model, - listType: m["list-type"] - })) + models: p.models })) }); @@ -672,10 +669,7 @@ export async function updatePrivateResources( provider: p.provider, accessMode: p["access-mode"], enabled: p.enabled, - models: p.models.map((m) => ({ - model: m.model, - listType: m["list-type"] - })) + models: p.models })) }); diff --git a/server/lib/blueprints/publicResources.ts b/server/lib/blueprints/publicResources.ts index b495b6e5e..2f4d9773d 100644 --- a/server/lib/blueprints/publicResources.ts +++ b/server/lib/blueprints/publicResources.ts @@ -708,10 +708,7 @@ export async function updatePublicResources( provider: p.provider, accessMode: p["access-mode"], enabled: p.enabled, - models: p.models.map((m) => ({ - model: m.model, - listType: m["list-type"] - })) + models: p.models }) ) }); @@ -1288,10 +1285,7 @@ export async function updatePublicResources( provider: p.provider, accessMode: p["access-mode"], enabled: p.enabled, - models: p.models.map((m) => ({ - model: m.model, - listType: m["list-type"] - })) + models: p.models })) }); diff --git a/server/lib/blueprints/types.ts b/server/lib/blueprints/types.ts index d9e9415d0..74c5d5ca2 100644 --- a/server/lib/blueprints/types.ts +++ b/server/lib/blueprints/types.ts @@ -188,11 +188,6 @@ export const HeaderSchema = z.object({ value: z.string().min(1) }); -export const AiProviderModelEntrySchema = z.object({ - model: z.string().min(1), - "list-type": z.enum(["allow", "block"]) -}); - export const AiProviderAttachmentSchema = z .object({ provider: z.string().min(1), @@ -201,7 +196,7 @@ export const AiProviderAttachmentSchema = z .optional() .default("inherit"), enabled: z.boolean().optional().default(true), - models: z.array(AiProviderModelEntrySchema).optional().default([]) + models: z.array(z.string()).optional().default([]) }) .refine( (provider) => { @@ -598,7 +593,10 @@ export const PrivateResourceSchema = z machines: z.array(z.string()).optional().default([]), labels: z.array(z.string().min(1)).optional().default([]), "auth-daemon": AuthDaemonSchema.optional(), - "ai-providers": z.array(AiProviderAttachmentSchema).optional().default([]), + "ai-providers": z + .array(AiProviderAttachmentSchema) + .optional() + .default([]), "ai-budget": AiBudgetListSchemaWithDefault }) .refine( @@ -608,7 +606,11 @@ export const PrivateResourceSchema = z data.mode === "ssh" && (data["auth-daemon"] === undefined || data["auth-daemon"].mode === "native"); - if (data.mode !== "inference" && !isNativeSSH && !data.destination) { + if ( + data.mode !== "inference" && + !isNativeSSH && + !data.destination + ) { return false; } return true; From d7bff686c0a9b7f2b25d3b3e96a4c8c2853f29a4 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 14 Aug 2026 12:23:50 -0400 Subject: [PATCH 193/296] add help banner to providers and virtual api keys --- messages/en-US.json | 5 +++ .../[orgId]/settings/ai-providers/page.tsx | 3 ++ .../settings/virtual-api-keys/page.tsx | 3 ++ src/components/AiProvidersBanner.tsx | 21 +++++++++ src/components/VirtualApiKeysBanner.tsx | 45 +++++++++++++++++++ 5 files changed, 77 insertions(+) create mode 100644 src/components/AiProvidersBanner.tsx create mode 100644 src/components/VirtualApiKeysBanner.tsx diff --git a/messages/en-US.json b/messages/en-US.json index 52458fcbe..b58ebfd0c 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1678,6 +1678,9 @@ "commandVirtualApiKeys": "Virtual API Keys", "virtualApiKeysTitle": "Manage Virtual API Keys", "virtualApiKeysDescription": "Create and manage manual API keys for AI Gateway access to public AI gateways", + "virtualApiKeysBannerTitle": "Identity Keys for Every User", + "virtualApiKeysBannerDescription": "Every user already has an identity key available at {keysUrl}. You can also manually generate keys here that grant direct access to public AI gateways.", + "virtualApiKeysBannerButtonText": "View Identity Keys", "virtualApiKeys": "Virtual API Keys", "virtualApiKeysSearch": "Search keys...", "virtualApiKeysCreate": "Create Virtual API Key", @@ -1742,6 +1745,8 @@ "myVirtualApiKeysViewSecretDescription": "This secret authenticates you to AI Gateway resources", "aiProvidersTitle": "AI Providers", "aiProvidersDescription": "Connect model providers for AI workloads in this organization", + "aiProvidersBannerTitle": "Connect Model Providers", + "aiProvidersBannerDescription": "Providers are the model backends Pangolin uses for AI workloads. Connect OpenAI, Anthropic, and other providers here, then attach them to public AI gateways so users can call models with identity-aware access, budgets, and logging.", "aiProvidersAdd": "Add Provider", "aiProvidersSearch": "Search providers...", "aiProvidersEmpty": "No AI providers yet", diff --git a/src/app/[orgId]/settings/ai-providers/page.tsx b/src/app/[orgId]/settings/ai-providers/page.tsx index 024170590..572ce90d4 100644 --- a/src/app/[orgId]/settings/ai-providers/page.tsx +++ b/src/app/[orgId]/settings/ai-providers/page.tsx @@ -1,3 +1,4 @@ +import AiProvidersBanner from "@app/components/AiProvidersBanner"; import AiProvidersTable from "@app/components/AiProvidersTable"; import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; import { internal } from "@app/lib/api"; @@ -49,6 +50,8 @@ export default async function AiProvidersPage({ params, searchParams }: Props) { description={t("aiProvidersDescription")} /> + + ({ diff --git a/src/app/[orgId]/settings/virtual-api-keys/page.tsx b/src/app/[orgId]/settings/virtual-api-keys/page.tsx index ba8f4ea01..28a73253e 100644 --- a/src/app/[orgId]/settings/virtual-api-keys/page.tsx +++ b/src/app/[orgId]/settings/virtual-api-keys/page.tsx @@ -6,6 +6,7 @@ import { redirect } from "next/navigation"; import { cache } from "react"; import { GetOrgResponse } from "@server/routers/org"; import OrgProvider from "@app/providers/OrgProvider"; +import VirtualApiKeysBanner from "@app/components/VirtualApiKeysBanner"; import VirtualApiKeysTable, { type VirtualApiKeyRow } from "@app/components/VirtualApiKeysTable"; @@ -142,6 +143,8 @@ export default async function VirtualApiKeysPage( description={t("virtualApiKeysDescription")} /> + + { + const t = useTranslations(); + + return ( + } + description={t("aiProvidersBannerDescription")} + /> + ); +}; + +export default AiProvidersBanner; diff --git a/src/components/VirtualApiKeysBanner.tsx b/src/components/VirtualApiKeysBanner.tsx new file mode 100644 index 000000000..dd2aa1186 --- /dev/null +++ b/src/components/VirtualApiKeysBanner.tsx @@ -0,0 +1,45 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { ArrowRight, KeyRound } from "lucide-react"; +import { useTranslations } from "next-intl"; +import Link from "next/link"; +import DismissableBanner from "./DismissableBanner"; + +type VirtualApiKeysBannerProps = { + orgId: string; +}; + +export const VirtualApiKeysBanner = ({ orgId }: VirtualApiKeysBannerProps) => { + const t = useTranslations(); + const { env } = useEnvContext(); + + const dashboardUrl = env.app.dashboardUrl?.replace(/\/$/, "") ?? ""; + const keysUrl = dashboardUrl + ? `${dashboardUrl}/${orgId}/keys` + : `/${orgId}/keys`; + + return ( + } + description={t("virtualApiKeysBannerDescription", { keysUrl })} + > + + + + + ); +}; + +export default VirtualApiKeysBanner; From ea740e12d18eaace1d70baff7ff20017aa391eca Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 14 Aug 2026 19:07:13 +0200 Subject: [PATCH 194/296] =?UTF-8?q?=F0=9F=92=84=20nice=20little=20animatio?= =?UTF-8?q?n=20on=20alert=20rule=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/alert-rule-editor/AlertRuleFields.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/components/alert-rule-editor/AlertRuleFields.tsx b/src/components/alert-rule-editor/AlertRuleFields.tsx index 147369a9e..155138866 100644 --- a/src/components/alert-rule-editor/AlertRuleFields.tsx +++ b/src/components/alert-rule-editor/AlertRuleFields.tsx @@ -60,6 +60,7 @@ import { useFormContext, useWatch } from "react-hook-form"; import { useDebounce } from "use-debounce"; import { RolesSelector } from "../roles-selector"; import { UsersSelector } from "../users-selector"; +import { cn } from "@app/lib/cn"; export function AddActionPanel({ onAdd @@ -149,7 +150,12 @@ export function AddActionPanel({ From 173d4a536fc71a7bcb6e59d12efa2719721fb827 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 14 Aug 2026 13:45:52 -0400 Subject: [PATCH 195/296] visual tweaks --- messages/en-US.json | 2 +- src/app/navigation.tsx | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index b58ebfd0c..e41d06f7b 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1690,7 +1690,7 @@ "virtualApiKeysName": "Name", "virtualApiKeysDescriptionOptional": "Description (optional)", "virtualApiKeysAssociateUserOptional": "Associate User (optional)", - "virtualApiKeysAssociateUserDescription": "Associate this key with a user to track usage.", + "virtualApiKeysAssociateUserDescription": "Associate this key with a user to track usage. Once created, this key immediately grants access to the selected public AI gateways. You do not need to associate a user to manually to the resource. The key will also show up in the user's profile.", "virtualApiKeysAllResources": "All public AI gateways", "virtualApiKeysAllResourcesDescription": "Allow this key to access every public AI gateway in the organization", "virtualApiKeysSelectResources": "Public AI Gateways", diff --git a/src/app/navigation.tsx b/src/app/navigation.tsx index 2f00744c0..724da9cc2 100644 --- a/src/app/navigation.tsx +++ b/src/app/navigation.tsx @@ -20,6 +20,8 @@ import { LayoutGrid, Link as LinkIcon, Logs, + MessageSquare, + MessagesSquare, MonitorUp, Plug, ReceiptText, @@ -233,7 +235,7 @@ export const orgNavSections = ( { title: "sidebarLogsAi", href: "/{orgId}/settings/logs/ai", - icon: + icon: }, { title: "sidebarLogsAiUsage", From 6d35b2088065ad19075069fc97fbad54c26356b1 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 14:14:56 -0400 Subject: [PATCH 196/296] Fix overlap check --- server/routers/resource/createResource.ts | 4 ++-- server/routers/resource/updateResource.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index 897d32f6a..c0c7a7d5d 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -496,8 +496,8 @@ async function createHttpResource( and( eq(resources.fullDomain, fullDomain), effectiveMode === "inference" - ? eq(resources.mode, "inference") - : ne(resources.mode, "inference") + ? ne(resources.mode, "inference") + : eq(resources.mode, "inference") ) ); diff --git a/server/routers/resource/updateResource.ts b/server/routers/resource/updateResource.ts index 3ed26c9cb..46e547b0b 100644 --- a/server/routers/resource/updateResource.ts +++ b/server/routers/resource/updateResource.ts @@ -609,8 +609,8 @@ async function updateHttpResource( and( eq(resources.fullDomain, fullDomain), resource.mode === "inference" - ? eq(resources.mode, "inference") - : ne(resources.mode, "inference") + ? ne(resources.mode, "inference") + : eq(resources.mode, "inference") ) ); From 3a18a3994f5e52e86ebebbe52bbbe11a3ef66da4 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 14:21:00 -0400 Subject: [PATCH 197/296] Fix private resource ssl when applying through blueprints --- server/lib/blueprints/privateResources.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/lib/blueprints/privateResources.ts b/server/lib/blueprints/privateResources.ts index 40d058634..9cec7b487 100644 --- a/server/lib/blueprints/privateResources.ts +++ b/server/lib/blueprints/privateResources.ts @@ -235,6 +235,13 @@ export async function updatePrivateResources( ? false : resourceData.enabled; + const resourceSsl = + resourceData.mode === "inference" || resourceData.mode === "http" + ? resourceData.ssl == undefined || resourceData.ssl == null + ? true + : resourceData.ssl + : resourceData.ssl; + if (existingResource) { let domainInfo: | { subdomain: string | null; domainId: string } @@ -286,7 +293,7 @@ export async function updatePrivateResources( .set({ name: resourceData.name || resourceNiceId, mode: resourceData.mode, - ssl: resourceData.ssl, + ssl: resourceSsl, scheme: resourceData.scheme, destination: resourceData.destination, destinationPort: resourceData["destination-port"], @@ -614,7 +621,7 @@ export async function updatePrivateResources( defaultNetworkId: network ? network.networkId : null, name: resourceData.name || resourceNiceId, mode: resourceData.mode, - ssl: resourceData.ssl, + ssl: resourceSsl, scheme: resourceData.scheme, destination: resourceData.destination, destinationPort: resourceData["destination-port"], From 32d76ee2accaf8e2e3b959fa2cf671f2fc0ed3cb Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 14:27:14 -0400 Subject: [PATCH 198/296] Resolve cert not generated without a provider --- .../private/lib/traefik/getTraefikConfig.ts | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index cc841fc1e..bea78fa3e 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -41,10 +41,7 @@ import { siteNetworks, siteResources, Target, - targets, - aiProviders, - resourceAiProviders, - siteResourceAiProviders + targets } from "@server/db"; import { sanitize, @@ -438,21 +435,9 @@ export async function getTraefikConfig( preferWildcardCert: domains.preferWildcardCert }) .from(resources) - .innerJoin( - resourceAiProviders, - eq(resources.resourceId, resourceAiProviders.resourceId) - ) - .innerJoin( - aiProviders, - eq(resourceAiProviders.providerId, aiProviders.providerId) - ) .leftJoin(domains, eq(domains.domainId, resources.domainId)) .where( - and( - eq(resources.mode, "inference"), - eq(resources.enabled, true), - eq(aiProviders.enabled, true) - ) + and(eq(resources.mode, "inference"), eq(resources.enabled, true)) ); const siteResourcesInference = await db From 111d4b1b8c510842c77f42670d17edea9a652ea7 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 14:33:59 -0400 Subject: [PATCH 199/296] Make sure certs are created when creating and updating private resources --- server/routers/siteResource/createSiteResource.ts | 2 +- server/routers/siteResource/updateSiteResource.ts | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 5887aae52..f3794d35a 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -769,7 +769,7 @@ export async function createSiteResource( if ( ssl && - mode === "http" && + (mode === "http" || mode == "inference") && domainId && fullDomain && build != "oss" diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index e0740e8c7..2e57b13a7 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -30,6 +30,8 @@ import createHttpError from "http-errors"; import { z } from "zod"; import { fromError } from "zod-validation-error"; import { clearSiteResourceAiConfig } from "@server/lib/aiInferenceResource"; +import { build } from "@server/build"; +import { createCertificate } from "../certificates/createCertificate"; const updateSiteResourceParamsSchema = z.strictObject({ siteResourceId: z.coerce.number().int().positive() @@ -735,6 +737,16 @@ export async function updateSiteResource( const finalUpdatedSiteResource = updatedSiteResource; + if ( + ssl && + (mode === "http" || mode == "inference") && + domainId && + fullDomain && + build != "oss" + ) { + await createCertificate(domainId, fullDomain, db); + } + rebuildClientAssociationsFromSiteResource(finalUpdatedSiteResource) .then(() => waitForSiteResourceRebuildIdle( From a3dfb30a428bf8d6825ddb7b4747d8d3490bc66d Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 14 Aug 2026 16:51:10 -0400 Subject: [PATCH 200/296] send email upon generate virtual api key --- messages/en-US.json | 12 ++ package.json | 4 +- .../templates/VirtualApiKeyGenerated.tsx | 118 +++++++++++++ server/lib/sendVirtualApiKeyEmail.ts | 161 ++++++++++++++++++ .../virtualApiKey/createVirtualApiKey.ts | 41 ++++- .../virtualApiKey/updateVirtualApiKey.ts | 38 +++++ server/routers/virtualApiKey/validation.ts | 75 ++++++-- src/components/CreateVirtualApiKeyForm.tsx | 55 +++++- src/components/EditVirtualApiKeyForm.tsx | 51 +++++- src/components/VirtualApiKeyEmailSection.tsx | 151 ++++++++++++++++ 10 files changed, 689 insertions(+), 17 deletions(-) create mode 100644 server/emails/templates/VirtualApiKeyGenerated.tsx create mode 100644 server/lib/sendVirtualApiKeyEmail.ts create mode 100644 src/components/VirtualApiKeyEmailSection.tsx diff --git a/messages/en-US.json b/messages/en-US.json index e41d06f7b..f918a51c3 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1725,6 +1725,18 @@ "virtualApiKeysFilterUnassigned": "Unassigned", "virtualApiKeysInferenceBudget": "Budget", "virtualApiKeysInferenceBudgetDescription": "Configure how this key restricts AI usage based on spending or token limits", + "virtualApiKeysEmailOnGenerate": "Email key upon generation", + "virtualApiKeysEmailThisKey": "Email this key", + "virtualApiKeysEmailOnGenerateDescription": "Send the key to the associated user and additional addresses after it is created", + "virtualApiKeysEmailThisKeyDescription": "Send the current key to the associated user and additional addresses", + "virtualApiKeysEmailSmtpRequired": "Email is not configured on this server", + "virtualApiKeysEmailSmtpRequiredDescription": "Configure SMTP to email virtual API keys.", + "virtualApiKeysEmailSendToUser": "Send to associated user", + "virtualApiKeysEmailSendToUserDescription": "Email the key to the associated user's account email", + "virtualApiKeysEmailSendToUserDisabled": "Associate a user to send the key to that user", + "virtualApiKeysEmailAdditional": "Additional emails", + "virtualApiKeysEmailAdditionalPlaceholder": "Add email and press Enter", + "virtualApiKeysEmailRecipientsRequired": "Select the associated user or add at least one email address", "myVirtualApiKeysTitle": "Your API Keys", "myVirtualApiKeysDescription": "View your identity key and any virtual API keys attributed to you in this organization", "myVirtualApiKeysResourceTitle": "Your API Keys for {resourceName}", diff --git a/package.json b/package.json index e94b2fda3..32c22d4fa 100644 --- a/package.json +++ b/package.json @@ -166,7 +166,7 @@ "@types/yargs": "17.0.35", "babel-plugin-react-compiler": "1.0.0", "drizzle-kit": "0.31.10", - "esbuild": "0.28.1", + "esbuild": "0.28.0", "esbuild-node-externals": "1.22.0", "eslint": "10.4.0", "eslint-config-next": "16.2.6", @@ -180,7 +180,7 @@ "typescript-eslint": "8.60.0" }, "overrides": { - "esbuild": "0.28.1", + "esbuild": "0.28.0", "dompurify": "3.4.0", "postcss": "8.5.15" } diff --git a/server/emails/templates/VirtualApiKeyGenerated.tsx b/server/emails/templates/VirtualApiKeyGenerated.tsx new file mode 100644 index 000000000..f156c1b6e --- /dev/null +++ b/server/emails/templates/VirtualApiKeyGenerated.tsx @@ -0,0 +1,118 @@ +import React from "react"; +import { Body, Head, Html, Preview, Tailwind } from "@react-email/components"; +import { themeColors } from "./lib/theme"; +import { + EmailContainer, + EmailFooter, + EmailGreeting, + EmailInfoSection, + EmailLetterHead, + EmailSection, + EmailSignature, + EmailText +} from "./components/Email"; + +type VirtualApiKeyGeneratedProps = { + orgName: string; + keyName: string | null; + credential: string; + resourceUrls: string[]; + hasMoreResources: boolean; +}; + +export const VirtualApiKeyGenerated = ({ + orgName, + keyName, + credential, + resourceUrls, + hasMoreResources +}: VirtualApiKeyGeneratedProps) => { + const previewText = `A virtual API key for ${orgName} has been shared with you`; + + return ( + + + {previewText} + + + + + + Hi there, + + + A virtual API key for {orgName} has + been shared with you. Treat this key like a password + and do not share it. + + + + Your virtual API key: +
+
+ + {credential} + +
+
+ + + + + {resourceUrls.length > 0 && ( + <> + + This key can be used to authenticate to the + following AI gateway resources: + +
+ {resourceUrls.map((url) => ( +

+ + {url} + +

+ ))} +
+ {hasMoreResources && ( + + Contact your administrator to get the + full list. + + )} + + )} + + + + + + + + + ); +}; + +export default VirtualApiKeyGenerated; diff --git a/server/lib/sendVirtualApiKeyEmail.ts b/server/lib/sendVirtualApiKeyEmail.ts new file mode 100644 index 000000000..48d7d0d6b --- /dev/null +++ b/server/lib/sendVirtualApiKeyEmail.ts @@ -0,0 +1,161 @@ +import { db, resources, users, virtualApiKeyResources } from "@server/db"; +import { and, asc, eq } from "drizzle-orm"; +import config from "@server/lib/config"; +import { sendEmail } from "@server/emails"; +import VirtualApiKeyGenerated from "@server/emails/templates/VirtualApiKeyGenerated"; +import { formatVirtualApiKeyCredential } from "@server/lib/virtualApiKey"; + +const EMAIL_GATEWAY_URL_LIMIT = 5; + +async function listVirtualApiKeyGatewayUrls(params: { + orgId: string; + allResources: boolean; + virtualApiKeyId: string; +}): Promise<{ urls: string[]; hasMore: boolean }> { + const rows = params.allResources + ? await db + .select({ + fullDomain: resources.fullDomain, + ssl: resources.ssl + }) + .from(resources) + .where( + and( + eq(resources.orgId, params.orgId), + eq(resources.mode, "inference") + ) + ) + .orderBy(asc(resources.name)) + .limit(EMAIL_GATEWAY_URL_LIMIT + 1) + : await db + .select({ + fullDomain: resources.fullDomain, + ssl: resources.ssl + }) + .from(virtualApiKeyResources) + .innerJoin( + resources, + eq(virtualApiKeyResources.resourceId, resources.resourceId) + ) + .where( + eq( + virtualApiKeyResources.virtualApiKeyId, + params.virtualApiKeyId + ) + ) + .orderBy(asc(resources.name)) + .limit(EMAIL_GATEWAY_URL_LIMIT + 1); + + const urls = rows + .map((row) => + row.fullDomain + ? `${row.ssl ? "https" : "http"}://${row.fullDomain}` + : null + ) + .filter((url): url is string => Boolean(url)); + + return { + urls: urls.slice(0, EMAIL_GATEWAY_URL_LIMIT), + hasMore: rows.length > EMAIL_GATEWAY_URL_LIMIT + }; +} + +export async function resolveVirtualApiKeyEmailRecipients(params: { + sendEmail: boolean; + sendToAttributedUser: boolean; + userId: string | null | undefined; + emails: string[]; +}): Promise< + { ok: true; recipients: string[] } | { ok: false; message: string } +> { + if (!params.sendEmail) { + return { ok: true, recipients: [] }; + } + + if (!config.getRawConfig().email) { + return { + ok: false, + message: "Email is not configured on this server" + }; + } + + const recipients = new Set( + params.emails.map((email) => email.trim().toLowerCase()).filter(Boolean) + ); + + if (params.sendToAttributedUser) { + if (!params.userId) { + return { + ok: false, + message: "Associate a user to email the key to that user" + }; + } + + const [user] = await db + .select({ email: users.email }) + .from(users) + .where(eq(users.userId, params.userId)) + .limit(1); + + if (!user?.email) { + return { + ok: false, + message: "The associated user does not have an email address" + }; + } + + recipients.add(user.email.toLowerCase()); + } + + if (recipients.size === 0) { + return { + ok: false, + message: "Select at least one email recipient" + }; + } + + return { ok: true, recipients: [...recipients] }; +} + +export async function sendVirtualApiKeyEmails(params: { + recipients: string[]; + orgName: string; + orgId: string; + keyName: string | null; + virtualApiKeyId: string; + secret: string; + allResources: boolean; +}): Promise { + if (params.recipients.length === 0) { + return; + } + + const credential = formatVirtualApiKeyCredential( + params.virtualApiKeyId, + params.secret + ); + const { urls, hasMore } = await listVirtualApiKeyGatewayUrls({ + orgId: params.orgId, + allResources: params.allResources, + virtualApiKeyId: params.virtualApiKeyId + }); + const from = config.getNoReplyEmail(); + const subject = `Virtual API key for ${params.orgName}`; + + for (const to of params.recipients) { + await sendEmail( + VirtualApiKeyGenerated({ + orgName: params.orgName, + keyName: params.keyName, + credential, + resourceUrls: urls, + hasMoreResources: hasMore + }), + { + to, + from, + subject + } + ); + } +} diff --git a/server/routers/virtualApiKey/createVirtualApiKey.ts b/server/routers/virtualApiKey/createVirtualApiKey.ts index 5e49a5717..068ce13cc 100644 --- a/server/routers/virtualApiKey/createVirtualApiKey.ts +++ b/server/routers/virtualApiKey/createVirtualApiKey.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { db, userOrgs, virtualApiKeys } from "@server/db"; +import { db, orgs, userOrgs, virtualApiKeys } from "@server/db"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; @@ -18,6 +18,10 @@ import { } from "@server/lib/virtualApiKey"; import type { CreateOrEditVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; import { createVirtualApiKeyBodySchema } from "@server/routers/virtualApiKey/validation"; +import { + resolveVirtualApiKeyEmailRecipients, + sendVirtualApiKeyEmails +} from "@server/lib/sendVirtualApiKeyEmail"; const paramsSchema = z.strictObject({ orgId: z.string().nonempty() @@ -78,7 +82,10 @@ export async function createVirtualApiKey( userId, allResources, resourceIds, - validForSeconds + validForSeconds, + sendEmail: doEmail, + sendToAttributedUser, + emails } = parsedBody.data; if (req.user && orgId && orgId !== req.userOrgId) { @@ -121,6 +128,18 @@ export async function createVirtualApiKey( ); } + const emailRecipients = await resolveVirtualApiKeyEmailRecipients({ + sendEmail: doEmail, + sendToAttributedUser, + userId, + emails + }); + if (!emailRecipients.ok) { + return next( + createHttpError(HttpCode.BAD_REQUEST, emailRecipients.message) + ); + } + const minted = mintVirtualApiKeySecret(); const expiresAt = validForSeconds ? createDate(new TimeSpan(validForSeconds, "s")).getTime() @@ -156,6 +175,24 @@ export async function createVirtualApiKey( return row; }); + if (emailRecipients.recipients.length > 0) { + const [org] = await db + .select() + .from(orgs) + .where(eq(orgs.orgId, orgId)) + .limit(1); + + await sendVirtualApiKeyEmails({ + recipients: emailRecipients.recipients, + orgName: org?.name || orgId, + orgId, + keyName: created.name, + virtualApiKeyId: created.virtualApiKeyId, + secret: minted.secret, + allResources: created.allResources + }); + } + return response(res, { data: { virtualApiKey: { diff --git a/server/routers/virtualApiKey/updateVirtualApiKey.ts b/server/routers/virtualApiKey/updateVirtualApiKey.ts index 42554803f..8307d4fd3 100644 --- a/server/routers/virtualApiKey/updateVirtualApiKey.ts +++ b/server/routers/virtualApiKey/updateVirtualApiKey.ts @@ -2,6 +2,7 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; import { db, + orgs, userOrgs, virtualApiKeyResources, virtualApiKeys @@ -16,11 +17,16 @@ import { and, eq } from "drizzle-orm"; import { createDate, TimeSpan } from "oslo"; import { assertManualKeyResourcesInOrg, + decryptVirtualApiKeyToken, replaceVirtualApiKeyResources, toPublicVirtualApiKey } from "@server/lib/virtualApiKey"; import type { CreateOrEditVirtualApiKeyResponse } from "@server/routers/virtualApiKey/types"; import { updateVirtualApiKeyBodySchema } from "@server/routers/virtualApiKey/validation"; +import { + resolveVirtualApiKeyEmailRecipients, + sendVirtualApiKeyEmails +} from "@server/lib/sendVirtualApiKeyEmail"; const paramsSchema = z.strictObject({ virtualApiKeyId: z.string().nonempty() @@ -146,6 +152,20 @@ export async function updateVirtualApiKey( } } + const nextUserId = + body.userId !== undefined ? body.userId : existing.userId; + const emailRecipients = await resolveVirtualApiKeyEmailRecipients({ + sendEmail: body.sendEmail, + sendToAttributedUser: body.sendToAttributedUser, + userId: nextUserId, + emails: body.emails + }); + if (!emailRecipients.ok) { + return next( + createHttpError(HttpCode.BAD_REQUEST, emailRecipients.message) + ); + } + const updates: Partial = {}; if (body.name !== undefined) { @@ -192,6 +212,24 @@ export async function updateVirtualApiKey( return row; }); + if (emailRecipients.recipients.length > 0) { + const [org] = await db + .select() + .from(orgs) + .where(eq(orgs.orgId, existing.orgId)) + .limit(1); + + await sendVirtualApiKeyEmails({ + recipients: emailRecipients.recipients, + orgName: org?.name || existing.orgId, + orgId: existing.orgId, + keyName: updated.name, + virtualApiKeyId: updated.virtualApiKeyId, + secret: decryptVirtualApiKeyToken(updated.token), + allResources: updated.allResources + }); + } + const resourceRows = await db .select({ resourceId: virtualApiKeyResources.resourceId }) .from(virtualApiKeyResources) diff --git a/server/routers/virtualApiKey/validation.ts b/server/routers/virtualApiKey/validation.ts index 42964b3ba..5e8d22e64 100644 --- a/server/routers/virtualApiKey/validation.ts +++ b/server/routers/virtualApiKey/validation.ts @@ -4,6 +4,43 @@ export const virtualApiKeyResourceIdsSchema = z .array(z.coerce.number().int().positive()) .optional(); +const virtualApiKeyEmailFieldsSchema = { + sendEmail: z.boolean().optional().default(false), + sendToAttributedUser: z.boolean().optional().default(false), + emails: z.array(z.email().toLowerCase()).max(20).optional().default([]) +}; + +function refineVirtualApiKeyEmailFields( + data: { + sendEmail: boolean; + sendToAttributedUser: boolean; + emails: string[]; + userId?: string | null; + }, + ctx: z.RefinementCtx +) { + if (!data.sendEmail) { + return; + } + + if (!data.sendToAttributedUser && data.emails.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "Select the associated user or add at least one email address", + path: ["sendEmail"] + }); + } + + if (data.sendToAttributedUser && !data.userId) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Associate a user to email the key to that user", + path: ["sendToAttributedUser"] + }); + } +} + export const createVirtualApiKeyBodySchema = z .strictObject({ name: z.string().nonempty(), @@ -11,7 +48,8 @@ export const createVirtualApiKeyBodySchema = z userId: z.string().optional().nullable(), allResources: z.boolean().optional().default(false), resourceIds: virtualApiKeyResourceIdsSchema, - validForSeconds: z.int().positive().optional() + validForSeconds: z.int().positive().optional(), + ...virtualApiKeyEmailFieldsSchema }) .refine( (data) => data.allResources || (data.resourceIds?.length ?? 0) > 0, @@ -20,13 +58,30 @@ export const createVirtualApiKeyBodySchema = z "Select at least one public inference resource, or enable all public inference resources", path: ["resourceIds"] } - ); + ) + .superRefine(refineVirtualApiKeyEmailFields); -export const updateVirtualApiKeyBodySchema = z.strictObject({ - name: z.string().nonempty().optional(), - description: z.string().optional().nullable(), - userId: z.string().optional().nullable(), - allResources: z.boolean().optional(), - resourceIds: virtualApiKeyResourceIdsSchema, - validForSeconds: z.int().positive().optional().nullable() -}); +export const updateVirtualApiKeyBodySchema = z + .strictObject({ + name: z.string().nonempty().optional(), + description: z.string().optional().nullable(), + userId: z.string().optional().nullable(), + allResources: z.boolean().optional(), + resourceIds: virtualApiKeyResourceIdsSchema, + validForSeconds: z.int().positive().optional().nullable(), + ...virtualApiKeyEmailFieldsSchema + }) + .superRefine((data, ctx) => { + if (!data.sendEmail) { + return; + } + + if (!data.sendToAttributedUser && data.emails.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "Select the associated user or add at least one email address", + path: ["sendEmail"] + }); + } + }); diff --git a/src/components/CreateVirtualApiKeyForm.tsx b/src/components/CreateVirtualApiKeyForm.tsx index 5c2e75d0f..14bca22f5 100644 --- a/src/components/CreateVirtualApiKeyForm.tsx +++ b/src/components/CreateVirtualApiKeyForm.tsx @@ -54,6 +54,8 @@ import { getBudgetRowsErrors, type BudgetRow } from "@app/components/BudgetsEditor"; +import VirtualApiKeyEmailSection from "@app/components/VirtualApiKeyEmailSection"; +import type { Tag } from "@app/components/tags/tag-input"; export type CreatedVirtualApiKey = { virtualApiKeyId: string; @@ -101,6 +103,9 @@ export default function CreateVirtualApiKeyForm({ >([]); const [pendingBudgetRows, setPendingBudgetRows] = useState([]); const [attemptedBudgetsSave, setAttemptedBudgetsSave] = useState(false); + const [sendEmail, setSendEmail] = useState(false); + const [sendToAttributedUser, setSendToAttributedUser] = useState(false); + const [emailTags, setEmailTags] = useState([]); const formSchema = z.object({ name: z.string().min(1), @@ -123,6 +128,9 @@ export default function CreateVirtualApiKeyForm({ setSelectedResources([]); setPendingBudgetRows([]); setAttemptedBudgetsSave(false); + setSendEmail(false); + setSendToAttributedUser(false); + setEmailTags([]); form.reset(); } @@ -141,6 +149,20 @@ export default function CreateVirtualApiKeyForm({ return; } + if ( + env.email.emailEnabled && + sendEmail && + !sendToAttributedUser && + emailTags.length === 0 + ) { + toast({ + variant: "destructive", + title: t("virtualApiKeysEmailRecipientsRequired"), + description: t("virtualApiKeysEmailRecipientsRequired") + }); + return; + } + return onSubmit(values); } @@ -157,7 +179,16 @@ export default function CreateVirtualApiKeyForm({ allResources, resourceIds: allResources ? [] - : selectedResources.map((r) => r.resourceId) + : selectedResources.map((r) => r.resourceId), + sendEmail: env.email.emailEnabled && sendEmail, + sendToAttributedUser: + env.email.emailEnabled && + sendEmail && + sendToAttributedUser, + emails: + env.email.emailEnabled && sendEmail + ? emailTags.map((tag) => tag.text) + : [] } ) .catch((e) => { @@ -473,6 +504,26 @@ export default function CreateVirtualApiKeyForm({
)}
+ +
@@ -505,7 +556,7 @@ export default function CreateVirtualApiKeyForm({
)}
+ +
diff --git a/src/components/VirtualApiKeyEmailSection.tsx b/src/components/VirtualApiKeyEmailSection.tsx new file mode 100644 index 000000000..a00abd69e --- /dev/null +++ b/src/components/VirtualApiKeyEmailSection.tsx @@ -0,0 +1,151 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { Checkbox } from "@app/components/ui/checkbox"; +import { FormLabel } from "@app/components/ui/form"; +import { TagInput, type Tag } from "@app/components/tags/tag-input"; +import { useTranslations } from "next-intl"; + +type VirtualApiKeyEmailSectionProps = { + emailEnabled: boolean; + mode: "create" | "edit"; + sendEmail: boolean; + onSendEmailChange: (value: boolean) => void; + sendToAttributedUser: boolean; + onSendToAttributedUserChange: (value: boolean) => void; + hasAssociatedUser: boolean; + emailTags: Tag[]; + onEmailTagsChange: (tags: Tag[]) => void; +}; + +export default function VirtualApiKeyEmailSection({ + emailEnabled, + mode, + sendEmail, + onSendEmailChange, + sendToAttributedUser, + onSendToAttributedUserChange, + hasAssociatedUser, + emailTags, + onEmailTagsChange +}: VirtualApiKeyEmailSectionProps) { + const t = useTranslations(); + const [activeEmailTagIndex, setActiveEmailTagIndex] = useState< + number | null + >(null); + + useEffect(() => { + if (!hasAssociatedUser && sendToAttributedUser) { + onSendToAttributedUserChange(false); + } + }, [hasAssociatedUser, sendToAttributedUser, onSendToAttributedUserChange]); + + const checkboxId = + mode === "create" + ? "virtual-api-key-send-email" + : "edit-virtual-api-key-send-email"; + const sendToUserId = + mode === "create" + ? "virtual-api-key-send-to-user" + : "edit-virtual-api-key-send-to-user"; + + return ( +
+
+ { + if (emailEnabled) { + onSendEmailChange(val === true); + } + }} + className="mt-0.5" + /> +
+ +

+ {emailEnabled + ? t( + mode === "create" + ? "virtualApiKeysEmailOnGenerateDescription" + : "virtualApiKeysEmailThisKeyDescription" + ) + : t("virtualApiKeysEmailSmtpRequiredDescription")} +

+
+
+ + {emailEnabled && sendEmail && ( +
+
+ + onSendToAttributedUserChange(val === true) + } + className="mt-0.5" + /> +
+ +

+ {hasAssociatedUser + ? t( + "virtualApiKeysEmailSendToUserDescription" + ) + : t( + "virtualApiKeysEmailSendToUserDisabled" + )} +

+
+
+ +
+ + {t("virtualApiKeysEmailAdditional")} + + { + const next = + typeof newTags === "function" + ? newTags(emailTags) + : newTags; + onEmailTagsChange(next as Tag[]); + }} + allowDuplicates={false} + sortTags + validateTag={(tag) => + /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(tag) + } + delimiterList={[",", "Enter"]} + /> +
+
+ )} +
+ ); +} From 53b1d8a9f34b595508716ad533618150d085ba54 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 14 Aug 2026 17:37:17 -0400 Subject: [PATCH 201/296] option to send identity keys in email --- messages/en-US.json | 23 ++ .../templates/IdentityApiKeyGenerated.tsx | 78 ++++++ .../templates/VirtualApiKeyGenerated.tsx | 5 +- server/emails/templates/components/Email.tsx | 2 +- server/lib/sendVirtualApiKeyEmail.ts | 53 +++- server/routers/external.ts | 9 + server/routers/integration.ts | 9 + .../virtualApiKey/emailIdentityKeys.ts | 264 ++++++++++++++++++ server/routers/virtualApiKey/index.ts | 1 + server/routers/virtualApiKey/types.ts | 5 + .../virtual-api-keys/(list)/identity/page.tsx | 16 ++ .../virtual-api-keys/(list)/keys/page.tsx | 142 ++++++++++ .../virtual-api-keys/(list)/layout.tsx | 37 +++ .../settings/virtual-api-keys/page.tsx | 149 +--------- src/components/EmailIdentityKeysForm.tsx | 192 +++++++++++++ src/components/IdentityKeysSplash.tsx | 117 ++++++++ src/components/VirtualApiKeysBanner.tsx | 45 --- 17 files changed, 942 insertions(+), 205 deletions(-) create mode 100644 server/emails/templates/IdentityApiKeyGenerated.tsx create mode 100644 server/routers/virtualApiKey/emailIdentityKeys.ts create mode 100644 src/app/[orgId]/settings/virtual-api-keys/(list)/identity/page.tsx create mode 100644 src/app/[orgId]/settings/virtual-api-keys/(list)/keys/page.tsx create mode 100644 src/app/[orgId]/settings/virtual-api-keys/(list)/layout.tsx create mode 100644 src/components/EmailIdentityKeysForm.tsx create mode 100644 src/components/IdentityKeysSplash.tsx delete mode 100644 src/components/VirtualApiKeysBanner.tsx diff --git a/messages/en-US.json b/messages/en-US.json index f918a51c3..ea227ebe2 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1678,6 +1678,29 @@ "commandVirtualApiKeys": "Virtual API Keys", "virtualApiKeysTitle": "Manage Virtual API Keys", "virtualApiKeysDescription": "Create and manage manual API keys for AI Gateway access to public AI gateways", + "virtualApiKeysTabIdentity": "Identity Keys", + "virtualApiKeysTabVirtual": "Virtual Keys", + "virtualApiKeysIdentitySplashTitle": "Identity Keys for Every User", + "virtualApiKeysIdentitySplashDescription": "Every user already has a Pangolin identity key for this organization. It is unique to their account and authenticates them to AI gateways they can access.", + "virtualApiKeysIdentitySplashExample": "Example", + "virtualApiKeysIdentitySplashRetrieveTitle": "How Users Get Their Key", + "virtualApiKeysIdentitySplashRetrieveResource": "Visit a public AI gateway URL in the browser and log in with their account.", + "virtualApiKeysIdentitySplashRetrievePage": "Or go to while signed in.", + "virtualApiKeysIdentitySplashManual": "You can create additional virtual API keys on the Virtual Keys tab. Those keys can be scoped to specific public AI gateways and optionally associated with a user. Creating a key immediately grants access to the selected public AI gateways.", + "virtualApiKeysIdentitySplashGoToVirtual": "Manually Create a Key", + "virtualApiKeysEmailIdentity": "Email Identity Keys", + "virtualApiKeysEmailIdentityDescription": "Send each selected user or role their Pangolin identity key.", + "virtualApiKeysEmailIdentitySendAll": "Send to all users", + "virtualApiKeysEmailIdentitySendAllDescription": "Email every organization member who has an account email.", + "virtualApiKeysEmailIdentitySelectUsers": "Users", + "virtualApiKeysEmailIdentitySelectRoles": "Roles", + "virtualApiKeysEmailIdentitySubmit": "Send Emails", + "virtualApiKeysEmailIdentitySuccess": "Identity keys emailed", + "virtualApiKeysEmailIdentitySuccessDescription": "Sent {sent} emails.", + "virtualApiKeysEmailIdentitySkipped": "{skipped} users were skipped because they do not have an email address.", + "virtualApiKeysEmailIdentityRecipientsRequired": "Select at least one user or role, or send to all users.", + "virtualApiKeysEmailIdentityError": "Error sending identity keys", + "virtualApiKeysEmailIdentityErrorDescription": "Failed to email identity keys", "virtualApiKeysBannerTitle": "Identity Keys for Every User", "virtualApiKeysBannerDescription": "Every user already has an identity key available at {keysUrl}. You can also manually generate keys here that grant direct access to public AI gateways.", "virtualApiKeysBannerButtonText": "View Identity Keys", diff --git a/server/emails/templates/IdentityApiKeyGenerated.tsx b/server/emails/templates/IdentityApiKeyGenerated.tsx new file mode 100644 index 000000000..be49bb8ff --- /dev/null +++ b/server/emails/templates/IdentityApiKeyGenerated.tsx @@ -0,0 +1,78 @@ +import React from "react"; +import { Body, Head, Html, Preview, Tailwind } from "@react-email/components"; +import { themeColors } from "./lib/theme"; +import { + EmailContainer, + EmailFooter, + EmailGreeting, + EmailHeading, + EmailInfoSection, + EmailLetterHead, + EmailSection, + EmailSignature, + EmailText +} from "./components/Email"; + +type IdentityApiKeyGeneratedProps = { + orgName: string; + accountLabel?: string | null; + credential: string; + resourceUrls: string[]; + hasMoreResources: boolean; +}; + +export const IdentityApiKeyGenerated = ({ + orgName, + accountLabel, + credential, + resourceUrls, + hasMoreResources +}: IdentityApiKeyGeneratedProps) => { + const previewText = `Your personal identity key for ${orgName}`; + + return ( + + + {previewText} + + + + + + Hi there, + + + This is your personal identity key for{" "} + {orgName}. It belongs to your + account and identifies you when you use public AI + gateways. + + + + Use it with resources your administrator has granted + you, or that your role has access to. Treat this key + like a password and do not share it. + + + + Your identity key: +
+
+ + {credential} + +
+
+ + + + + + + + + + ); +}; + +export default IdentityApiKeyGenerated; diff --git a/server/emails/templates/VirtualApiKeyGenerated.tsx b/server/emails/templates/VirtualApiKeyGenerated.tsx index f156c1b6e..df7b700a8 100644 --- a/server/emails/templates/VirtualApiKeyGenerated.tsx +++ b/server/emails/templates/VirtualApiKeyGenerated.tsx @@ -42,8 +42,9 @@ export const VirtualApiKeyGenerated = ({ A virtual API key for {orgName} has - been shared with you. Treat this key like a password - and do not share it. + been shared with you. This key grants access to the + public AI gateways it was created for. Treat this + key like a password and do not share it. diff --git a/server/emails/templates/components/Email.tsx b/server/emails/templates/components/Email.tsx index f74046042..61e1b7116 100644 --- a/server/emails/templates/components/Email.tsx +++ b/server/emails/templates/components/Email.tsx @@ -18,7 +18,7 @@ export function EmailLetterHead() { Pangolin Logo diff --git a/server/lib/sendVirtualApiKeyEmail.ts b/server/lib/sendVirtualApiKeyEmail.ts index 48d7d0d6b..4eecd180f 100644 --- a/server/lib/sendVirtualApiKeyEmail.ts +++ b/server/lib/sendVirtualApiKeyEmail.ts @@ -2,6 +2,7 @@ import { db, resources, users, virtualApiKeyResources } from "@server/db"; import { and, asc, eq } from "drizzle-orm"; import config from "@server/lib/config"; import { sendEmail } from "@server/emails"; +import IdentityApiKeyGenerated from "@server/emails/templates/IdentityApiKeyGenerated"; import VirtualApiKeyGenerated from "@server/emails/templates/VirtualApiKeyGenerated"; import { formatVirtualApiKeyCredential } from "@server/lib/virtualApiKey"; @@ -60,6 +61,17 @@ async function listVirtualApiKeyGatewayUrls(params: { }; } +export async function listOrgInferenceGatewayUrls(orgId: string): Promise<{ + urls: string[]; + hasMore: boolean; +}> { + return listVirtualApiKeyGatewayUrls({ + orgId, + allResources: true, + virtualApiKeyId: "" + }); +} + export async function resolveVirtualApiKeyEmailRecipients(params: { sendEmail: boolean; sendToAttributedUser: boolean; @@ -125,6 +137,9 @@ export async function sendVirtualApiKeyEmails(params: { virtualApiKeyId: string; secret: string; allResources: boolean; + isIdentityKey?: boolean; + accountLabel?: string | null; + gatewayUrls?: { urls: string[]; hasMore: boolean }; }): Promise { if (params.recipients.length === 0) { return; @@ -134,23 +149,35 @@ export async function sendVirtualApiKeyEmails(params: { params.virtualApiKeyId, params.secret ); - const { urls, hasMore } = await listVirtualApiKeyGatewayUrls({ - orgId: params.orgId, - allResources: params.allResources, - virtualApiKeyId: params.virtualApiKeyId - }); + const { urls, hasMore } = + params.gatewayUrls ?? + (await listVirtualApiKeyGatewayUrls({ + orgId: params.orgId, + allResources: params.allResources, + virtualApiKeyId: params.virtualApiKeyId + })); const from = config.getNoReplyEmail(); - const subject = `Virtual API key for ${params.orgName}`; + const subject = params.isIdentityKey + ? `Your identity key for ${params.orgName}` + : `Virtual API key for ${params.orgName}`; for (const to of params.recipients) { await sendEmail( - VirtualApiKeyGenerated({ - orgName: params.orgName, - keyName: params.keyName, - credential, - resourceUrls: urls, - hasMoreResources: hasMore - }), + params.isIdentityKey + ? IdentityApiKeyGenerated({ + orgName: params.orgName, + accountLabel: params.accountLabel, + credential, + resourceUrls: urls, + hasMoreResources: hasMore + }) + : VirtualApiKeyGenerated({ + orgName: params.orgName, + keyName: params.keyName, + credential, + resourceUrls: urls, + hasMoreResources: hasMore + }), { to, from, diff --git a/server/routers/external.ts b/server/routers/external.ts index e182d2b44..b51fbf2de 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -1735,6 +1735,15 @@ authenticated.get( virtualApiKey.listVirtualApiKeys ); +authenticated.post( + "/org/:orgId/virtual-api-keys/email-identity-keys", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.getVirtualApiKey), + virtualApiKey.emailIdentityKeysRateLimit, + logActionAudit(ActionsEnum.getVirtualApiKey), + virtualApiKey.emailIdentityKeys +); + authenticated.get( "/org/:orgId/my-virtual-api-keys", verifyOrgAccess, diff --git a/server/routers/integration.ts b/server/routers/integration.ts index f609443d4..214c1ff27 100644 --- a/server/routers/integration.ts +++ b/server/routers/integration.ts @@ -1769,6 +1769,15 @@ authenticated.get( virtualApiKey.listVirtualApiKeys ); +authenticated.post( + "/org/:orgId/virtual-api-keys/email-identity-keys", + verifyApiKeyOrgAccess, + verifyApiKeyHasAction(ActionsEnum.getVirtualApiKey), + virtualApiKey.emailIdentityKeysRateLimit, + logActionAudit(ActionsEnum.getVirtualApiKey), + virtualApiKey.emailIdentityKeys +); + authenticated.get( "/virtual-api-key/:virtualApiKeyId", verifyApiKeyVirtualApiKeyAccess, diff --git a/server/routers/virtualApiKey/emailIdentityKeys.ts b/server/routers/virtualApiKey/emailIdentityKeys.ts new file mode 100644 index 000000000..4c4963751 --- /dev/null +++ b/server/routers/virtualApiKey/emailIdentityKeys.ts @@ -0,0 +1,264 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db, orgs, roles, userOrgRoles, userOrgs, users } from "@server/db"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { OpenAPITags, registry } from "@server/openApi"; +import { and, eq, inArray } from "drizzle-orm"; +import config from "@server/lib/config"; +import { getOrCreateUserVirtualApiKey } from "@server/lib/virtualApiKey"; +import { + sendVirtualApiKeyEmails, + listOrgInferenceGatewayUrls +} from "@server/lib/sendVirtualApiKeyEmail"; +import type { EmailIdentityKeysResponse } from "@server/routers/virtualApiKey/types"; +import rateLimit, { ipKeyGenerator } from "express-rate-limit"; +import { createStore } from "#dynamic/lib/rateLimitStore"; + +const EMAIL_IDENTITY_KEYS_WINDOW_MINUTES = 15; +const EMAIL_IDENTITY_KEYS_MAX = 3; + +export const emailIdentityKeysRateLimit = rateLimit({ + windowMs: EMAIL_IDENTITY_KEYS_WINDOW_MINUTES * 60 * 1000, + max: EMAIL_IDENTITY_KEYS_MAX, + keyGenerator: (req) => { + const actor = + req.user?.userId || + req.apiKey?.apiKeyId || + ipKeyGenerator(req.ip || ""); + const orgId = + typeof req.params.orgId === "string" ? req.params.orgId : ""; + return `emailIdentityKeys:${actor}:${orgId}`; + }, + handler: (_req, _res, next) => { + const message = `You can only email identity keys ${EMAIL_IDENTITY_KEYS_MAX} times every ${EMAIL_IDENTITY_KEYS_WINDOW_MINUTES} minutes. Please try again later.`; + return next(createHttpError(HttpCode.TOO_MANY_REQUESTS, message)); + }, + store: createStore() +}); + +const paramsSchema = z.strictObject({ + orgId: z.string().nonempty() +}); + +const bodySchema = z + .strictObject({ + sendToAll: z.boolean().optional().default(false), + userIds: z.array(z.string().nonempty()).optional().default([]), + roleIds: z.array(z.number().int().positive()).optional().default([]) + }) + .superRefine((data, ctx) => { + if ( + !data.sendToAll && + data.userIds.length === 0 && + data.roleIds.length === 0 + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "Select at least one user or role, or send to all users", + path: ["userIds"] + }); + } + }); + +registry.registerPath({ + method: "post", + path: "/org/{orgId}/virtual-api-keys/email-identity-keys", + description: + "Email identity virtual API keys to selected organization members and roles, or to all members.", + tags: [OpenAPITags.VirtualApiKey], + request: { + params: paramsSchema, + body: { + content: { + "application/json": { + schema: bodySchema + } + } + } + }, + responses: { + 200: { + description: "Successful response" + } + } +}); + +export async function emailIdentityKeys( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = paramsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const parsedBody = bodySchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + if (!config.getRawConfig().email) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Email is not configured on this server" + ) + ); + } + + const { orgId } = parsedParams.data; + const { sendToAll, userIds, roleIds } = parsedBody.data; + + if (req.user && orgId && orgId !== req.userOrgId) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + const uniqueUserIds = [...new Set(userIds)]; + const uniqueRoleIds = [...new Set(roleIds)]; + + if (!sendToAll && uniqueRoleIds.length > 0) { + const orgRoles = await db + .select({ roleId: roles.roleId }) + .from(roles) + .where( + and( + eq(roles.orgId, orgId), + inArray(roles.roleId, uniqueRoleIds) + ) + ); + + if (orgRoles.length !== uniqueRoleIds.length) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "One or more roles are invalid for this organization" + ) + ); + } + } + + let targetUserIds: string[] | null = null; + if (!sendToAll) { + let roleUserIds: string[] = []; + if (uniqueRoleIds.length > 0) { + const roleMembers = await db + .select({ userId: userOrgRoles.userId }) + .from(userOrgRoles) + .where( + and( + eq(userOrgRoles.orgId, orgId), + inArray(userOrgRoles.roleId, uniqueRoleIds) + ) + ); + roleUserIds = roleMembers.map((row) => row.userId); + } + + targetUserIds = [...new Set([...uniqueUserIds, ...roleUserIds])]; + if (targetUserIds.length === 0) { + return response(res, { + data: { sent: 0, skipped: 0 }, + success: true, + error: false, + message: "Identity keys emailed successfully", + status: HttpCode.OK + }); + } + } + + const memberConditions = [eq(userOrgs.orgId, orgId)]; + if (targetUserIds) { + memberConditions.push(inArray(users.userId, targetUserIds)); + } + + const members = await db + .select({ user: users }) + .from(users) + .innerJoin(userOrgs, eq(userOrgs.userId, users.userId)) + .where(and(...memberConditions)); + + if (!sendToAll && uniqueUserIds.length > 0) { + const foundIds = new Set(members.map((row) => row.user.userId)); + if (uniqueUserIds.some((id) => !foundIds.has(id))) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "One or more users are not members of this organization" + ) + ); + } + } + + const [org] = await db + .select() + .from(orgs) + .where(eq(orgs.orgId, orgId)) + .limit(1); + + const orgName = org?.name || orgId; + const gatewayUrls = await listOrgInferenceGatewayUrls(orgId); + let sent = 0; + let skipped = 0; + + for (const { user } of members) { + if (!user.email) { + skipped += 1; + continue; + } + + const { key, secret } = await getOrCreateUserVirtualApiKey({ + orgId, + user, + createdByUserId: req.user?.userId ?? null + }); + + await sendVirtualApiKeyEmails({ + recipients: [user.email], + orgName, + orgId, + keyName: key.name, + virtualApiKeyId: key.virtualApiKeyId, + secret, + allResources: true, + isIdentityKey: true, + accountLabel: user.email || user.name || user.username, + gatewayUrls + }); + sent += 1; + } + + return response(res, { + data: { sent, skipped }, + success: true, + error: false, + message: "Identity keys emailed successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/virtualApiKey/index.ts b/server/routers/virtualApiKey/index.ts index 2cb06a5ca..0a6c16a06 100644 --- a/server/routers/virtualApiKey/index.ts +++ b/server/routers/virtualApiKey/index.ts @@ -5,4 +5,5 @@ export * from "./getVirtualApiKey"; export * from "./getMyVirtualApiKey"; export * from "./updateVirtualApiKey"; export * from "./deleteVirtualApiKey"; +export * from "./emailIdentityKeys"; export * from "./types"; diff --git a/server/routers/virtualApiKey/types.ts b/server/routers/virtualApiKey/types.ts index 91bf17c0a..e6a387446 100644 --- a/server/routers/virtualApiKey/types.ts +++ b/server/routers/virtualApiKey/types.ts @@ -28,3 +28,8 @@ export type ListMyVirtualApiKeysResponse = { export type GetMyVirtualApiKeyResponse = { virtualApiKey: VirtualApiKeyWithResources; }; + +export type EmailIdentityKeysResponse = { + sent: number; + skipped: number; +}; diff --git a/src/app/[orgId]/settings/virtual-api-keys/(list)/identity/page.tsx b/src/app/[orgId]/settings/virtual-api-keys/(list)/identity/page.tsx new file mode 100644 index 000000000..4709345ed --- /dev/null +++ b/src/app/[orgId]/settings/virtual-api-keys/(list)/identity/page.tsx @@ -0,0 +1,16 @@ +import type { Metadata } from "next"; +import IdentityKeysSplash from "@app/components/IdentityKeysSplash"; + +export const metadata: Metadata = { + title: "Identity Keys" +}; + +type IdentityKeysPageProps = { + params: Promise<{ orgId: string }>; +}; + +export default async function IdentityKeysPage(props: IdentityKeysPageProps) { + const params = await props.params; + + return ; +} diff --git a/src/app/[orgId]/settings/virtual-api-keys/(list)/keys/page.tsx b/src/app/[orgId]/settings/virtual-api-keys/(list)/keys/page.tsx new file mode 100644 index 000000000..2a5cb0e57 --- /dev/null +++ b/src/app/[orgId]/settings/virtual-api-keys/(list)/keys/page.tsx @@ -0,0 +1,142 @@ +import { internal } from "@app/lib/api"; +import { authCookieHeader } from "@app/lib/api/cookies"; +import { AxiosResponse } from "axios"; +import { redirect } from "next/navigation"; +import { cache } from "react"; +import { GetOrgResponse } from "@server/routers/org"; +import OrgProvider from "@app/providers/OrgProvider"; +import VirtualApiKeysTable, { + type VirtualApiKeyRow +} from "@app/components/VirtualApiKeysTable"; +import { getTranslations } from "next-intl/server"; +import type { Metadata } from "next"; +import type { ListVirtualApiKeysResponse } from "@server/routers/virtualApiKey/types"; +import type { ListUsersResponse } from "@server/routers/user"; +import type { ListResourcesResponse } from "@server/routers/resource"; + +export const metadata: Metadata = { + title: "Virtual Keys" +}; + +type VirtualApiKeysTablePageProps = { + params: Promise<{ orgId: string }>; +}; + +export const dynamic = "force-dynamic"; + +export default async function VirtualApiKeysTablePage( + props: VirtualApiKeysTablePageProps +) { + const params = await props.params; + const cookieHeader = await authCookieHeader(); + const t = await getTranslations(); + + let keys: ListVirtualApiKeysResponse["virtualApiKeys"] = []; + let users: { + userId: string; + email: string | null; + name: string | null; + username: string | null; + }[] = []; + let resources: { + resourceId: number; + name: string; + niceId: string; + }[] = []; + + try { + const [keysRes, usersRes, resourcesRes] = await Promise.all([ + internal.get>( + `/org/${params.orgId}/virtual-api-keys?page=1&pageSize=1000`, + cookieHeader + ), + internal.get>( + `/org/${params.orgId}/users?page=1&pageSize=1000`, + cookieHeader + ), + internal.get>( + `/org/${params.orgId}/resources?page=1&pageSize=1000`, + cookieHeader + ) + ]); + + keys = keysRes.data.data.virtualApiKeys ?? []; + users = (usersRes.data.data.users ?? []).map((u) => ({ + userId: u.id, + email: u.email ?? null, + name: u.name ?? null, + username: u.username ?? null + })); + resources = (resourcesRes.data.data.resources ?? []).map((r) => ({ + resourceId: r.resourceId, + name: r.name, + niceId: r.niceId + })); + } catch { + // leave empty; page still renders + } + + let org = null; + try { + const getOrg = cache(async () => + internal.get>( + `/org/${params.orgId}`, + cookieHeader + ) + ); + const res = await getOrg(); + org = res.data.data; + } catch { + redirect(`/${params.orgId}/settings/resources`); + } + + if (!org) { + redirect(`/${params.orgId}/settings/resources`); + } + + const userById = new Map(users.map((u) => [u.userId, u])); + const resourceById = new Map(resources.map((r) => [r.resourceId, r])); + + const rows: VirtualApiKeyRow[] = keys.map((key) => { + const user = key.userId ? userById.get(key.userId) : undefined; + const keyResources = key.resourceIds + .map((id) => resourceById.get(id)) + .filter(Boolean) as { + resourceId: number; + name: string; + niceId: string; + }[]; + + const resourceNames = key.allResources + ? t("virtualApiKeysAllResources") + : keyResources.map((r) => r.name).join(", ") || + t("virtualApiKeysNoResources"); + + return { + virtualApiKeyId: key.virtualApiKeyId, + orgId: key.orgId, + kind: key.kind, + userId: key.userId, + name: key.name, + description: key.description, + lastChars: key.lastChars, + allResources: key.allResources, + expiresAt: key.expiresAt, + lastUsedAt: key.lastUsedAt, + createdAt: key.createdAt, + createdByUserId: key.createdByUserId, + resourceIds: key.resourceIds, + userName: user?.name ?? null, + username: user?.username ?? null, + userEmail: user?.email ?? null, + resourceNames, + resources: keyResources + }; + }); + + return ( + + + + ); +} diff --git a/src/app/[orgId]/settings/virtual-api-keys/(list)/layout.tsx b/src/app/[orgId]/settings/virtual-api-keys/(list)/layout.tsx new file mode 100644 index 000000000..20d3af73f --- /dev/null +++ b/src/app/[orgId]/settings/virtual-api-keys/(list)/layout.tsx @@ -0,0 +1,37 @@ +import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; +import { HorizontalTabs } from "@app/components/HorizontalTabs"; +import { getTranslations } from "next-intl/server"; + +type VirtualApiKeysListLayoutProps = { + children: React.ReactNode; + params: Promise<{ orgId: string }>; +}; + +export default async function VirtualApiKeysListLayout({ + children, + params +}: VirtualApiKeysListLayoutProps) { + const { orgId } = await params; + const t = await getTranslations(); + + const navItems = [ + { + title: t("virtualApiKeysTabIdentity"), + href: `/${orgId}/settings/virtual-api-keys/identity` + }, + { + title: t("virtualApiKeysTabVirtual"), + href: `/${orgId}/settings/virtual-api-keys/keys` + } + ]; + + return ( + <> + + {children} + + ); +} diff --git a/src/app/[orgId]/settings/virtual-api-keys/page.tsx b/src/app/[orgId]/settings/virtual-api-keys/page.tsx index 28a73253e..d841a6e34 100644 --- a/src/app/[orgId]/settings/virtual-api-keys/page.tsx +++ b/src/app/[orgId]/settings/virtual-api-keys/page.tsx @@ -1,156 +1,17 @@ -import { internal } from "@app/lib/api"; -import { authCookieHeader } from "@app/lib/api/cookies"; -import { AxiosResponse } from "axios"; -import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; -import { redirect } from "next/navigation"; -import { cache } from "react"; -import { GetOrgResponse } from "@server/routers/org"; -import OrgProvider from "@app/providers/OrgProvider"; -import VirtualApiKeysBanner from "@app/components/VirtualApiKeysBanner"; -import VirtualApiKeysTable, { - type VirtualApiKeyRow -} from "@app/components/VirtualApiKeysTable"; -import { getTranslations } from "next-intl/server"; import type { Metadata } from "next"; -import type { ListVirtualApiKeysResponse } from "@server/routers/virtualApiKey/types"; -import type { ListUsersResponse } from "@server/routers/user"; -import type { ListResourcesResponse } from "@server/routers/resource"; +import { redirect } from "next/navigation"; export const metadata: Metadata = { title: "Virtual API Keys" }; -type VirtualApiKeysPageProps = { +type VirtualApiKeysIndexPageProps = { params: Promise<{ orgId: string }>; }; -export const dynamic = "force-dynamic"; - -export default async function VirtualApiKeysPage( - props: VirtualApiKeysPageProps +export default async function VirtualApiKeysIndexPage( + props: VirtualApiKeysIndexPageProps ) { const params = await props.params; - const cookieHeader = await authCookieHeader(); - const t = await getTranslations(); - - let keys: ListVirtualApiKeysResponse["virtualApiKeys"] = []; - let users: { - userId: string; - email: string | null; - name: string | null; - username: string | null; - }[] = []; - let resources: { - resourceId: number; - name: string; - niceId: string; - }[] = []; - - try { - const [keysRes, usersRes, resourcesRes] = await Promise.all([ - internal.get>( - `/org/${params.orgId}/virtual-api-keys?page=1&pageSize=1000`, - cookieHeader - ), - internal.get>( - `/org/${params.orgId}/users?page=1&pageSize=1000`, - cookieHeader - ), - internal.get>( - `/org/${params.orgId}/resources?page=1&pageSize=1000`, - cookieHeader - ) - ]); - - keys = keysRes.data.data.virtualApiKeys ?? []; - users = (usersRes.data.data.users ?? []).map((u) => ({ - userId: u.id, - email: u.email ?? null, - name: u.name ?? null, - username: u.username ?? null - })); - resources = (resourcesRes.data.data.resources ?? []).map((r) => ({ - resourceId: r.resourceId, - name: r.name, - niceId: r.niceId - })); - } catch { - // leave empty; page still renders - } - - let org = null; - try { - const getOrg = cache(async () => - internal.get>( - `/org/${params.orgId}`, - cookieHeader - ) - ); - const res = await getOrg(); - org = res.data.data; - } catch { - redirect(`/${params.orgId}/settings/resources`); - } - - if (!org) { - redirect(`/${params.orgId}/settings/resources`); - } - - const userById = new Map(users.map((u) => [u.userId, u])); - const resourceById = new Map(resources.map((r) => [r.resourceId, r])); - - const rows: VirtualApiKeyRow[] = keys.map((key) => { - const user = key.userId ? userById.get(key.userId) : undefined; - const keyResources = key.resourceIds - .map((id) => resourceById.get(id)) - .filter(Boolean) as { - resourceId: number; - name: string; - niceId: string; - }[]; - - const resourceNames = key.allResources - ? t("virtualApiKeysAllResources") - : keyResources.map((r) => r.name).join(", ") || - t("virtualApiKeysNoResources"); - - return { - virtualApiKeyId: key.virtualApiKeyId, - orgId: key.orgId, - kind: key.kind, - userId: key.userId, - name: key.name, - description: key.description, - lastChars: key.lastChars, - allResources: key.allResources, - expiresAt: key.expiresAt, - lastUsedAt: key.lastUsedAt, - createdAt: key.createdAt, - createdByUserId: key.createdByUserId, - resourceIds: key.resourceIds, - userName: user?.name ?? null, - username: user?.username ?? null, - userEmail: user?.email ?? null, - resourceNames, - resources: keyResources - }; - }); - - return ( - <> - - - - - - - - - ); + redirect(`/${params.orgId}/settings/virtual-api-keys/identity`); } diff --git a/src/components/EmailIdentityKeysForm.tsx b/src/components/EmailIdentityKeysForm.tsx new file mode 100644 index 000000000..f0189bf0a --- /dev/null +++ b/src/components/EmailIdentityKeysForm.tsx @@ -0,0 +1,192 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { Checkbox } from "@app/components/ui/checkbox"; +import { + Credenza, + CredenzaBody, + CredenzaClose, + CredenzaContent, + CredenzaDescription, + CredenzaFooter, + CredenzaHeader, + CredenzaTitle +} from "@app/components/Credenza"; +import { Label } from "@app/components/ui/label"; +import { + RolesSelector, + type SelectedRole +} from "@app/components/roles-selector"; +import { + UsersSelector, + type SelectedUser +} from "@app/components/users-selector"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import type { EmailIdentityKeysResponse } from "@server/routers/virtualApiKey/types"; +import { AxiosResponse } from "axios"; +import { useState } from "react"; +import { useTranslations } from "next-intl"; + +type EmailIdentityKeysFormProps = { + orgId: string; + open: boolean; + setOpen: (open: boolean) => void; +}; + +export default function EmailIdentityKeysForm({ + orgId, + open, + setOpen +}: EmailIdentityKeysFormProps) { + const t = useTranslations(); + const api = createApiClient(useEnvContext()); + const [sendToAll, setSendToAll] = useState(false); + const [selectedUsers, setSelectedUsers] = useState([]); + const [selectedRoles, setSelectedRoles] = useState([]); + const [loading, setLoading] = useState(false); + + function resetState() { + setSendToAll(false); + setSelectedUsers([]); + setSelectedRoles([]); + setLoading(false); + } + + async function onSubmit() { + if ( + !sendToAll && + selectedUsers.length === 0 && + selectedRoles.length === 0 + ) { + toast({ + variant: "destructive", + title: t("virtualApiKeysEmailIdentityRecipientsRequired"), + description: t("virtualApiKeysEmailIdentityRecipientsRequired") + }); + return; + } + + setLoading(true); + try { + const res = await api.post< + AxiosResponse + >(`/org/${orgId}/virtual-api-keys/email-identity-keys`, { + sendToAll, + userIds: sendToAll ? [] : selectedUsers.map((user) => user.id), + roleIds: sendToAll + ? [] + : selectedRoles.map((role) => Number(role.id)) + }); + + const { sent, skipped } = res.data.data; + toast({ + title: t("virtualApiKeysEmailIdentitySuccess"), + description: + skipped > 0 + ? `${t("virtualApiKeysEmailIdentitySuccessDescription", { sent })} ${t("virtualApiKeysEmailIdentitySkipped", { skipped })}` + : t("virtualApiKeysEmailIdentitySuccessDescription", { + sent + }) + }); + setOpen(false); + resetState(); + } catch (e) { + toast({ + variant: "destructive", + title: t("virtualApiKeysEmailIdentityError"), + description: formatAxiosError( + e, + t("virtualApiKeysEmailIdentityErrorDescription") + ) + }); + } + setLoading(false); + } + + return ( + { + setOpen(val); + if (!val) { + resetState(); + } + }} + > + + + + {t("virtualApiKeysEmailIdentity")} + + + {t("virtualApiKeysEmailIdentityDescription")} + + + +
+
+ + setSendToAll(val === true) + } + className="mt-0.5" + /> +
+ +

+ {t( + "virtualApiKeysEmailIdentitySendAllDescription" + )} +

+
+
+
+ + +
+
+ + +
+
+ + + + + + + + + + ); +} diff --git a/src/components/IdentityKeysSplash.tsx b/src/components/IdentityKeysSplash.tsx new file mode 100644 index 000000000..b939dedab --- /dev/null +++ b/src/components/IdentityKeysSplash.tsx @@ -0,0 +1,117 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { + SettingsSection, + SettingsSectionBody, + SettingsSectionFooter +} from "@app/components/Settings"; +import EmailIdentityKeysForm from "@app/components/EmailIdentityKeysForm"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { formatVirtualApiKeyCredential } from "@app/lib/virtualApiKeyFormat"; +import { ArrowRight, ExternalLink, Globe, KeyRound, Mail } from "lucide-react"; +import { useTranslations } from "next-intl"; +import Link from "next/link"; +import { useState } from "react"; + +const EXAMPLE_IDENTITY_KEY = formatVirtualApiKeyCredential( + "k7m2n9qx", + "a8f3c1e0b5d24791" +); + +type IdentityKeysSplashProps = { + orgId: string; +}; + +export default function IdentityKeysSplash({ orgId }: IdentityKeysSplashProps) { + const t = useTranslations(); + const { env } = useEnvContext(); + const [emailOpen, setEmailOpen] = useState(false); + const emailEnabled = env.email.emailEnabled; + + const dashboardUrl = env.app.dashboardUrl?.replace(/\/$/, "") ?? ""; + const keysPath = `/${orgId}/keys`; + const keysUrl = dashboardUrl ? `${dashboardUrl}${keysPath}` : keysPath; + + return ( + <> + + +
+ +

+ {t("virtualApiKeysIdentitySplashTitle")} +

+

+ {t("virtualApiKeysIdentitySplashDescription")} +

+ +
+

+ {t("virtualApiKeysIdentitySplashRetrieveTitle")} +

+
    +
  • + + + {t( + "virtualApiKeysIdentitySplashRetrieveResource" + )} + +
    • +
  • + + + {t.rich( + "virtualApiKeysIdentitySplashRetrievePage", + { + url: () => ( + + {keysUrl} + + ) + } + )} + +
    • +
+
+ +

+ {t("virtualApiKeysIdentitySplashManual")} +

+ {!emailEnabled && ( +

+ {t( + "virtualApiKeysEmailSmtpRequiredDescription" + )} +

+ )} +
+ + + + + + + + + ); +} diff --git a/src/components/VirtualApiKeysBanner.tsx b/src/components/VirtualApiKeysBanner.tsx deleted file mode 100644 index dd2aa1186..000000000 --- a/src/components/VirtualApiKeysBanner.tsx +++ /dev/null @@ -1,45 +0,0 @@ -"use client"; - -import { Button } from "@app/components/ui/button"; -import { useEnvContext } from "@app/hooks/useEnvContext"; -import { ArrowRight, KeyRound } from "lucide-react"; -import { useTranslations } from "next-intl"; -import Link from "next/link"; -import DismissableBanner from "./DismissableBanner"; - -type VirtualApiKeysBannerProps = { - orgId: string; -}; - -export const VirtualApiKeysBanner = ({ orgId }: VirtualApiKeysBannerProps) => { - const t = useTranslations(); - const { env } = useEnvContext(); - - const dashboardUrl = env.app.dashboardUrl?.replace(/\/$/, "") ?? ""; - const keysUrl = dashboardUrl - ? `${dashboardUrl}/${orgId}/keys` - : `/${orgId}/keys`; - - return ( - } - description={t("virtualApiKeysBannerDescription", { keysUrl })} - > - - - - - ); -}; - -export default VirtualApiKeysBanner; From ca7ea72ff155e98c68091acc403f975d170a16c9 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 14 Aug 2026 17:53:53 -0400 Subject: [PATCH 202/296] restyle domain picker --- src/components/DomainPicker.tsx | 234 +++++++++++++++----------------- 1 file changed, 110 insertions(+), 124 deletions(-) diff --git a/src/components/DomainPicker.tsx b/src/components/DomainPicker.tsx index 957d759eb..2abb403cb 100644 --- a/src/components/DomainPicker.tsx +++ b/src/components/DomainPicker.tsx @@ -38,14 +38,11 @@ import { useQuery } from "@tanstack/react-query"; import { AxiosResponse } from "axios"; import { AlertCircle, - Building2, - Check, - CheckCircle2, + CheckIcon, ChevronsUpDown, ExternalLink, Globe, - KeyRound, - Zap + KeyRound } from "lucide-react"; import { useTranslations } from "next-intl"; import Link from "next/link"; @@ -609,61 +606,72 @@ export default function DomainPicker({ - - + + - -
+ + {t("domainPickerNoDomainsFound")} -
- - - {organizationDomains.length > 0 && ( - <> + + {organizationDomains.length > 0 && ( - - {organizationDomains.map( - (orgDomain) => ( + {organizationDomains.map( + (orgDomain) => { + const description = + orgDomain.type === + "wildcard" + ? t( + "domainPickerManual" + ) + : `${orgDomain.type.toUpperCase()} · ${ + orgDomain.verified + ? t( + "domainPickerVerified" + ) + : t( + "domainPickerUnverified" + ) + }`; + const optionId = `org-${orgDomain.domainId}`; + + return ( handleBaseDomainSelect( { - id: `org-${orgDomain.domainId}`, + id: optionId, domain: orgDomain.baseDomain, type: "organization", verified: @@ -675,80 +683,63 @@ export default function DomainPicker({ } ) } - className="mx-2 rounded-md" - disabled={ - !orgDomain.verified - } > -
- -
-
- - { - orgDomain.baseDomain - } - - - {orgDomain.type === - "wildcard" ? ( - t( - "domainPickerManual" - ) - ) : ( - <> - {orgDomain.type.toUpperCase()}{" "} - •{" "} - {orgDomain.verified - ? t( - "domainPickerVerified" - ) - : t( - "domainPickerUnverified" - )} - - )} - -
- +
+ + { + orgDomain.baseDomain + } + + + { + description + } + +
- ) - )} - - - {(build === "saas" || - build === "enterprise") && - !hideFreeDomain && ( - + ); + } )} - - )} - - {(build === "saas" || build === "enterprise") && - !hideFreeDomain && ( - - + + )} + {organizationDomains.length > 0 && + (build === "saas" || + build === "enterprise") && + !hideFreeDomain && } + {(build === "saas" || + build === "enterprise") && + !hideFreeDomain && ( + handleBaseDomainSelect({ id: "provided-search", @@ -764,14 +755,18 @@ export default function DomainPicker({ type: "provided-search" }) } - className="mx-2 rounded-md" - disabled={requiresPaywall} > -
- -
-
- + +
+ {build === "enterprise" ? t( @@ -781,25 +776,16 @@ export default function DomainPicker({ "domainPickerFreeProvidedDomain" )} - + {t( "domainPickerSearchForAvailableDomains" )}
- - - - )} + + )} + From b51aecf45a609f2c97977bf3a64a3dcd8908c972 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Aug 2026 16:33:05 -0400 Subject: [PATCH 203/296] Remove advanced resources paywall --- server/lib/billing/tierMatrix.ts | 10 +- server/lib/blueprints/privateResources.ts | 24 -- server/lib/blueprints/publicResources.ts | 16 +- .../routers/billing/featureLifecycle.ts | 17 +- server/private/routers/external.ts | 28 +- server/private/routers/internal.ts | 7 - server/private/routers/ssh/index.ts | 14 - server/routers/external.ts | 26 +- server/routers/internal.ts | 15 +- server/routers/resource/createResource.ts | 17 +- server/routers/role/createRole.ts | 2 +- server/routers/role/updateRole.ts | 2 +- .../siteResource/createSiteResource.ts | 48 +-- .../siteResource/updateSiteResource.ts | 29 +- server/routers/ssh/index.ts | 1 + .../{private => }/routers/ssh/signSshKey.ts | 28 -- .../resources/private/[niceId]/http/page.tsx | 7 +- .../resources/private/[niceId]/ssh/page.tsx | 116 +++--- .../resources/private/create/page.tsx | 177 +++------ .../public/[niceId]/maintenance/page.tsx | 4 +- .../resources/public/[niceId]/rdp/page.tsx | 9 +- .../resources/public/[niceId]/ssh/page.tsx | 9 +- .../resources/public/[niceId]/vnc/page.tsx | 9 +- .../settings/resources/public/create/page.tsx | 62 +-- src/components/CreateRoleForm.tsx | 2 +- src/components/EditRoleForm.tsx | 7 +- src/components/PrivateResourceHttpFields.tsx | 13 +- src/components/PrivateResourceSshFields.tsx | 9 - src/components/RoleForm.tsx | 368 +++++++++--------- src/lib/setClientCookie.ts | 7 - 30 files changed, 368 insertions(+), 715 deletions(-) delete mode 100644 server/private/routers/ssh/index.ts create mode 100644 server/routers/ssh/index.ts rename server/{private => }/routers/ssh/signSshKey.ts (97%) diff --git a/server/lib/billing/tierMatrix.ts b/server/lib/billing/tierMatrix.ts index 7c0b591ca..7e49121dc 100644 --- a/server/lib/billing/tierMatrix.ts +++ b/server/lib/billing/tierMatrix.ts @@ -10,7 +10,7 @@ export enum TierFeature { ActionLogs = "actionLogs", // set the retention period to none on downgrade ConnectionLogs = "connectionLogs", RotateCredentials = "rotateCredentials", - MaintencePage = "maintencePage", // handle downgrade + MaintenancePage = "maintenancePage", // handle downgrade DevicePosture = "devicePosture", TwoFactorEnforcement = "twoFactorEnforcement", // handle downgrade by setting to optional SessionDurationPolicies = "sessionDurationPolicies", // handle downgrade by setting to default duration @@ -25,8 +25,7 @@ export enum TierFeature { WildcardSubdomain = "wildcardSubdomain", NewtAutoUpdate = "newtAutoUpdate", ResourcePolicies = "resourcePolicies", - AdvancedPublicResources = "advancedPublicResources", - AdvancedPrivateResources = "advancedPrivateResources" + RoleBasedSSHControls = "roleBasedSSHControls" } export const tierMatrix: Record = { @@ -39,7 +38,7 @@ export const tierMatrix: Record = { [TierFeature.ActionLogs]: ["tier2", "tier3", "enterprise"], [TierFeature.ConnectionLogs]: ["tier2", "tier3", "enterprise"], [TierFeature.RotateCredentials]: ["tier1", "tier2", "tier3", "enterprise"], - [TierFeature.MaintencePage]: ["tier1", "tier2", "tier3", "enterprise"], + [TierFeature.MaintenancePage]: ["tier1", "tier2", "tier3", "enterprise"], [TierFeature.DevicePosture]: ["tier2", "tier3", "enterprise"], [TierFeature.TwoFactorEnforcement]: [ "tier1", @@ -69,6 +68,5 @@ export const tierMatrix: Record = { [TierFeature.WildcardSubdomain]: ["tier1", "tier2", "tier3", "enterprise"], [TierFeature.NewtAutoUpdate]: ["tier1", "tier2", "tier3", "enterprise"], [TierFeature.ResourcePolicies]: ["tier3", "enterprise"], - [TierFeature.AdvancedPublicResources]: ["tier3", "enterprise"], - [TierFeature.AdvancedPrivateResources]: ["tier3", "enterprise"] + [TierFeature.RoleBasedSSHControls]: ["tier3", "enterprise"] }; diff --git a/server/lib/blueprints/privateResources.ts b/server/lib/blueprints/privateResources.ts index 9cec7b487..cb8f8e8b5 100644 --- a/server/lib/blueprints/privateResources.ts +++ b/server/lib/blueprints/privateResources.ts @@ -128,30 +128,6 @@ export async function updatePrivateResources( for (const [resourceNiceId, resourceData] of Object.entries( config["client-resources"] )) { - if (resourceData.mode === "http") { - const hasHttpFeature = await isLicensedOrSubscribed( - orgId, - tierMatrix.advancedPrivateResources - ); - if (!hasHttpFeature) { - throw new Error( - "HTTP private resources are not included in your current plan. Please upgrade." - ); - } - } - - if (resourceData.mode === "ssh") { - const hasSshFeature = await isLicensedOrSubscribed( - orgId, - tierMatrix.advancedPrivateResources - ); - if (!hasSshFeature) { - throw new Error( - "SSH private resources are not included in your current plan. Please upgrade." - ); - } - } - const [existingResource] = await trx .select() .from(siteResources) diff --git a/server/lib/blueprints/publicResources.ts b/server/lib/blueprints/publicResources.ts index 2f4d9773d..61985a06e 100644 --- a/server/lib/blueprints/publicResources.ts +++ b/server/lib/blueprints/publicResources.ts @@ -262,18 +262,6 @@ export async function updatePublicResources( headers = JSON.stringify(resourceData.headers); } - if (["ssh", "rdp", "vnc"].includes(resourceData.mode || "")) { - const isLicensed = await isLicensedOrSubscribed( - orgId, - tierMatrix.advancedPublicResources - ); - if (!isLicensed) { - throw new Error( - "Your current subscription does not support browser gateway resources. Please upgrade to access this feature." - ); - } - } - if (resourceData.policy) { const isLicensed = await isLicensedOrSubscribed( orgId, @@ -331,7 +319,7 @@ export async function updatePublicResources( const isLicensed = await isLicensedOrSubscribed( orgId, - tierMatrix.maintencePage + tierMatrix.maintenancePage ); if (!isLicensed) { resourceData.maintenance = undefined; @@ -1138,7 +1126,7 @@ export async function updatePublicResources( const isLicensed = await isLicensedOrSubscribed( orgId, - tierMatrix.maintencePage + tierMatrix.maintenancePage ); if (!isLicensed) { resourceData.maintenance = undefined; diff --git a/server/private/routers/billing/featureLifecycle.ts b/server/private/routers/billing/featureLifecycle.ts index 84a7b4f5a..b32d83f7e 100644 --- a/server/private/routers/billing/featureLifecycle.ts +++ b/server/private/routers/billing/featureLifecycle.ts @@ -295,8 +295,8 @@ async function disableFeature( await disableRotateCredentials(orgId); break; - case TierFeature.MaintencePage: - await disableMaintencePage(orgId); + case TierFeature.MaintenancePage: + await disablemaintenancePage(orgId); break; case TierFeature.DevicePosture: @@ -319,10 +319,6 @@ async function disableFeature( await disableAutoProvisioning(orgId); break; - case TierFeature.AdvancedPrivateResources: - await disableAdvancedPrivateResources(orgId); - break; - case TierFeature.FullRbac: await disableFullRbac(orgId); break; @@ -368,13 +364,6 @@ async function disableDeviceApprovals(orgId: string): Promise { logger.info(`Disabled device approvals on all roles for org ${orgId}`); } -async function disableAdvancedPrivateResources(orgId: string): Promise { - // TODO: implement logic to disable advanced private resourcs like ssh and ssh pam - // logger.info( - // `Disabled advanced private resources on all roles and site resources for org ${orgId}` - // ); -} - async function disableFullRbac(orgId: string): Promise { logger.info(`Disabled full RBAC for org ${orgId}`); } @@ -506,7 +495,7 @@ async function disableConnectionLogs(orgId: string): Promise { async function disableRotateCredentials(orgId: string): Promise {} -async function disableMaintencePage(orgId: string): Promise { +async function disablemaintenancePage(orgId: string): Promise { await db .update(resources) .set({ diff --git a/server/private/routers/external.ts b/server/private/routers/external.ts index fab026418..32cb68945 100644 --- a/server/private/routers/external.ts +++ b/server/private/routers/external.ts @@ -20,19 +20,16 @@ import * as orgIdp from "#private/routers/orgIdp"; import * as domain from "#private/routers/domain"; import * as auth from "#private/routers/auth"; import * as license from "#private/routers/license"; -import * as generateLicense from "./generatedLicense"; +import * as generateLicense from "#private/routers/generatedLicense"; import * as logs from "#private/routers/auditLogs"; import * as misc from "#private/routers/misc"; import * as reKey from "#private/routers/re-key"; import * as approval from "#private/routers/approvals"; -import * as ssh from "#private/routers/ssh"; import * as user from "#private/routers/user"; import * as siteProvisioning from "#private/routers/siteProvisioning"; import * as eventStreamingDestination from "#private/routers/eventStreamingDestination"; import * as alertRule from "#private/routers/alertRule"; import * as healthChecks from "#private/routers/healthChecks"; -import * as client from "@server/routers/client"; -import * as resource from "#private/routers/resource"; import * as policy from "#private/routers/policy"; import { @@ -652,17 +649,6 @@ authenticated.put( reKey.reGenerateExitNodeSecret ); -authenticated.post( - "/org/:orgId/ssh/sign-key", - verifyValidLicense, - verifyValidSubscription(tierMatrix.advancedPrivateResources), - verifyOrgAccess, - verifyLimits, - // verifyUserHasAction(ActionsEnum.signSshKey), // this check happens inside of the function now - // logActionAudit(ActionsEnum.signSshKey), // it is handled inside of the function below so we can include more metadata - ssh.signSshKey -); - authenticated.post( "/user/:userId/add-role/:roleId", verifyRoleAccess, @@ -868,18 +854,6 @@ authenticated.get( healthChecks.getBatchedHealthCheckStatusHistory ); -authenticated.get( - "/client/:clientId/verify-associations-cache", - verifyClientAccess, - client.verifyClientAssociationsCache -); - -authenticated.post( - "/client/:clientId/rebuild-associations-cache", - verifyClientAccess, - client.rebuildClientAssociationsCacheRoute -); - authenticated.post( "/org/:orgId/logs/access/attempt", verifyOrgAccess, diff --git a/server/private/routers/internal.ts b/server/private/routers/internal.ts index c45fe36b9..23188dcd7 100644 --- a/server/private/routers/internal.ts +++ b/server/private/routers/internal.ts @@ -17,7 +17,6 @@ import * as orgIdp from "#private/routers/orgIdp"; import * as billing from "#private/routers/billing"; import * as license from "#private/routers/license"; import * as resource from "#private/routers/resource"; -import * as ssh from "#private/routers/ssh"; import * as ws from "@server/routers/ws"; import * as browserTarget from "#private/routers/browserGatewayTarget"; @@ -47,12 +46,6 @@ internalRouter.get(`/license/status`, license.getLicenseStatus); internalRouter.get("/maintenance/info", resource.getMaintenanceInfo); -internalRouter.post( - "/org/:orgId/ssh/sign-key", - verifyUserFromResourceSessionMiddleware, - ssh.signSshKey -); - internalRouter.get( "/ws/round-trip-message/:messageId", verifyUserFromResourceSessionMiddleware, diff --git a/server/private/routers/ssh/index.ts b/server/private/routers/ssh/index.ts deleted file mode 100644 index d2f607f81..000000000 --- a/server/private/routers/ssh/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * This file is part of a proprietary work. - * - * Copyright (c) 2025-2026 Fossorial, Inc. - * All rights reserved. - * - * This file is licensed under the Fossorial Commercial License. - * You may not use this file except in compliance with the License. - * Unauthorized use, copying, modification, or distribution is strictly prohibited. - * - * This file is not licensed under the AGPLv3. - */ - -export * from "./signSshKey"; \ No newline at end of file diff --git a/server/routers/external.ts b/server/routers/external.ts index b51fbf2de..9c273fce5 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -20,6 +20,7 @@ import * as logs from "./auditLogs"; import * as launcher from "./launcher"; import * as newt from "./newt"; import * as olm from "./olm"; +import * as ssh from "./ssh"; import * as serverInfo from "./serverInfo"; import HttpCode from "@server/types/HttpCode"; import { @@ -56,12 +57,13 @@ import rateLimit, { ipKeyGenerator } from "express-rate-limit"; import createHttpError from "http-errors"; import { build } from "@server/build"; import { createStore } from "#dynamic/lib/rateLimitStore"; -import { logActionAudit } from "#dynamic/middlewares"; +import { logActionAudit, verifyValidLicense } from "#dynamic/middlewares"; import { checkRoundTripMessage } from "./ws"; import * as labels from "@server/routers/labels"; import * as aiProvider from "@server/routers/aiProvider"; import * as aiBudget from "@server/routers/aiBudget"; import * as virtualApiKey from "@server/routers/virtualApiKey"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; // Root routes export const unauthenticated = Router(); @@ -1863,6 +1865,28 @@ authenticated.put( labels.detachLabelFromItem ); +authenticated.post( + "/org/:orgId/ssh/sign-key", + verifyValidLicense, + verifyOrgAccess, + verifyLimits, + // verifyUserHasAction(ActionsEnum.signSshKey), // this check happens inside of the function now + // logActionAudit(ActionsEnum.signSshKey), // it is handled inside of the function below so we can include more metadata + ssh.signSshKey +); + +authenticated.get( + "/client/:clientId/verify-associations-cache", + verifyClientAccess, + client.verifyClientAssociationsCache +); + +authenticated.post( + "/client/:clientId/rebuild-associations-cache", + verifyClientAccess, + client.rebuildClientAssociationsCacheRoute +); + // Auth routes export const authRouter = Router(); unauthenticated.use("/auth", authRouter); diff --git a/server/routers/internal.ts b/server/routers/internal.ts index b06015a5f..27414c47b 100644 --- a/server/routers/internal.ts +++ b/server/routers/internal.ts @@ -1,15 +1,17 @@ import { Router } from "express"; import * as gerbil from "@server/routers/gerbil"; import * as traefik from "@server/routers/traefik"; -import * as resource from "./resource"; -import * as badger from "./badger"; +import * as resource from "@server/routers/resource"; +import * as badger from "@server/routers/badger"; import * as auth from "@server/routers/auth"; import * as supporterKey from "@server/routers/supporterKey"; import * as idp from "@server/routers/idp"; +import * as ssh from "@server/routers/ssh"; import HttpCode from "@server/types/HttpCode"; import { verifyResourceAccess, - verifySessionUserMiddleware + verifySessionUserMiddleware, + verifyUserFromResourceSessionMiddleware } from "@server/middlewares"; // Root routes @@ -42,6 +44,12 @@ internalRouter.get("/idp", idp.listIdps); internalRouter.get("/idp/:idpId", idp.getIdp); +internalRouter.post( + "/org/:orgId/ssh/sign-key", + verifyUserFromResourceSessionMiddleware, + ssh.signSshKey +); + // Gerbil routes const gerbilRouter = Router(); internalRouter.use("/gerbil", gerbilRouter); @@ -63,4 +71,3 @@ internalRouter.use("/badger", badgerRouter); badgerRouter.post("/verify-session", badger.verifyResourceSession); badgerRouter.post("/exchange-session", badger.exchangeSession); - diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index c0c7a7d5d..094e092b3 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -31,7 +31,7 @@ import { } from "@server/lib/domainUtils"; import { isSubscribed } from "#dynamic/lib/isSubscribed"; import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed"; -import { TierFeature, tierMatrix } from "@server/lib/billing/tierMatrix"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; import { getUniqueResourceName, getUniqueResourcePolicyName @@ -454,21 +454,6 @@ async function createHttpResource( } } - if ( - ["ssh", "rdp", "vnc"].includes(effectiveMode) && - !isLicensedOrSubscribed( - orgId!, - tierMatrix[TierFeature.AdvancedPublicResources] - ) - ) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "Your current subscription does not support browser gateway resources. Please upgrade to access this feature." - ) - ); - } - // Validate domain and construct full domain const domainResult = await validateAndConstructDomain( domainId, diff --git a/server/routers/role/createRole.ts b/server/routers/role/createRole.ts index 6d1ecb503..1cce7a5ae 100644 --- a/server/routers/role/createRole.ts +++ b/server/routers/role/createRole.ts @@ -135,7 +135,7 @@ export async function createRole( const isLicensedSshPam = await isLicensedOrSubscribed( orgId, - tierMatrix.advancedPrivateResources + tierMatrix.roleBasedSSHControls ); const roleInsertValues: Record = { name: roleData.name, diff --git a/server/routers/role/updateRole.ts b/server/routers/role/updateRole.ts index aa01899db..2c1dcc887 100644 --- a/server/routers/role/updateRole.ts +++ b/server/routers/role/updateRole.ts @@ -144,7 +144,7 @@ export async function updateRole( const isLicensedSshPam = await isLicensedOrSubscribed( orgId, - tierMatrix.advancedPrivateResources + tierMatrix.roleBasedSSHControls ); if (!isLicensedSshPam) { delete updateData.sshSudoMode; diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index f3794d35a..54a1bc004 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -10,8 +10,7 @@ import { SiteResource, siteResources, sites, - userSiteResources, - primaryDb + userSiteResources } from "@server/db"; import { getUniqueSiteResourceName } from "@server/db/names"; import { @@ -19,8 +18,6 @@ import { isIpInCidr, portRangeStringSchema } from "@server/lib/ip"; -import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed"; -import { TierFeature, tierMatrix } from "@server/lib/billing/tierMatrix"; import { rebuildClientAssociationsFromSiteResource, isOrgRebuildRateLimited @@ -408,21 +405,6 @@ export async function createSiteResource( } } - if (mode == "http") { - const hasHttpFeature = await isLicensedOrSubscribed( - orgId, - tierMatrix[TierFeature.AdvancedPrivateResources] - ); - if (!hasHttpFeature) { - return next( - createHttpError( - HttpCode.FORBIDDEN, - "HTTP private resources are not included in your current plan. Please upgrade." - ) - ); - } - } - // Verify the site exists and belongs to the org const sitesToAssign = await db .select() @@ -557,20 +539,6 @@ export async function createSiteResource( } } - const isLicensedSshPam = await isLicensedOrSubscribed( - orgId, - tierMatrix.advancedPrivateResources - ); - - if (mode == "ssh" && !isLicensedSshPam) { - return next( - createHttpError( - HttpCode.FORBIDDEN, - "SSH private resources are not included in your current plan. Please upgrade." - ) - ); - } - let updatedNiceId = niceId; if (!niceId) { updatedNiceId = await getUniqueSiteResourceName(orgId); @@ -646,13 +614,13 @@ export async function createSiteResource( fullDomain, requiresExitNodeConnection: mode === "inference" // in the future we might want to have different modes that do this }; - if (isLicensedSshPam) { - if (authDaemonPort !== undefined) - insertValues.authDaemonPort = authDaemonPort; - if (authDaemonMode !== undefined) - insertValues.authDaemonMode = authDaemonMode; - if (pamMode !== undefined) insertValues.pamMode = pamMode; - } + + if (authDaemonPort !== undefined) + insertValues.authDaemonPort = authDaemonPort; + if (authDaemonMode !== undefined) + insertValues.authDaemonMode = authDaemonMode; + if (pamMode !== undefined) insertValues.pamMode = pamMode; + [newSiteResource] = await trx .insert(siteResources) .values(insertValues) diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index 2e57b13a7..f0ee514a7 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -10,8 +10,6 @@ import { sites, userSiteResources } from "@server/db"; -import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed"; -import { TierFeature, tierMatrix } from "@server/lib/billing/tierMatrix"; import { validateAndConstructDomain } from "@server/lib/domainUtils"; import response from "@server/lib/response"; import { eq, and, ne, inArray } from "drizzle-orm"; @@ -362,26 +360,6 @@ export async function updateSiteResource( ); } - if (mode == "http") { - const hasHttpFeature = await isLicensedOrSubscribed( - existingSiteResource.orgId, - tierMatrix[TierFeature.AdvancedPrivateResources] - ); - if (!hasHttpFeature) { - return next( - createHttpError( - HttpCode.FORBIDDEN, - "HTTP private resources are not included in your current plan. Please upgrade." - ) - ); - } - } - - const isLicensedSshPam = await isLicensedOrSubscribed( - existingSiteResource.orgId, - tierMatrix.advancedPrivateResources - ); - const [org] = await db .select() .from(orgs) @@ -541,10 +519,9 @@ export async function updateSiteResource( await db.transaction(async (trx) => { // Update the site resource const sshPamSet = - isLicensedSshPam && - (authDaemonPort !== undefined || - authDaemonMode !== undefined || - pamMode !== undefined) + authDaemonPort !== undefined || + authDaemonMode !== undefined || + pamMode !== undefined ? { ...(authDaemonPort !== undefined && { authDaemonPort diff --git a/server/routers/ssh/index.ts b/server/routers/ssh/index.ts new file mode 100644 index 000000000..4cfcb6df3 --- /dev/null +++ b/server/routers/ssh/index.ts @@ -0,0 +1 @@ +export * from "./signSshKey"; \ No newline at end of file diff --git a/server/private/routers/ssh/signSshKey.ts b/server/routers/ssh/signSshKey.ts similarity index 97% rename from server/private/routers/ssh/signSshKey.ts rename to server/routers/ssh/signSshKey.ts index ae74a07a0..dfd73fd13 100644 --- a/server/private/routers/ssh/signSshKey.ts +++ b/server/routers/ssh/signSshKey.ts @@ -1,16 +1,3 @@ -/* - * This file is part of a proprietary work. - * - * Copyright (c) 2025-2026 Fossorial, Inc. - * All rights reserved. - * - * This file is licensed under the Fossorial Commercial License. - * You may not use this file except in compliance with the License. - * Unauthorized use, copying, modification, or distribution is strictly prohibited. - * - * This file is not licensed under the AGPLv3. - */ - import { Request, Response, NextFunction } from "express"; import { randomInt } from "crypto"; import { z } from "zod"; @@ -35,8 +22,6 @@ import { SiteResource } from "@server/db"; import { logAccessAudit } from "#private/lib/logAccessAudit"; -import { isLicensedOrSubscribed } from "#private/lib/isLicencedOrSubscribed"; -import { tierMatrix } from "@server/lib/billing/tierMatrix"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; @@ -163,19 +148,6 @@ export async function signSshKey( ); } - const isLicensed = await isLicensedOrSubscribed( - orgId, - tierMatrix.advancedPrivateResources - ); - if (!isLicensed) { - return next( - createHttpError( - HttpCode.FORBIDDEN, - "SSH key signing requires a paid plan" - ) - ); - } - // Get and decrypt the org's CA keys const caKeys = await getOrgCAKeys( orgId, diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/http/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/http/page.tsx index c1263c162..c764b0446 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/http/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/http/page.tsx @@ -35,10 +35,6 @@ import { buildSelectedSitesForResource } from "@app/lib/privateResourceUtils"; export default function PrivateResourceHttpPage() { const t = useTranslations(); const { save, siteResource } = useSaveSiteResource(); - const { isPaidUser } = usePaidStatus(); - const httpSectionDisabled = !isPaidUser( - tierMatrix.advancedPrivateResources - ); const [selectedSites, setSelectedSites] = useState(() => buildSelectedSitesForResource(siteResource) ); @@ -120,7 +116,7 @@ export default function PrivateResourceHttpPage() { )} orgId={siteResource.orgId} watch={asAnyWatch(form.watch)} - disabled={httpSectionDisabled} + disabled={false} siteResourceId={siteResource.id} /> @@ -135,7 +131,6 @@ export default function PrivateResourceHttpPage() { type="submit" form="private-resource-http-form" loading={saveLoading} - disabled={httpSectionDisabled} > {t("saveSettings")} diff --git a/src/app/[orgId]/settings/resources/private/[niceId]/ssh/page.tsx b/src/app/[orgId]/settings/resources/private/[niceId]/ssh/page.tsx index 9a739a073..ba60e41e6 100644 --- a/src/app/[orgId]/settings/resources/private/[niceId]/ssh/page.tsx +++ b/src/app/[orgId]/settings/resources/private/[niceId]/ssh/page.tsx @@ -12,16 +12,13 @@ import { SettingsFormGrid } from "@app/components/Settings"; import { SshServerSettingsFields } from "@app/components/SshServerSettingsFields"; -import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; import { Button } from "@app/components/ui/button"; import { Form } from "@app/components/ui/form"; -import { usePaidStatus } from "@app/hooks/usePaidStatus"; import { createSshFormSchema, inferSshPamMode } from "@app/lib/privateResourceForm"; import { zodResolver } from "@hookform/resolvers/zod"; -import { tierMatrix } from "@server/lib/billing/tierMatrix"; import { useTranslations } from "next-intl"; import { useActionState, useMemo, useState } from "react"; import { useForm } from "react-hook-form"; @@ -39,8 +36,6 @@ import { buildSelectedSitesForResource } from "@app/lib/privateResourceUtils"; export default function PrivateResourceSshPage() { const t = useTranslations(); const { save, siteResource } = useSaveSiteResource(); - const { isPaidUser } = usePaidStatus(); - const sshSectionDisabled = !isPaidUser(tierMatrix.advancedPrivateResources); const isNative = siteResource.authDaemonMode === "native"; const [sshServerMode] = useState<"standard" | "native">( isNative ? "native" : "standard" @@ -150,7 +145,6 @@ export default function PrivateResourceSshPage() { return ( - @@ -161,68 +155,56 @@ export default function PrivateResourceSshPage() { -
-
- - - - - form.setValue( - "authDaemonPort", - value, - { shouldValidate: true } - ) - } - authDaemonPortError={ - form.formState.errors.authDaemonPort - ?.message - } - sshServerMode={sshServerMode} - serverModeDisplay="badge" - /> - - - - + + + + + + form.setValue("authDaemonPort", value, { + shouldValidate: true + }) + } + authDaemonPortError={ + form.formState.errors.authDaemonPort + ?.message + } + sshServerMode={sshServerMode} + serverModeDisplay="badge" + /> + + + + - - - - -
- -
+ +
+ +
+ + ); diff --git a/src/app/[orgId]/settings/resources/private/create/page.tsx b/src/app/[orgId]/settings/resources/private/create/page.tsx index 96e8f84d8..717464283 100644 --- a/src/app/[orgId]/settings/resources/private/create/page.tsx +++ b/src/app/[orgId]/settings/resources/private/create/page.tsx @@ -16,7 +16,6 @@ import { type DescribedSelectOption } from "@app/components/DescribedSelect"; import DomainPicker from "@app/components/DomainPicker"; -import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; import { Button } from "@app/components/ui/button"; import { Form, @@ -30,7 +29,6 @@ import { import { Input } from "@app/components/ui/input"; import type { Selectedsite } from "@app/components/site-selector"; import { useEnvContext } from "@app/hooks/useEnvContext"; -import { usePaidStatus } from "@app/hooks/usePaidStatus"; import { toast } from "@app/hooks/useToast"; import { createApiClient, formatAxiosError } from "@app/lib/api"; import { @@ -77,12 +75,6 @@ export default function CreatePrivateResourcePage() { const { env } = useEnvContext(); const api = createApiClient({ env }); const orgId = params.orgId as string; - const disableEnterpriseFeatures = env.flags.disableEnterpriseFeatures; - const { isPaidUser } = usePaidStatus(); - const httpSectionDisabled = !isPaidUser( - tierMatrix.advancedPrivateResources - ); - const sshSectionDisabled = !isPaidUser(tierMatrix.advancedPrivateResources); const [isSubmitting, startTransition] = useTransition(); const siteIdParam = searchParams.get("siteId"); @@ -158,20 +150,16 @@ export default function CreatePrivateResourcePage() { title: t("createInternalResourceDialogModeCidr"), description: t("privateResourceTypeCidrDescription") }, - ...(!disableEnterpriseFeatures - ? [ - { - value: "http" as const, - title: t("createInternalResourceDialogModeHttp"), - description: t("privateResourceTypeHttpDescription") - }, - { - value: "ssh" as const, - title: t("createInternalResourceDialogModeSsh"), - description: t("privateResourceTypeSshDescription") - } - ] - : []), + { + value: "http" as const, + title: t("createInternalResourceDialogModeHttp"), + description: t("privateResourceTypeHttpDescription") + }, + { + value: "ssh" as const, + title: t("createInternalResourceDialogModeSsh"), + description: t("privateResourceTypeSshDescription") + }, { value: "inference" as const, title: t("createInternalResourceDialogModeInference"), @@ -179,11 +167,6 @@ export default function CreatePrivateResourcePage() { } ]; - const submitDisabled = - isSubmitting || - (mode === "http" && httpSectionDisabled) || - (mode === "ssh" && sshSectionDisabled); - function onSubmit(values: FormValues) { startTransition(async () => { try { @@ -467,10 +450,7 @@ export default function CreatePrivateResourcePage() { )} watch={asAnyWatch(form.watch)} labelPrefix="create" - disabled={ - mode === "ssh" && - sshSectionDisabled - } + disabled={false} /> )} @@ -584,9 +564,6 @@ export default function CreatePrivateResourcePage() { {/* HTTP configuration */} {mode === "http" && ( - {t("httpSettings")} @@ -597,62 +574,44 @@ export default function CreatePrivateResourcePage() { )} -
- - - - - - - - - - - - -
+ + + + + + + + + + + + + )} {/* SSH server */} {mode === "ssh" && ( - {t("sshSettings")} @@ -661,37 +620,23 @@ export default function CreatePrivateResourcePage() { {t("sshServerDescription")} -
- - - - - -
+ + + + + )} @@ -776,7 +721,7 @@ export default function CreatePrivateResourcePage() {