From 365a905e697971476ac96511bc6d7bc02ddbadee Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 20 Aug 2026 15:17:48 -0400 Subject: [PATCH] Add more data to the models catalog list --- docs/ai-gateway-provider-selection.md | 33 +++++- server/lib/aiModelCatalog.ts | 112 ++++++++++++++---- server/lib/aiModelDiscovery.ts | 80 +++++++++++-- .../private/routers/aiGateway/logAiSession.ts | 2 +- server/routers/aiGateway/anthropicModels.ts | 21 +++- 5 files changed, 205 insertions(+), 43 deletions(-) diff --git a/docs/ai-gateway-provider-selection.md b/docs/ai-gateway-provider-selection.md index d35e8afe7..af67fd4f6 100644 --- a/docs/ai-gateway-provider-selection.md +++ b/docs/ai-gateway-provider-selection.md @@ -161,11 +161,34 @@ enumerable. Provider types with no catalog mapping (`openRouter`, allow on those types lists nothing - **add exact allow entries to make their models discoverable.** -Fields the API declares nullable and an allow/block list cannot supply -(`max_input_tokens`, `max_tokens`, `capabilities`) are returned as `null`. -`display_name` and `created_at` come from the configured model row when the id -matches one; otherwise the id doubles as the display name and `created_at` is -the epoch, which the Models API permits when the release date is unknown. +### Where each field comes from + +Token limits and capability flags can't be derived from an allow/block list. +They come from the model catalog (`server/lib/aiModelCatalog.ts`), which the +Fossorial API builds from LiteLLM: + +| Field | Source | +|-------|--------| +| `max_input_tokens` | catalog `limits.input` | +| `max_tokens` | catalog `limits.output` | +| `capabilities` | catalog flags, mapped to the Models API shape by `capabilitiesFromCatalog` | +| `display_name` | the configured model row's name, else the model id | +| `created_at` | the configured model row's timestamp, else the epoch | + +A model the catalog doesn't know (an exact allow entry for a fine-tune, say) +reports `null` for all three metadata fields. The Models API declares them +nullable, so that is a valid answer rather than a broken one. + +The catalog's flags are coarser than the Models API describes: it carries a +single `reasoning` flag with no way to distinguish adaptive from +`budget_tokens`-style thinking, and nothing at all for batch, citations, code +execution, PDF input, or context management. Anything it reports as unknown +(`null`) is surfaced as unsupported rather than invented, so `capabilities` +understates rather than overstates what a model can do. + +The gateway does **not** query the provider's own `/v1/models`. Discovery is +answered entirely from local state. + Results are ordered newest-first with the id as tie-break, and paginated with Anthropic's `limit` / `after_id` / `before_id` semantics (default 20, max 1000). diff --git a/server/lib/aiModelCatalog.ts b/server/lib/aiModelCatalog.ts index 5f2f7fb94..db5c0b692 100644 --- a/server/lib/aiModelCatalog.ts +++ b/server/lib/aiModelCatalog.ts @@ -44,6 +44,20 @@ export function getCatalogProviderForType( return PROVIDER_CATALOG_MAP[type]; } +/** + * Per-model feature flags as reported upstream. `null` means the catalog has + * no data for that model - deliberately distinct from `false`, so consumers + * can tell "unsupported" apart from "unknown". + */ +export type AiModelCapabilityFlags = { + functionCalling: boolean | null; + vision: boolean | null; + promptCaching: boolean | null; + reasoning: boolean | null; + responseSchema: boolean | null; + webSearch: boolean | null; +}; + export type AiModelCatalogEntry = { provider: CatalogProvider; model: string; @@ -53,8 +67,20 @@ export type AiModelCatalogEntry = { cache: number | null; reasoning: number | null; }; + limits: { + /** Context window. */ + input: number | null; + /** Cap on the output/max_tokens request parameter. */ + output: number | null; + }; + capabilities: AiModelCapabilityFlags; }; +const flag = z.boolean().nullable().optional(); + +// limits/capabilities are optional so a catalog published before they were +// added (or an operator's own merge_file) still parses - those entries just +// report unknown metadata rather than failing the whole payload. const catalogEntrySchema = z.object({ model: z.string(), provider: z.string(), @@ -65,6 +91,22 @@ const catalogEntrySchema = z.object({ cache: z.number().nullable().optional(), reasoning: z.number().nullable().optional() }) + .optional(), + limits: z + .object({ + input: z.number().nullable().optional(), + output: z.number().nullable().optional() + }) + .optional(), + capabilities: z + .object({ + functionCalling: flag, + vision: flag, + promptCaching: flag, + reasoning: flag, + responseSchema: flag, + webSearch: flag + }) .optional() }); @@ -108,6 +150,18 @@ function normalizeEntry(raw: RawCatalogEntry): AiModelCatalogEntry | null { out: raw.pricing?.out ?? null, cache: raw.pricing?.cache ?? null, reasoning: raw.pricing?.reasoning ?? null + }, + limits: { + input: raw.limits?.input ?? null, + output: raw.limits?.output ?? null + }, + capabilities: { + functionCalling: raw.capabilities?.functionCalling ?? null, + vision: raw.capabilities?.vision ?? null, + promptCaching: raw.capabilities?.promptCaching ?? null, + reasoning: raw.capabilities?.reasoning ?? null, + responseSchema: raw.capabilities?.responseSchema ?? null, + webSearch: raw.capabilities?.webSearch ?? null } }; } @@ -284,34 +338,44 @@ export class AiModelCatalog { export const aiModelCatalog = new AiModelCatalog(); +/** + * Full catalog entries for a provider type, deduplicated by model id and + * sorted by id. Model discovery uses these to report real token limits and + * capability flags; `listCatalogModelsForType` is the id-only view of the + * same list. + */ +export function listCatalogEntriesForType( + type: AiProviderType, + query?: string +): AiModelCatalogEntry[] { + const catalogProvider = getCatalogProviderForType(type); + + let entries = catalogProvider ? aiModelCatalog.list(catalogProvider) : []; + + if (query) { + const q = query.toLowerCase(); + entries = entries.filter((e) => e.model.toLowerCase().includes(q)); + } + + const seen = new Set(); + entries = entries.filter((e) => { + if (seen.has(e.model)) { + return false; + } + seen.add(e.model); + return true; + }); + + return [...entries].sort((a, b) => a.model.localeCompare(b.model)); +} + export function listCatalogModelsForType( type: AiProviderType, query?: string ): { model: string }[] { - const catalogProvider = getCatalogProviderForType(type); - - let models = catalogProvider - ? aiModelCatalog.list(catalogProvider).map((entry) => ({ - model: entry.model - })) - : []; - - if (query) { - const q = query.toLowerCase(); - models = models.filter((m) => m.model.toLowerCase().includes(q)); - } - - const seen = new Set(); - models = models.filter((m) => { - if (seen.has(m.model)) { - return false; - } - seen.add(m.model); - return true; - }); - - models.sort((a, b) => a.model.localeCompare(b.model)); - return models; + return listCatalogEntriesForType(type, query).map((entry) => ({ + model: entry.model + })); } /** diff --git a/server/lib/aiModelDiscovery.ts b/server/lib/aiModelDiscovery.ts index 873abdb98..93ef28ea9 100644 --- a/server/lib/aiModelDiscovery.ts +++ b/server/lib/aiModelDiscovery.ts @@ -2,6 +2,7 @@ import { isAllowedByLists, isModelKeyPattern } from "@server/lib/aiModelKeyMatch"; +import type { AiModelCapabilityFlags } from "@server/lib/aiModelCatalog"; // Anthropic's Models API pagination: 20 per page by default, 1..1000. export const MODEL_PAGE_DEFAULT_LIMIT = 20; @@ -25,12 +26,66 @@ export type AnthropicModelInfo = { created_at: string; max_input_tokens: number | null; max_tokens: number | null; - capabilities: null; + capabilities: Record | null; }; /** A model row an administrator configured explicitly on a provider. */ export type ConfiguredModel = { name: string; createdAt: number }; +/** What the pricing catalog knows about a model beyond its id. */ +export type CatalogModelMetadata = { + maxInputTokens: number | null; + maxOutputTokens: number | null; + capabilities: AiModelCapabilityFlags; +}; + +/** + * Translates the catalog's flat feature flags into the nested shape + * Anthropic's Models API uses. Best-effort by nature: the catalog carries a + * coarser set of flags than the Models API describes, so anything it reports + * as unknown (`null`) is surfaced as unsupported rather than invented. + */ +export function capabilitiesFromCatalog( + flags: AiModelCapabilityFlags +): Record { + const supported = (value: boolean | null) => ({ + supported: value === true + }); + // The catalog has a single `reasoning` flag and no way to distinguish + // adaptive from budget_tokens-style thinking, so both variants follow it. + const reasoning = flags.reasoning === true; + + return { + batch: supported(null), + citations: supported(null), + code_execution: supported(null), + context_management: { + supported: false, + clear_thinking_20251015: null, + clear_tool_uses_20250919: null, + compact_20260112: null + }, + effort: { + supported: reasoning, + low: supported(flags.reasoning), + medium: supported(flags.reasoning), + high: supported(flags.reasoning), + max: supported(flags.reasoning), + xhigh: null + }, + image_input: supported(flags.vision), + pdf_input: supported(null), + structured_outputs: supported(flags.responseSchema), + thinking: { + supported: reasoning, + types: { + adaptive: { supported: reasoning }, + enabled: { supported: reasoning } + } + } + }; +} + /** * One attached provider's contribution to a resource's model listing, with the * allow/block lists already resolved for the attachment's access mode. @@ -40,12 +95,13 @@ export type ModelDiscoveryProvider = { allows: string[]; blocks: string[]; /** - * Concrete model ids the provider's type is known to serve. This is what - * lets a wildcard allow such as `claude-*` enumerate into real ids; - * provider types with no catalog (aggregators, custom) pass an empty list - * and surface only their exact allow entries. + * Concrete model ids the provider's type is known to serve, with whatever + * the catalog knows about each. This is what lets a wildcard allow such as + * `claude-*` enumerate into real ids; provider types with no catalog + * (aggregators, custom) pass an empty map and surface only their exact + * allow entries. */ - catalogModelIds: string[]; + catalog: Map; /** Keyed by model key, for display names and creation times. */ configured: Map; }; @@ -73,7 +129,7 @@ export function expandProviderModels( candidates.add(allow); } } - for (const modelId of provider.catalogModelIds) { + for (const modelId of provider.catalog.keys()) { candidates.add(modelId); } @@ -83,6 +139,8 @@ export function expandProviderModels( continue; } const configured = provider.configured.get(modelKey); + const catalog = provider.catalog.get(modelKey); + models.push({ type: "model", id: modelKey, @@ -90,9 +148,11 @@ export function expandProviderModels( created_at: configured ? new Date(configured.createdAt).toISOString() : UNKNOWN_CREATED_AT, - max_input_tokens: null, - max_tokens: null, - capabilities: null + max_input_tokens: catalog?.maxInputTokens ?? null, + max_tokens: catalog?.maxOutputTokens ?? null, + capabilities: catalog + ? capabilitiesFromCatalog(catalog.capabilities) + : null }); } diff --git a/server/private/routers/aiGateway/logAiSession.ts b/server/private/routers/aiGateway/logAiSession.ts index 527dc0b5b..47b0f039e 100644 --- a/server/private/routers/aiGateway/logAiSession.ts +++ b/server/private/routers/aiGateway/logAiSession.ts @@ -15,7 +15,7 @@ import { logsDb, db, orgs, aiSessionLog, type AiProvider } from "@server/db"; import type { InferInsertModel } from "drizzle-orm"; import logger from "@server/logger"; import { and, eq, lt } from "drizzle-orm"; -import cache from "#dynamic/lib/cache"; +import cache from "#private/lib/cache"; import { calculateCutoffTimestamp } from "@server/lib/cleanupLogs"; import { sanitizeString } from "@server/lib/sanitize"; import type { AiCapability } from "@server/lib/aiCapabilities"; diff --git a/server/routers/aiGateway/anthropicModels.ts b/server/routers/aiGateway/anthropicModels.ts index b89868510..9e15a78de 100644 --- a/server/routers/aiGateway/anthropicModels.ts +++ b/server/routers/aiGateway/anthropicModels.ts @@ -15,12 +15,13 @@ import { isAiGatewayTrustHeaderValid } from "@server/lib/aiGatewayTrust"; import { resolveEffectiveLists } from "@server/lib/aiInferenceResource"; -import { listCatalogModelsForType } from "@server/lib/aiModelCatalog"; +import { listCatalogEntriesForType } from "@server/lib/aiModelCatalog"; import { listPermittedModels, paginateModels, MODEL_PAGE_DEFAULT_LIMIT, MODEL_PAGE_MAX_LIMIT, + type CatalogModelMetadata, type ConfiguredModel, type ModelDiscoveryProvider } from "@server/lib/aiModelDiscovery"; @@ -113,6 +114,20 @@ async function loadProviderModelLists( return lists; } +function catalogMetadataForType( + type: AiProviderType +): Map { + const metadata = new Map(); + for (const entry of listCatalogEntriesForType(type)) { + metadata.set(entry.model, { + maxInputTokens: entry.limits.input, + maxOutputTokens: entry.limits.output, + capabilities: entry.capabilities + }); + } + return metadata; +} + function buildDiscoveryProviders( attachments: ProviderAttachment[], resourceListsByProvider: Map, @@ -133,9 +148,9 @@ function buildDiscoveryProviders( providerId, allows, blocks, - catalogModelIds: listCatalogModelsForType( + catalog: catalogMetadataForType( attachment.provider.type as AiProviderType - ).map((entry) => entry.model), + ), configured: lists.configuredByProvider.get(providerId) ?? new Map() }; });