Compare commits

..

4 Commits

Author SHA1 Message Date
Milo Schwartz 929acc5b1c Merge pull request #3614 from fosrl/dev
Dev
2026-08-19 12:05:12 -04:00
Owen 437ec50019 Update the usage records to use logs db 2026-08-19 12:03:30 -04:00
miloschwartz a7bbafd2f0 bump badger version 2026-08-19 12:01:01 -04:00
Owen 6824ec6e3e Clean up empty execute in migration 2026-08-19 11:51:12 -04:00
12 changed files with 3445 additions and 2146 deletions
+3331 -2038
View File
File diff suppressed because it is too large Load Diff
+72 -72
View File
@@ -32,49 +32,49 @@
"format": "prettier --write ."
},
"dependencies": {
"@asteasolutions/zod-to-openapi": "9.1.0",
"@aws-sdk/client-s3": "3.1111.0",
"@asteasolutions/zod-to-openapi": "8.5.0",
"@aws-sdk/client-s3": "3.1056.0",
"@devolutions/iron-remote-desktop": "https://static.pangolin.net/packages/devolutions-iron-remote-desktop-0.0.0.tgz",
"@devolutions/iron-remote-desktop-rdp": "https://static.pangolin.net/packages/devolutions-iron-remote-desktop-rdp-0.0.1.tgz",
"@headlessui/react": "2.2.10",
"@hookform/resolvers": "5.9.0",
"@hookform/resolvers": "5.4.0",
"@monaco-editor/react": "4.7.0",
"@node-rs/argon2": "2.1.0",
"@node-rs/argon2": "2.0.2",
"@novnc/novnc": "^1.7.0",
"@oslojs/crypto": "1.0.1",
"@oslojs/encoding": "1.1.0",
"@radix-ui/react-avatar": "1.2.6",
"@radix-ui/react-checkbox": "1.3.11",
"@radix-ui/react-collapsible": "1.1.20",
"@radix-ui/react-dialog": "1.1.23",
"@radix-ui/react-dropdown-menu": "2.1.24",
"@radix-ui/react-avatar": "1.1.11",
"@radix-ui/react-checkbox": "1.3.3",
"@radix-ui/react-collapsible": "1.1.12",
"@radix-ui/react-dialog": "1.1.15",
"@radix-ui/react-dropdown-menu": "2.1.16",
"@radix-ui/react-icons": "1.3.2",
"@radix-ui/react-label": "2.1.15",
"@radix-ui/react-popover": "1.1.23",
"@radix-ui/react-progress": "1.1.16",
"@radix-ui/react-radio-group": "1.4.7",
"@radix-ui/react-scroll-area": "1.2.18",
"@radix-ui/react-select": "2.3.7",
"@radix-ui/react-separator": "1.1.15",
"@radix-ui/react-slot": "1.3.3",
"@radix-ui/react-switch": "1.3.7",
"@radix-ui/react-tabs": "1.1.21",
"@radix-ui/react-toast": "1.2.23",
"@radix-ui/react-tooltip": "1.2.16",
"@radix-ui/react-label": "2.1.8",
"@radix-ui/react-popover": "1.1.15",
"@radix-ui/react-progress": "1.1.8",
"@radix-ui/react-radio-group": "1.3.8",
"@radix-ui/react-scroll-area": "1.2.10",
"@radix-ui/react-select": "2.2.6",
"@radix-ui/react-separator": "1.1.8",
"@radix-ui/react-slot": "1.2.4",
"@radix-ui/react-switch": "1.2.6",
"@radix-ui/react-tabs": "1.1.13",
"@radix-ui/react-toast": "1.2.15",
"@radix-ui/react-tooltip": "1.2.8",
"@react-email/body": "0.3.0",
"@react-email/components": "1.0.12",
"@react-email/render": "2.1.0",
"@react-email/render": "2.0.8",
"@react-email/tailwind": "2.0.7",
"@simplewebauthn/browser": "13.3.0",
"@simplewebauthn/server": "13.3.2",
"@simplewebauthn/server": "13.3.1",
"@tailwindcss/forms": "0.5.11",
"@tanstack/react-query": "5.101.4",
"@tanstack/react-table": "9.1.2",
"@tanstack/react-query": "5.100.14",
"@tanstack/react-table": "8.21.3",
"@xterm/addon-fit": "^0.11.0",
"@xterm/addon-web-links": "^0.12.0",
"@xterm/xterm": "^6.0.0",
"arctic": "3.7.0",
"axios": "1.19.0",
"axios": "1.18.0",
"better-sqlite3": "11.9.1",
"canvas-confetti": "1.9.4",
"class-variance-authority": "0.7.1",
@@ -86,62 +86,62 @@
"d3": "7.9.0",
"drizzle-orm": "0.45.2",
"express": "5.2.1",
"express-rate-limit": "8.6.2",
"express-rate-limit": "8.5.2",
"glob": "13.0.6",
"gpt-tokenizer": "^4.0.0",
"helmet": "8.3.0",
"gpt-tokenizer": "^3.4.0",
"helmet": "8.2.0",
"http-errors": "2.0.1",
"input-otp": "1.4.2",
"ioredis": "6.0.0",
"ioredis": "5.11.0",
"jmespath": "0.16.0",
"js-yaml": "5.3.0",
"js-yaml": "4.3.1",
"jsonwebtoken": "9.0.3",
"lucide-react": "1.31.0",
"maxmind": "5.0.7",
"lucide-react": "1.17.0",
"maxmind": "5.0.6",
"moment": "2.30.1",
"next": "16.3.1",
"next-intl": "4.13.6",
"next-intl": "4.13.0",
"next-themes": "0.4.6",
"nextjs-toploader": "3.9.17",
"node-cache": "5.1.2",
"nodemailer": "9.0.5",
"nodemailer": "9.0.1",
"oslo": "1.2.1",
"pg": "8.23.0",
"posthog-node": "5.49.1",
"pg": "8.21.0",
"posthog-node": "5.35.6",
"qrcode.react": "4.2.0",
"react": "19.2.8",
"react-day-picker": "10.0.1",
"react-dom": "19.2.8",
"react": "19.2.6",
"react-day-picker": "9.14.0",
"react-dom": "19.2.6",
"react-easy-sort": "1.8.0",
"react-hook-form": "7.85.0",
"react-icons": "5.7.0",
"recharts": "3.10.1",
"react-hook-form": "7.76.1",
"react-icons": "5.6.0",
"recharts": "3.8.1",
"reodotdev": "1.1.0",
"semver": "7.8.5",
"semver": "7.8.1",
"sshpk": "1.18.0",
"stripe": "22.5.0",
"stripe": "22.2.0",
"swagger-ui-express": "5.0.1",
"tailwind-merge": "3.6.0",
"topojson-client": "3.1.0",
"tw-animate-css": "1.4.0",
"use-debounce": "10.1.1",
"uuid": "14.0.1",
"uuid": "14.0.0",
"vaul": "1.1.2",
"visionscarto-world-atlas": "1.0.0",
"winston": "3.19.0",
"winston-daily-rotate-file": "5.0.0",
"ws": "8.21.3",
"ws": "8.21.0",
"yaml": "2.9.0",
"yargs": "18.1.0",
"yargs": "18.0.0",
"zod": "4.4.3",
"zod-validation-error": "5.0.0"
},
"devDependencies": {
"@dotenvx/dotenvx": "2.21.0",
"@dotenvx/dotenvx": "1.69.1",
"@esbuild-plugins/tsconfig-paths": "0.1.2",
"@react-email/ui": "^6.9.2",
"@tailwindcss/postcss": "4.3.3",
"@tanstack/react-query-devtools": "5.101.4",
"@tailwindcss/postcss": "4.3.0",
"@tanstack/react-query-devtools": "5.100.14",
"@types/better-sqlite3": "7.6.13",
"@types/cookie-parser": "1.4.10",
"@types/cors": "2.8.19",
@@ -152,36 +152,36 @@
"@types/jmespath": "0.15.2",
"@types/js-yaml": "4.0.9",
"@types/jsonwebtoken": "9.0.10",
"@types/node": "26.2.0",
"@types/nodemailer": "8.0.1",
"@types/node": "25.9.1",
"@types/nodemailer": "8.0.0",
"@types/nprogress": "0.2.3",
"@types/pg": "8.21.0",
"@types/react": "19.2.18",
"@types/react-dom": "19.2.4",
"@types/semver": "7.8.0",
"@types/sshpk": "1.17.5",
"@types/pg": "8.20.0",
"@types/react": "19.2.15",
"@types/react-dom": "19.2.3",
"@types/semver": "7.7.1",
"@types/sshpk": "1.17.4",
"@types/swagger-ui-express": "4.1.8",
"@types/topojson-client": "3.1.5",
"@types/ws": "8.18.1",
"@types/yargs": "17.0.35",
"babel-plugin-react-compiler": "1.0.0",
"drizzle-kit": "0.31.10",
"esbuild": "0.28.2",
"esbuild-node-externals": "2.0.0",
"eslint": "10.8.1",
"eslint-config-next": "16.3.1",
"postcss": "8.5.26",
"prettier": "3.9.6",
"react-email": "6.9.2",
"tailwindcss": "4.3.3",
"tsc-alias": "1.9.1",
"tsx": "4.23.12",
"typescript": "7.0.2",
"typescript-eslint": "8.67.0"
"esbuild": "0.28.0",
"esbuild-node-externals": "1.22.0",
"eslint": "10.4.0",
"eslint-config-next": "16.2.6",
"postcss": "8.5.23",
"prettier": "3.8.3",
"react-email": "6.5.0",
"tailwindcss": "4.3.0",
"tsc-alias": "1.8.17",
"tsx": "4.22.3",
"typescript": "6.0.3",
"typescript-eslint": "8.60.0"
},
"overrides": {
"esbuild": "0.28.2",
"esbuild": "0.28.0",
"dompurify": "3.4.0",
"postcss": "8.5.26"
"postcss": "8.5.23"
}
}
+4 -3
View File
@@ -16,6 +16,7 @@ import {
aiModels,
aiUsageRecords,
db,
logsDb,
userOrgRoles
} from "@server/db";
import { modelKeyMatches } from "@server/lib/aiModelKeyMatch";
@@ -168,7 +169,7 @@ async function sumUsageAmount(
): Promise<number> {
const column =
unit === "usd" ? aiUsageRecords.costUsd : aiUsageRecords.totalTokens;
const [row] = await db
const [row] = await logsDb
.select({ total: sql<number>`coalesce(sum(${column}), 0)` })
.from(aiUsageRecords)
.where(where);
@@ -200,7 +201,7 @@ export async function sumUsageForBudget(
if (!model) {
return 0;
}
const rows = await db
const rows = await logsDb
.select({
requestedModel: aiUsageRecords.requestedModel,
costUsd: aiUsageRecords.costUsd,
@@ -496,7 +497,7 @@ async function flushUsageRecords() {
try {
// Use a transaction to ensure all inserts succeed or fail together
await db.transaction(async (tx) => {
await logsDb.transaction(async (tx) => {
// Batch insert in groups to avoid overwhelming the database
const DB_BATCH_SIZE = 25;
for (let i = 0; i < recordsToWrite.length; i += DB_BATCH_SIZE) {
@@ -312,7 +312,7 @@ async function enrichWithDetails(
>();
const sessionIds = logs.map((log) => log.sessionId);
if (sessionIds.length > 0) {
const usageDetails = await primaryDb
const usageDetails = await logsDb
.select({
sessionId: aiUsageRecords.sessionId,
promptTokens: aiUsageRecords.promptTokens,
@@ -1,5 +1,6 @@
import {
db,
logsDb,
aiUsageRecords,
aiProviders,
resources,
@@ -78,22 +79,22 @@ async function query(data: Q) {
uniqueUsers,
uniqueVirtualApiKeys
] = await Promise.all([
db
logsDb
.selectDistinct({ id: aiUsageRecords.providerId })
.from(aiUsageRecords)
.where(baseConditions)
.limit(DISTINCT_LIMIT + 1),
db
logsDb
.selectDistinct({ model: aiUsageRecords.requestedModel })
.from(aiUsageRecords)
.where(baseConditions)
.limit(DISTINCT_LIMIT + 1),
db
logsDb
.selectDistinct({ id: aiUsageRecords.resourceId })
.from(aiUsageRecords)
.where(and(baseConditions, not(isNull(aiUsageRecords.resourceId))))
.limit(DISTINCT_LIMIT + 1),
db
logsDb
.selectDistinct({ id: aiUsageRecords.siteResourceId })
.from(aiUsageRecords)
.where(
@@ -104,12 +105,12 @@ async function query(data: Q) {
)
)
.limit(DISTINCT_LIMIT + 1),
db
logsDb
.selectDistinct({ userId: aiUsageRecords.userId })
.from(aiUsageRecords)
.where(and(baseConditions, not(isNull(aiUsageRecords.userId))))
.limit(DISTINCT_LIMIT + 1),
db
logsDb
.selectDistinct({ id: aiUsageRecords.virtualApiKeyId })
.from(aiUsageRecords)
.where(
@@ -1,4 +1,4 @@
import { db, aiUsageRecords } from "@server/db";
import { logsDb, aiUsageRecords } from "@server/db";
import { registry } from "@server/openApi";
import { NextFunction } from "express";
import { Request, Response } from "express";
@@ -29,7 +29,7 @@ async function query(data: Q) {
const roleUserIds = await resolveRoleUserIds(data.orgId, data.roleId);
const baseConditions = buildAiUsageWhere(data, roleUserIds);
const [totalsRow] = await db
const [totalsRow] = await logsDb
.select({
requests: count(),
promptTokens: sql<number>`COALESCE(SUM(${aiUsageRecords.promptTokens}), 0)`,
@@ -46,7 +46,7 @@ async function query(data: Q) {
const dayExpr = dayBucketExpr();
const requestsPerDay = await db
const requestsPerDay = await logsDb
.select({
day: dayExpr.as("day"),
requests: count()
@@ -56,7 +56,7 @@ async function query(data: Q) {
.groupBy(dayExpr)
.orderBy(dayExpr);
const tokensPerDay = await db
const tokensPerDay = await logsDb
.select({
day: dayExpr.as("day"),
promptTokens: sql<number>`COALESCE(SUM(${aiUsageRecords.promptTokens}), 0)`,
@@ -70,7 +70,7 @@ async function query(data: Q) {
.groupBy(dayExpr)
.orderBy(dayExpr);
const costPerDay = await db
const costPerDay = await logsDb
.select({
day: dayExpr.as("day"),
cost: sql<number>`COALESCE(SUM(${aiUsageRecords.costUsd}), 0)`
@@ -80,7 +80,7 @@ async function query(data: Q) {
.groupBy(dayExpr)
.orderBy(dayExpr);
const modelByDay = await db
const modelByDay = await logsDb
.select({
day: dayExpr.as("day"),
model: aiUsageRecords.requestedModel,
@@ -117,7 +117,7 @@ async function query(data: Q) {
topModelsByTokens
);
const topModelsRaw = await db
const topModelsRaw = await logsDb
.select({
model: aiUsageRecords.requestedModel,
requests: count(),
@@ -1,4 +1,4 @@
import { db, aiUsageRecords, aiProviders } from "@server/db";
import { db, logsDb, aiUsageRecords, aiProviders } from "@server/db";
import { registry } from "@server/openApi";
import { NextFunction } from "express";
import { Request, Response } from "express";
@@ -29,7 +29,7 @@ async function query(data: Q) {
const baseConditions = buildAiUsageWhere(data, roleUserIds);
const dayExpr = dayBucketExpr();
const providerByDay = await db
const providerByDay = await logsDb
.select({
day: dayExpr.as("day"),
providerId: aiUsageRecords.providerId,
@@ -69,7 +69,7 @@ async function query(data: Q) {
topByTokens
);
const topProvidersRaw = await db
const topProvidersRaw = await logsDb
.select({
providerId: aiUsageRecords.providerId,
requests: count(),
@@ -1,4 +1,4 @@
import { db, aiUsageRecords, resources, siteResources } from "@server/db";
import { db, logsDb, aiUsageRecords, resources, siteResources } from "@server/db";
import { registry } from "@server/openApi";
import { NextFunction } from "express";
import { Request, Response } from "express";
@@ -39,7 +39,7 @@ async function query(data: Q) {
const baseConditions = buildAiUsageWhere(data, roleUserIds);
const dayExpr = dayBucketExpr();
const resourceByDay = await db
const resourceByDay = await logsDb
.select({
day: dayExpr.as("day"),
resourceId: aiUsageRecords.resourceId,
@@ -80,7 +80,7 @@ async function query(data: Q) {
topByTokens
);
const topResourcesRaw = await db
const topResourcesRaw = await logsDb
.select({
resourceId: aiUsageRecords.resourceId,
siteResourceId: aiUsageRecords.siteResourceId,
@@ -1,4 +1,11 @@
import { db, aiUsageRecords, users, roles, userOrgRoles } from "@server/db";
import {
db,
logsDb,
aiUsageRecords,
users,
roles,
userOrgRoles
} from "@server/db";
import { registry } from "@server/openApi";
import { NextFunction } from "express";
import { Request, Response } from "express";
@@ -36,7 +43,7 @@ async function query(data: Q) {
// userId, so role totals are derived by expanding each user's usage into
// every role they hold in the org (per-role double counting for
// multi-role users is expected/accepted).
const userByDay = await db
const userByDay = await logsDb
.select({
day: dayExpr.as("day"),
userId: aiUsageRecords.userId,
@@ -48,7 +55,7 @@ async function query(data: Q) {
.groupBy(dayExpr, aiUsageRecords.userId)
.orderBy(dayExpr);
const userTotalsRaw = await db
const userTotalsRaw = await logsDb
.select({
userId: aiUsageRecords.userId,
requests: count(),
@@ -1,4 +1,4 @@
import { db, aiUsageRecords, virtualApiKeys } from "@server/db";
import { db, logsDb, aiUsageRecords, virtualApiKeys } from "@server/db";
import { registry } from "@server/openApi";
import { NextFunction } from "express";
import { Request, Response } from "express";
@@ -31,7 +31,7 @@ async function query(data: Q) {
const baseConditions = buildAiUsageWhere(data, roleUserIds);
const dayExpr = dayBucketExpr();
const virtualApiKeyByDay = await db
const virtualApiKeyByDay = await logsDb
.select({
day: dayExpr.as("day"),
virtualApiKeyId: aiUsageRecords.virtualApiKeyId,
@@ -43,7 +43,7 @@ async function query(data: Q) {
.groupBy(dayExpr, aiUsageRecords.virtualApiKeyId)
.orderBy(dayExpr);
const virtualApiKeyTotalsRaw = await db
const virtualApiKeyTotalsRaw = await logsDb
.select({
virtualApiKeyId: aiUsageRecords.virtualApiKeyId,
requests: count(),
+2 -5
View File
@@ -18,9 +18,6 @@ export default async function migration() {
try {
await db.execute(sql`BEGIN`);
await db.execute(sql`
`);
await db.execute(sql`
CREATE TABLE "aiBudgetBreachEvents" (
"id" serial PRIMARY KEY NOT NULL,
@@ -454,14 +451,14 @@ export default async function migration() {
throw new Error(fromZodError(parsedConfig.error).toString());
}
traefikConfig.experimental.plugins.badger.version = "v1.6.0";
traefikConfig.experimental.plugins.badger.version = "v1.6.1";
const updatedTraefikYaml = yaml.dump(traefikConfig);
fs.writeFileSync(traefikPath, updatedTraefikYaml, "utf8");
console.log(
"Updated the version of Badger in your Traefik configuration to v1.6.0"
"Updated the version of Badger in your Traefik configuration to v1.6.1"
);
} catch (e) {
console.log(
+2 -2
View File
@@ -456,14 +456,14 @@ export default async function migration() {
throw new Error(fromZodError(parsedConfig.error).toString());
}
traefikConfig.experimental.plugins.badger.version = "v1.6.0";
traefikConfig.experimental.plugins.badger.version = "v1.6.1";
const updatedTraefikYaml = yaml.dump(traefikConfig);
fs.writeFileSync(traefikPath, updatedTraefikYaml, "utf8");
console.log(
"Updated the version of Badger in your Traefik configuration to v1.6.0"
"Updated the version of Badger in your Traefik configuration to v1.6.1"
);
} catch (e) {
console.log(