diff --git a/server/lib/textCompression.ts b/server/lib/textCompression.ts new file mode 100644 index 000000000..2ef0769f7 --- /dev/null +++ b/server/lib/textCompression.ts @@ -0,0 +1,15 @@ +import { gzipSync, gunzipSync } from "zlib"; + +/** + * Gzip a string and return it as base64 so it can be stored in a TEXT column. + */ +export function compressText(value: string): string { + return gzipSync(Buffer.from(value, "utf8")).toString("base64"); +} + +/** + * Reverse of compressText - base64-decode and gunzip back to the original string. + */ +export function decompressText(value: string): string { + return gunzipSync(Buffer.from(value, "base64")).toString("utf8"); +} diff --git a/server/private/lib/logStreaming/LogStreamingManager.ts b/server/private/lib/logStreaming/LogStreamingManager.ts index 14ada27b0..719c79895 100644 --- a/server/private/lib/logStreaming/LogStreamingManager.ts +++ b/server/private/lib/logStreaming/LogStreamingManager.ts @@ -25,6 +25,7 @@ import { import logger from "@server/logger"; import { and, eq, gt, desc, max, sql } from "drizzle-orm"; import { decrypt } from "@server/lib/crypto"; +import { decompressText } from "@server/lib/textCompression"; import config from "@server/lib/config"; import { LogType, @@ -680,8 +681,8 @@ export class LogStreamingManager { Record & { id: number } >; - case "aiSession": - return (await logsDb + case "aiSession": { + const rows = (await logsDb .select() .from(aiSessionLog) .where( @@ -694,6 +695,33 @@ export class LogStreamingManager { .limit(limit)) as Array< Record & { id: number } >; + + const compressedFields = [ + "requestBody", + "responseBody", + "normalizedRequest", + "normalizedResponse" + ] as const; + + for (const row of rows) { + for (const field of compressedFields) { + const value = row[field]; + if (typeof value !== "string") { + continue; + } + try { + row[field] = decompressText(value); + } catch (error) { + logger.error( + `Failed to decompress AI session log field ${field}`, + { error } + ); + } + } + } + + return rows; + } } } diff --git a/server/private/routers/aiGateway/logAiSession.ts b/server/private/routers/aiGateway/logAiSession.ts index 1532aa5c5..b16e97f0b 100644 --- a/server/private/routers/aiGateway/logAiSession.ts +++ b/server/private/routers/aiGateway/logAiSession.ts @@ -18,6 +18,7 @@ import { and, eq, lt } from "drizzle-orm"; import cache from "#private/lib/cache"; import { calculateCutoffTimestamp } from "@server/lib/cleanupLogs"; import { sanitizeString } from "@server/lib/sanitize"; +import { compressText } from "@server/lib/textCompression"; import type { AiCapability } from "@server/lib/aiCapabilities"; import { normalizeAiRequest, @@ -151,7 +152,7 @@ async function getRetentionDays(orgId: string): Promise { } export async function cleanUpOldLogs(orgId: string, retentionDays: number) { - const cutoffTimestamp = calculateCutoffTimestamp(retentionDays) * 1000; + const cutoffTimestamp = calculateCutoffTimestamp(retentionDays); try { await logsDb @@ -255,13 +256,19 @@ export function logAiSession(data: { ), requestedModel: sanitizeString(data.requestedModel), isStream: data.isStream, - requestBody: sanitizeString(requestBodyText.value), - responseBody: sanitizeString(responseBodyText.value), + requestBody: compressText( + sanitizeString(requestBodyText.value) + ), + responseBody: compressText( + sanitizeString(responseBodyText.value) + ), normalizedRequest: normalizedRequestText - ? sanitizeString(normalizedRequestText.value) + ? compressText(sanitizeString(normalizedRequestText.value)) : undefined, normalizedResponse: normalizedResponseText - ? sanitizeString(normalizedResponseText.value) + ? compressText( + sanitizeString(normalizedResponseText.value) + ) : undefined, truncated: requestBodyText.truncated || diff --git a/server/routers/auditLogs/exportAiSessionLog.ts b/server/routers/auditLogs/exportAiSessionLog.ts index 2e673c107..d786c3eaa 100644 --- a/server/routers/auditLogs/exportAiSessionLog.ts +++ b/server/routers/auditLogs/exportAiSessionLog.ts @@ -11,7 +11,8 @@ import { queryAiSessionLogsQuery, queryAiSessionLogsParams, queryAiSession, - countAiSessionQuery + countAiSessionQuery, + decompressAiSessionLogRow } from "./queryAiSessionLog"; import { generateCSV } from "./generateCSV"; @@ -87,7 +88,9 @@ export async function exportAiSessionLogs( const baseQuery = queryAiSession(data); - const log = await baseQuery.limit(MAX_EXPORT_LIMIT); + const log = (await baseQuery.limit(MAX_EXPORT_LIMIT)).map( + decompressAiSessionLogRow + ); const csvData = generateCSV(log); diff --git a/server/routers/auditLogs/queryAiSessionLog.ts b/server/routers/auditLogs/queryAiSessionLog.ts index 3d0309905..5cd3cf8c1 100644 --- a/server/routers/auditLogs/queryAiSessionLog.ts +++ b/server/routers/auditLogs/queryAiSessionLog.ts @@ -24,6 +24,7 @@ import { AI_CAPABILITIES } from "@server/lib/aiCapabilities"; import response from "@server/lib/response"; import logger from "@server/logger"; import { getSevenDaysAgo } from "@app/lib/getSevenDaysAgo"; +import { decompressText } from "@server/lib/textCompression"; export const queryAiSessionLogsQuery = z.strictObject({ // iso string just validate its a parseable date @@ -166,6 +167,35 @@ export function queryAiSession(data: Q) { .orderBy(desc(aiSessionLog.createdAt)); } +function decompressField(value: string | null): string | null { + if (value == null) { + return value; + } + try { + return decompressText(value); + } catch (error) { + logger.error("Failed to decompress AI session log field", { error }); + return value; + } +} + +export function decompressAiSessionLogRow< + T extends { + requestBody: string | null; + responseBody: string | null; + normalizedRequest: string | null; + normalizedResponse: string | null; + } +>(row: T): T { + return { + ...row, + requestBody: decompressField(row.requestBody), + responseBody: decompressField(row.responseBody), + normalizedRequest: decompressField(row.normalizedRequest), + normalizedResponse: decompressField(row.normalizedResponse) + }; +} + async function enrichWithDetails( logs: Awaited> ) { @@ -620,7 +650,9 @@ export async function queryAiSessionLogs( const baseQuery = queryAiSession(data); - const logsRaw = await baseQuery.limit(data.limit).offset(data.offset); + const logsRaw = ( + await baseQuery.limit(data.limit).offset(data.offset) + ).map(decompressAiSessionLogRow); const log = await enrichWithDetails(logsRaw);