Merge pull request #3644 from fosrl/dev

1.22.0-s.2

server/lib/textCompression.ts

@@ -0,0 +1,15 @@
+import { gzipSync, gunzipSync } from "zlib";
+
+/**
+ * Gzip a string and return it as base64 so it can be stored in a TEXT column.
+ */
+export function compressText(value: string): string {
+    return gzipSync(Buffer.from(value, "utf8")).toString("base64");
+}
+
+/**
+ * Reverse of compressText - base64-decode and gunzip back to the original string.
+ */
+export function decompressText(value: string): string {
+    return gunzipSync(Buffer.from(value, "base64")).toString("utf8");
+}

server/private/lib/logStreaming/LogStreamingManager.ts

@@ -25,6 +25,7 @@ import {
 import logger from "@server/logger";
 import { and, eq, gt, desc, max, sql } from "drizzle-orm";
 import { decrypt } from "@server/lib/crypto";
+import { decompressText } from "@server/lib/textCompression";
 import config from "@server/lib/config";
 import {
     LogType,
@@ -680,8 +681,8 @@ export class LogStreamingManager {
                     Record<string, unknown> & { id: number }
                 >;
 
-            case "aiSession":
+            case "aiSession": {
-                return (await logsDb
+                const rows = (await logsDb
                     .select()
                     .from(aiSessionLog)
                     .where(
@@ -694,6 +695,33 @@ export class LogStreamingManager {
                     .limit(limit)) as Array<
                     Record<string, unknown> & { id: number }
                 >;
+
+                const compressedFields = [
+                    "requestBody",
+                    "responseBody",
+                    "normalizedRequest",
+                    "normalizedResponse"
+                ] as const;
+
+                for (const row of rows) {
+                    for (const field of compressedFields) {
+                        const value = row[field];
+                        if (typeof value !== "string") {
+                            continue;
+                        }
+                        try {
+                            row[field] = decompressText(value);
+                        } catch (error) {
+                            logger.error(
+                                `Failed to decompress AI session log field ${field}`,
+                                { error }
+                            );
+                        }
+                    }
+                }
+
+                return rows;
+            }
         }
     }
 

server/private/routers/aiGateway/logAiSession.ts

@@ -18,6 +18,7 @@ import { and, eq, lt } from "drizzle-orm";
 import cache from "#private/lib/cache";
 import { calculateCutoffTimestamp } from "@server/lib/cleanupLogs";
 import { sanitizeString } from "@server/lib/sanitize";
+import { compressText } from "@server/lib/textCompression";
 import type { AiCapability } from "@server/lib/aiCapabilities";
 import {
     normalizeAiRequest,
@@ -151,7 +152,7 @@ async function getRetentionDays(orgId: string): Promise<number> {
 }
 
 export async function cleanUpOldLogs(orgId: string, retentionDays: number) {
-    const cutoffTimestamp = calculateCutoffTimestamp(retentionDays) * 1000;
+    const cutoffTimestamp = calculateCutoffTimestamp(retentionDays);
 
     try {
         await logsDb
@@ -255,13 +256,19 @@ export function logAiSession(data: {
                 ),
                 requestedModel: sanitizeString(data.requestedModel),
                 isStream: data.isStream,
-                requestBody: sanitizeString(requestBodyText.value),
+                requestBody: compressText(
-                responseBody: sanitizeString(responseBodyText.value),
+                    sanitizeString(requestBodyText.value)
+                ),
+                responseBody: compressText(
+                    sanitizeString(responseBodyText.value)
+                ),
                 normalizedRequest: normalizedRequestText
-                    ? sanitizeString(normalizedRequestText.value)
+                    ? compressText(sanitizeString(normalizedRequestText.value))
                     : undefined,
                 normalizedResponse: normalizedResponseText
-                    ? sanitizeString(normalizedResponseText.value)
+                    ? compressText(
+                          sanitizeString(normalizedResponseText.value)
+                      )
                     : undefined,
                 truncated:
                     requestBodyText.truncated ||

server/routers/auditLogs/exportAiSessionLog.ts

@@ -11,7 +11,8 @@ import {
     queryAiSessionLogsQuery,
     queryAiSessionLogsParams,
     queryAiSession,
-    countAiSessionQuery
+    countAiSessionQuery,
+    decompressAiSessionLogRow
 } from "./queryAiSessionLog";
 import { generateCSV } from "./generateCSV";
 
@@ -87,7 +88,9 @@ export async function exportAiSessionLogs(
 
         const baseQuery = queryAiSession(data);
 
-        const log = await baseQuery.limit(MAX_EXPORT_LIMIT);
+        const log = (await baseQuery.limit(MAX_EXPORT_LIMIT)).map(
+            decompressAiSessionLogRow
+        );
 
         const csvData = generateCSV(log);
 

server/routers/auditLogs/queryAiSessionLog.ts

@@ -24,6 +24,7 @@ import { AI_CAPABILITIES } from "@server/lib/aiCapabilities";
 import response from "@server/lib/response";
 import logger from "@server/logger";
 import { getSevenDaysAgo } from "@app/lib/getSevenDaysAgo";
+import { decompressText } from "@server/lib/textCompression";
 
 export const queryAiSessionLogsQuery = z.strictObject({
     // iso string just validate its a parseable date
@@ -166,6 +167,35 @@ export function queryAiSession(data: Q) {
         .orderBy(desc(aiSessionLog.createdAt));
 }
 
+function decompressField(value: string | null): string | null {
+    if (value == null) {
+        return value;
+    }
+    try {
+        return decompressText(value);
+    } catch (error) {
+        logger.error("Failed to decompress AI session log field", { error });
+        return value;
+    }
+}
+
+export function decompressAiSessionLogRow<
+    T extends {
+        requestBody: string | null;
+        responseBody: string | null;
+        normalizedRequest: string | null;
+        normalizedResponse: string | null;
+    }
+>(row: T): T {
+    return {
+        ...row,
+        requestBody: decompressField(row.requestBody),
+        responseBody: decompressField(row.responseBody),
+        normalizedRequest: decompressField(row.normalizedRequest),
+        normalizedResponse: decompressField(row.normalizedResponse)
+    };
+}
+
 async function enrichWithDetails(
     logs: Awaited<ReturnType<typeof queryAiSession>>
 ) {
@@ -620,7 +650,9 @@ export async function queryAiSessionLogs(
 
         const baseQuery = queryAiSession(data);
 
-        const logsRaw = await baseQuery.limit(data.limit).offset(data.offset);
+        const logsRaw = (
+            await baseQuery.limit(data.limit).offset(data.offset)
+        ).map(decompressAiSessionLogRow);
 
         const log = await enrichWithDetails(logsRaw);
 

server/setup/scriptsPg/1.22.0.ts

@@ -223,7 +223,7 @@ export default async function migration() {
             sql`ALTER TABLE "subscriptions" ADD COLUMN "override" boolean DEFAULT false;`
         );
         await db.execute(
-            sql`ALTER TABLE "orgs" ADD COLUMN "settingsLogRetentionDaysAISessions" integer DEFAULT 7 NOT NULL;`
+            sql`ALTER TABLE "orgs" ADD COLUMN "settingsLogRetentionDaysAISessions" integer DEFAULT 0 NOT NULL;`
         );
         await db.execute(
             sql`ALTER TABLE "siteResources" ADD COLUMN "requiresExitNodeConnection" boolean DEFAULT false NOT NULL;`

server/setup/scriptsSqlite/1.22.0.ts

@@ -397,7 +397,7 @@ export default async function migration() {
                 `ALTER TABLE 'clients' ADD 'exitNodeSubnet' text;`
             ).run();
             db.prepare(
-                `ALTER TABLE 'orgs' ADD 'settingsLogRetentionDaysAISessions' integer DEFAULT 7 NOT NULL;`
+                `ALTER TABLE 'orgs' ADD 'settingsLogRetentionDaysAISessions' integer DEFAULT 0 NOT NULL;`
             ).run();
             db.prepare(
                 `ALTER TABLE 'siteResources' ADD 'requiresExitNodeConnection' integer DEFAULT false NOT NULL;`