Refactor

DiscordChatExporter.Core/Discord/DiscordClient.cs

@@ -592,6 +592,33 @@ public class DiscordClient(
         }
     }
 
+    public async ValueTask<Message?> TryGetMessageAsync(
+        Snowflake channelId,
+        Snowflake messageId,
+        CancellationToken cancellationToken = default
+    )
+    {
+        // Use the regular message listing endpoint with the 'around' parameter instead of the
+        // dedicated single-message endpoint, because the latter is not accessible to user tokens.
+        var url = new UrlBuilder()
+            .SetPath($"channels/{channelId}/messages")
+            .SetQueryParameter("around", messageId.ToString())
+            .SetQueryParameter("limit", "1")
+            .Build();
+
+        // Can be null on channels that the user cannot access
+        var response = await TryGetJsonResponseAsync(url, cancellationToken);
+        if (response is null)
+            return null;
+
+        // The endpoint returns messages around the requested ID, so make sure to only return
+        // the message that exactly matches it (it may be absent if it has been deleted).
+        return response
+            .Value.EnumerateArray()
+            .Select(Message.Parse)
+            .FirstOrDefault(m => m.Id == messageId);
+    }
+
     private async ValueTask<Message?> TryGetFirstMessageAsync(
         Snowflake channelId,
         Snowflake? after = null,
@@ -634,59 +661,56 @@ public class DiscordClient(
         return response.Value.EnumerateArray().Select(Message.Parse).LastOrDefault();
     }
 
-    public async ValueTask<Message?> TryGetMessageAsync(
+    private async IAsyncEnumerable<Message> GetMessagesAsync(
         Snowflake channelId,
-        Snowflake messageId,
+        Snowflake? after,
-        CancellationToken cancellationToken = default
+        Snowflake? before,
+        IProgress<Percentage>? progress,
+        bool isReverse,
+        [EnumeratorCancellation] CancellationToken cancellationToken
     )
     {
-        // Use the regular message listing endpoint with the 'around' parameter instead of the
+        // To keep the understanding of message history independent of the fetching direction,
-        // dedicated single-message endpoint, because the latter is not accessible to user tokens.
+        // we'll refer to the two ends of the range as Alpha and Omega.
-        var url = new UrlBuilder()
+        // Depending on the direction, these are either 'before' and 'after', or 'after' and 'before'.
-            .SetPath($"channels/{channelId}/messages")
+        //
-            .SetQueryParameter("around", messageId.ToString())
+        // Chronological order:
-            .SetQueryParameter("limit", "1")
+        // <after> Alpha [----->----->----] Omega <before>
-            .Build();
+        // Reverse chronological order:
+        // <after> Omega [-----<-----<----] Alpha <before>
 
-        // Can be null on channels that the user cannot access
+        // Because Discord API doesn't allow us to provide both 'after' and 'before' parameters
-        var response = await TryGetJsonResponseAsync(url, cancellationToken);
+        // at the same time, we have to establish at least one end of the boundary manually.
-        if (response is null)
+        // To do that, we'll fetch the Omega message, which will be the terminal message in the range:
-            return null;
+        // last message in chronological order, or first message in reverse chronological order.
+        // This snapshotting also has the side benefit of allowing us to calculate progress by comparing
+        // the timestamps of the Alpha message, Omega message, and the message being currently processed.
+        var omegaMessage = !isReverse
+            ? await TryGetLastMessageAsync(channelId, before, cancellationToken)
+            : await TryGetFirstMessageAsync(channelId, after, cancellationToken);
 
-        // The endpoint returns messages around the requested ID, so make sure to only return
+        // If the Omega doesn't exist or falls outside of the range, then there are simply no messages
-        // the message that exactly matches it (it may be absent if it has been deleted).
+        // satisfying the specified range.
-        return response
+        if (
-            .Value.EnumerateArray()
+            omegaMessage is null
-            .Select(Message.Parse)
+            || (!isReverse && omegaMessage.Timestamp < after?.ToDate())
-            .FirstOrDefault(m => m.Id == messageId);
+            || (isReverse && omegaMessage.Timestamp > before?.ToDate())
-    }
+        )
-
+        {
-    public async IAsyncEnumerable<Message> GetMessagesAsync(
-        Snowflake channelId,
-        Snowflake? after = null,
-        Snowflake? before = null,
-        IProgress<Percentage>? progress = null,
-        [EnumeratorCancellation] CancellationToken cancellationToken = default
-    )
-    {
-        // Get the last message in the specified range, so we can later calculate the
-        // progress based on the difference between message timestamps.
-        // This also snapshots the boundaries, which means that messages posted after
-        // the export started will not appear in the output.
-        var lastMessage = await TryGetLastMessageAsync(channelId, before, cancellationToken);
-        if (lastMessage is null || lastMessage.Timestamp < after?.ToDate())
             yield break;
+        }
 
-        // Keep track of the first message in range in order to calculate the progress
+        // Persist the Alpha message as soon as we fetch the initial batch of messages.
-        var firstMessage = default(Message);
+        // This is only used for calculating progress.
+        var alphaMessage = default(Message);
 
-        var currentAfter = after ?? Snowflake.Zero;
+        var currentBoundary = !isReverse ? after ?? Snowflake.Zero : before;
         while (true)
         {
             var url = new UrlBuilder()
                 .SetPath($"channels/{channelId}/messages")
                 .SetQueryParameter("limit", "100")
-                .SetQueryParameter("after", currentAfter.ToString())
+                .SetQueryParameter(!isReverse ? "after" : "before", currentBoundary?.ToString())
                 .Build();
 
             var response = await GetJsonResponseAsync(url, cancellationToken);
@@ -694,8 +718,8 @@ public class DiscordClient(
             var messages = response
                 .EnumerateArray()
                 .Select(Message.Parse)
-                // Messages are returned from newest to oldest, so we need to reverse them
+                // Messages in batches are always returned from newest to oldest, so reverse if needed
-                .Reverse()
+                .Pipe(messages => isReverse ? messages : messages.Reverse())
                 .ToArray();
 
             // Break if there are no messages (can happen if messages are deleted during execution)
@@ -710,24 +734,31 @@ public class DiscordClient(
 
             foreach (var message in messages)
             {
-                firstMessage ??= message;
+                // Ensure that we're still in range by checking against the Omega
-
+                if (!isReverse ? message.Id > omegaMessage.Id : message.Id < omegaMessage.Id)
-                // Ensure that the messages are in range
+                {
-                if (message.Timestamp > lastMessage.Timestamp)
                     yield break;
+                }
+
+                alphaMessage ??= message;
 
                 // Report progress based on timestamps
                 if (progress is not null)
                 {
-                    var exportedDuration = (message.Timestamp - firstMessage.Timestamp).Duration();
+                    var fetchedDuration = isReverse
-                    var totalDuration = (lastMessage.Timestamp - firstMessage.Timestamp).Duration();
+                        ? alphaMessage.Timestamp - message.Timestamp
+                        : message.Timestamp - alphaMessage.Timestamp;
+
+                    var totalDuration = isReverse
+                        ? alphaMessage.Timestamp - omegaMessage.Timestamp
+                        : omegaMessage.Timestamp - alphaMessage.Timestamp;
 
                     progress.Report(
                         Percentage.FromFraction(
                             // Avoid division by zero if all messages have the exact same timestamp
                             // (which happens when there's only one message in the channel)
                             totalDuration > TimeSpan.Zero
-                                ? exportedDuration / totalDuration
+                                ? fetchedDuration / totalDuration
                                 : 1
                         )
                     );
@@ -747,9 +778,34 @@ public class DiscordClient(
                         : null;
 
                 yield return actualMessage ?? message;
-
-                currentAfter = message.Id;
             }
+
+            // The new boundary is always determined by the last message in the batch,
+            // because we order the messages based on fetching direction.
+            currentBoundary = messages.Last().Id;
+        }
+    }
+
+    public async IAsyncEnumerable<Message> GetMessagesAsync(
+        Snowflake channelId,
+        Snowflake? after = null,
+        Snowflake? before = null,
+        IProgress<Percentage>? progress = null,
+        [EnumeratorCancellation] CancellationToken cancellationToken = default
+    )
+    {
+        await foreach (
+            var message in GetMessagesAsync(
+                channelId,
+                after,
+                before,
+                progress,
+                false,
+                cancellationToken
+            )
+        )
+        {
+            yield return message;
         }
     }
 
@@ -761,81 +817,18 @@ public class DiscordClient(
         [EnumeratorCancellation] CancellationToken cancellationToken = default
     )
     {
-        // Get the first message in the specified range, so we can later calculate the
+        await foreach (
-        // progress based on the difference between message timestamps.
+            var message in GetMessagesAsync(
-        // Snapshotting is not necessary here because new messages can't appear in the past.
+                channelId,
-        var firstMessage = await TryGetFirstMessageAsync(channelId, after, cancellationToken);
+                after,
-        if (firstMessage is null || firstMessage.Timestamp > before?.ToDate())
+                before,
-            yield break;
+                progress,
-
+                true,
-        // Keep track of the last message in range in order to calculate the progress
+                cancellationToken
-        var lastMessage = default(Message);
+            )
-
+        )
-        var currentBefore = before;
-        while (true)
         {
-            var url = new UrlBuilder()
+            yield return message;
-                .SetPath($"channels/{channelId}/messages")
-                .SetQueryParameter("limit", "100")
-                .SetQueryParameter("before", currentBefore?.ToString())
-                .Build();
-
-            var response = await GetJsonResponseAsync(url, cancellationToken);
-
-            var messages = response.EnumerateArray().Select(Message.Parse).ToArray();
-
-            // Break if there are no messages (can happen if messages are deleted during execution)
-            if (!messages.Any())
-                yield break;
-
-            // If all messages are empty, make sure that it's not because the bot account doesn't
-            // have the MESSAGE_CONTENT intent enabled.
-            // https://github.com/Tyrrrz/DiscordChatExporter/issues/1106#issuecomment-1741548959
-            if (messages.All(m => m.IsEmpty))
-                await EnsureMessageContentIntentAsync(cancellationToken);
-
-            foreach (var message in messages)
-            {
-                // Ensure that the messages are in range
-                if (message.Timestamp <= after?.ToDate())
-                    yield break;
-
-                lastMessage ??= message;
-
-                // Report progress based on timestamps
-                if (progress is not null)
-                {
-                    var exportedDuration = (lastMessage.Timestamp - message.Timestamp).Duration();
-                    var totalDuration = (lastMessage.Timestamp - firstMessage.Timestamp).Duration();
-
-                    progress.Report(
-                        Percentage.FromFraction(
-                            // Avoid division by zero if all messages have the exact same timestamp
-                            // (which happens when there's only one message in the channel)
-                            totalDuration > TimeSpan.Zero
-                                ? exportedDuration / totalDuration
-                                : 1
-                        )
-                    );
-                }
-
-                // Some messages, for example thread starter messages, are returned by the API as content-less references.
-                // Try to resolve them to the actual message so that they appear as they do in the Discord client.
-                var actualMessage =
-                    message.Kind == MessageKind.ThreadStarterMessage
-                    && message.Reference?.ChannelId is { } referencedChannelId
-                    && message.Reference?.MessageId is { } referencedMessageId
-                        ? await TryGetMessageAsync(
-                            referencedChannelId,
-                            referencedMessageId,
-                            cancellationToken
-                        )
-                        : null;
-
-                yield return actualMessage ?? message;
-            }
-
-            currentBefore = messages.Last().Id;
         }
     }
 

DiscordChatExporter.Core/Utils/UrlBuilder.cs

@@ -20,9 +20,9 @@ public class UrlBuilder
         return this;
     }
 
-    public UrlBuilder SetQueryParameter(string key, string? value, bool ignoreUnsetValue = true)
+    public UrlBuilder SetQueryParameter(string key, string? value, bool ignoreIfEmptyValue = true)
     {
-        if (ignoreUnsetValue && string.IsNullOrWhiteSpace(value))
+        if (ignoreIfEmptyValue && string.IsNullOrWhiteSpace(value))
             return this;
 
         var keyEncoded = Uri.EscapeDataString(key);