Update version

.github/workflows/main.yml

@@ -148,6 +148,6 @@ jobs:
           body: |
             {
               "avatar_url": "https://raw.githubusercontent.com/${{ github.event.repository.full_name }}/${{ github.ref_name }}/favicon.png",
-              "content": "**${{ github.event.repository.name }}** new version released!\nVersion: `${{ github.ref_name }}`\nChangelog: <${{ github.event.repository.html_url }}/blob/${{ github.ref_name }}/Changelog.md>"
+              "content": "**${{ github.event.repository.name }}** new version released!\nVersion: `${{ github.ref_name }}`\nDownload: <${{ github.event.repository.html_url }}/releases/tag/${{ github.ref_name }}>\nChangelog: <${{ github.event.repository.html_url }}/blob/${{ github.ref_name }}/Changelog.md>"
             }
           retry-count: 5

Changelog.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v2.42 (12-Oct-2023)
+
+- General changes:
+  - Fixed an issue where the "reuse assets" option did not work as intended due to Discord recently introducing signatures in the CDN URLs. (Thanks [@slatinsky](https://github.com/slatinsky))
+- CLI changes:
+  - Added progress reporting for the "fetching channels" stage of the `exportguild` and `exportall` commands. (Thanks [@slatinsky](https://github.com/slatinsky))
+
 ## v2.41.2 (05-Oct-2023)
 
 - General changes:

Directory.Build.props

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net7.0</TargetFramework>
-    <Version>2.41.2</Version>
+    <Version>2.42</Version>
     <Company>Tyrrrz</Company>
     <Copyright>Copyright (c) Oleksii Holub</Copyright>
     <LangVersion>preview</LangVersion>

DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs

@@ -186,7 +186,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
                 // https://github.com/Tyrrrz/DiscordChatExporter/issues/1124
                 ParallelLimit > 1
             )
-            .StartAsync(async progressContext =>
+            .StartAsync(async ctx =>
             {
                 await Parallel.ForEachAsync(
                     channels,
@@ -199,7 +199,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
                     {
                         try
                         {
-                            await progressContext.StartTaskAsync(
+                            await ctx.StartTaskAsync(
                                 channel.GetHierarchicalName(),
                                 async progress =>
                                 {
@@ -257,7 +257,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
             using (console.WithForegroundColor(ConsoleColor.Red))
             {
                 await console.Error.WriteLineAsync(
-                    $"Failed to export {errorsByChannel.Count} channel(s):"
+                    $"Failed to export {errorsByChannel.Count} the following channel(s):"
                 );
             }
 

DiscordChatExporter.Cli/Commands/ExportAllCommand.cs

@@ -1,17 +1,17 @@
+using System;
 using System.Collections.Generic;
-using System.IO.Compression;
+using System.Linq;
-using System.Text.Json;
 using System.Threading.Tasks;
 using CliFx.Attributes;
-using CliFx.Exceptions;
 using CliFx.Infrastructure;
 using DiscordChatExporter.Cli.Commands.Base;
 using DiscordChatExporter.Cli.Commands.Converters;
 using DiscordChatExporter.Cli.Commands.Shared;
-using DiscordChatExporter.Core.Discord;
+using DiscordChatExporter.Cli.Utils.Extensions;
 using DiscordChatExporter.Core.Discord.Data;
+using DiscordChatExporter.Core.Discord.Dump;
 using DiscordChatExporter.Core.Exceptions;
-using JsonExtensions.Reading;
+using Spectre.Console;
 
 namespace DiscordChatExporter.Cli.Commands;
 
@@ -51,39 +51,78 @@ public class ExportAllCommand : ExportCommandBase
         // Pull from the API
         if (string.IsNullOrWhiteSpace(DataPackageFilePath))
         {
-            await console.Output.WriteLineAsync("Fetching channels...");
-
             await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
             {
                 // Regular channels
-                await foreach (
+                await console.Output.WriteLineAsync(
-                    var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken)
+                    $"Fetching channels for guild '{guild.Name}'..."
-                )
+                );
-                {
-                    if (channel.IsCategory)
-                        continue;
 
-                    if (!IncludeVoiceChannels && channel.IsVoice)
+                var fetchedChannelsCount = 0;
-                        continue;
+                await console
+                    .CreateStatusTicker()
+                    .StartAsync(
+                        "...",
+                        async ctx =>
+                        {
+                            await foreach (
+                                var channel in Discord.GetGuildChannelsAsync(
+                                    guild.Id,
+                                    cancellationToken
+                                )
+                            )
+                            {
+                                if (channel.IsCategory)
+                                    continue;
 
-                    channels.Add(channel);
+                                if (!IncludeVoiceChannels && channel.IsVoice)
-                }
+                                    continue;
+
+                                channels.Add(channel);
+
+                                ctx.Status($"Fetched '{channel.GetHierarchicalName()}'.");
+                                fetchedChannelsCount++;
+                            }
+                        }
+                    );
+
+                await console.Output.WriteLineAsync($"Fetched {fetchedChannelsCount} channel(s).");
 
                 // Threads
                 if (ThreadInclusionMode != ThreadInclusionMode.None)
                 {
-                    await foreach (
+                    await console.Output.WriteLineAsync(
-                        var thread in Discord.GetGuildThreadsAsync(
+                        $"Fetching threads for guild '{guild.Name}'..."
-                            guild.Id,
+                    );
-                            ThreadInclusionMode == ThreadInclusionMode.All,
+
-                            Before,
+                    var fetchedThreadsCount = 0;
-                            After,
+                    await console
-                            cancellationToken
+                        .CreateStatusTicker()
-                        )
+                        .StartAsync(
-                    )
+                            "...",
-                    {
+                            async ctx =>
-                        channels.Add(thread);
+                            {
-                    }
+                                await foreach (
+                                    var thread in Discord.GetGuildThreadsAsync(
+                                        guild.Id,
+                                        ThreadInclusionMode == ThreadInclusionMode.All,
+                                        Before,
+                                        After,
+                                        cancellationToken
+                                    )
+                                )
+                                {
+                                    channels.Add(thread);
+
+                                    ctx.Status($"Fetched '{thread.GetHierarchicalName()}'.");
+                                    fetchedThreadsCount++;
+                                }
+                            }
+                        );
+
+                    await console.Output.WriteLineAsync(
+                        $"Fetched {fetchedThreadsCount} thread(s)."
+                    );
                 }
             }
         }
@@ -91,39 +130,55 @@ public class ExportAllCommand : ExportCommandBase
         else
         {
             await console.Output.WriteLineAsync("Extracting channels...");
-            using var archive = ZipFile.OpenRead(DataPackageFilePath);
 
-            var entry = archive.GetEntry("messages/index.json");
+            var dump = await DataDump.LoadAsync(DataPackageFilePath, cancellationToken);
-            if (entry is null)
+            var inaccessibleChannels = new List<DataDumpChannel>();
-                throw new CommandException("Could not find channel index inside the data package.");
 
-            await using var stream = entry.Open();
+            await console
-            using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken);
+                .CreateStatusTicker()
+                .StartAsync(
+                    "...",
+                    async ctx =>
+                    {
+                        foreach (var dumpChannel in dump.Channels)
+                        {
+                            ctx.Status($"Fetching '{dumpChannel.Name}' ({dumpChannel.Id})...");
 
-            foreach (var property in document.RootElement.EnumerateObjectOrEmpty())
+                            try
-            {
+                            {
-                var channelId = Snowflake.Parse(property.Name);
+                                var channel = await Discord.GetChannelAsync(
-                var channelName = property.Value.GetString();
+                                    dumpChannel.Id,
+                                    cancellationToken
+                                );
 
-                // Null items refer to deleted channels
+                                channels.Add(channel);
-                if (channelName is null)
+                            }
-                    continue;
+                            catch (DiscordChatExporterException)
-
+                            {
-                await console.Output.WriteLineAsync(
+                                inaccessibleChannels.Add(dumpChannel);
-                    $"Fetching channel '{channelName}' ({channelId})..."
+                            }
+                        }
+                    }
                 );
 
-                try
+            await console.Output.WriteLineAsync($"Fetched {channels} channel(s).");
-                {
+
-                    var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
+            // Print inaccessible channels
-                    channels.Add(channel);
+            if (inaccessibleChannels.Any())
-                }
+            {
-                catch (DiscordChatExporterException)
+                await console.Output.WriteLineAsync();
+
+                using (console.WithForegroundColor(ConsoleColor.Red))
                 {
                     await console.Error.WriteLineAsync(
-                        $"Channel '{channelName}' ({channelId}) is inaccessible."
+                        "Failed to access the following channel(s):"
                     );
                 }
+
+                foreach (var dumpChannel in inaccessibleChannels)
+                    await console.Error.WriteLineAsync($"{dumpChannel.Name} ({dumpChannel.Id})");
+
+                await console.Error.WriteLineAsync();
             }
         }
 

DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs

@@ -1,12 +1,15 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using CliFx.Attributes;
 using CliFx.Infrastructure;
 using DiscordChatExporter.Cli.Commands.Base;
 using DiscordChatExporter.Cli.Commands.Converters;
 using DiscordChatExporter.Cli.Commands.Shared;
+using DiscordChatExporter.Cli.Utils.Extensions;
 using DiscordChatExporter.Core.Discord;
 using DiscordChatExporter.Core.Discord.Data;
+using Spectre.Console;
 
 namespace DiscordChatExporter.Cli.Commands;
 
@@ -33,35 +36,67 @@ public class ExportGuildCommand : ExportCommandBase
         var cancellationToken = console.RegisterCancellationHandler();
         var channels = new List<Channel>();
 
+        // Regular channels
         await console.Output.WriteLineAsync("Fetching channels...");
 
-        // Regular channels
+        var fetchedChannelsCount = 0;
-        await foreach (var channel in Discord.GetGuildChannelsAsync(GuildId, cancellationToken))
+        await console
-        {
+            .CreateStatusTicker()
-            if (channel.IsCategory)
+            .StartAsync(
-                continue;
+                "...",
+                async ctx =>
+                {
+                    await foreach (
+                        var channel in Discord.GetGuildChannelsAsync(GuildId, cancellationToken)
+                    )
+                    {
+                        if (channel.IsCategory)
+                            continue;
 
-            if (!IncludeVoiceChannels && channel.IsVoice)
+                        if (!IncludeVoiceChannels && channel.IsVoice)
-                continue;
+                            continue;
 
-            channels.Add(channel);
+                        channels.Add(channel);
-        }
+
+                        ctx.Status($"Fetched '{channel.GetHierarchicalName()}'.");
+                        fetchedChannelsCount++;
+                    }
+                }
+            );
+
+        await console.Output.WriteLineAsync($"Fetched {fetchedChannelsCount} channel(s).");
 
         // Threads
         if (ThreadInclusionMode != ThreadInclusionMode.None)
         {
-            await foreach (
+            await console.Output.WriteLineAsync("Fetching threads...");
-                var thread in Discord.GetGuildThreadsAsync(
+
-                    GuildId,
+            var fetchedThreadsCount = 0;
-                    ThreadInclusionMode == ThreadInclusionMode.All,
+            await console
-                    Before,
+                .CreateStatusTicker()
-                    After,
+                .StartAsync(
-                    cancellationToken
+                    "...",
-                )
+                    async ctx =>
-            )
+                    {
-            {
+                        await foreach (
-                channels.Add(thread);
+                            var thread in Discord.GetGuildThreadsAsync(
-            }
+                                GuildId,
+                                ThreadInclusionMode == ThreadInclusionMode.All,
+                                Before,
+                                After,
+                                cancellationToken
+                            )
+                        )
+                        {
+                            channels.Add(thread);
+
+                            ctx.Status($"Fetched '{thread.GetHierarchicalName()}'.");
+                            fetchedThreadsCount++;
+                        }
+                    }
+                );
+
+            await console.Output.WriteLineAsync($"Fetched {fetchedThreadsCount} thread(s).");
         }
 
         await ExportAsync(console, channels);

DiscordChatExporter.Cli/Utils/Extensions/ConsoleExtensions.cs

@@ -17,6 +17,9 @@ internal static class ConsoleExtensions
             }
         );
 
+    public static Status CreateStatusTicker(this IConsole console) =>
+        console.CreateAnsiConsole().Status().AutoRefresh(true);
+
     public static Progress CreateProgressTicker(this IConsole console) =>
         console
             .CreateAnsiConsole()
@@ -31,16 +34,16 @@ internal static class ConsoleExtensions
             );
 
     public static async ValueTask StartTaskAsync(
-        this ProgressContext progressContext,
+        this ProgressContext context,
         string description,
         Func<ProgressTask, ValueTask> performOperationAsync
     )
     {
         // Description cannot be empty
         // https://github.com/Tyrrrz/DiscordChatExporter/issues/1133
-        var actualDescription = !string.IsNullOrWhiteSpace(description) ? description : "?";
+        var actualDescription = !string.IsNullOrWhiteSpace(description) ? description : "...";
 
-        var progressTask = progressContext.AddTask(
+        var progressTask = context.AddTask(
             // Don't recognize random square brackets as style tags
             Markup.Escape(actualDescription),
             new ProgressTaskSettings { MaxValue = 1 }

DiscordChatExporter.Core/Discord/Dump/DataDump.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.IO.Compression;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using JsonExtensions.Reading;
+
+namespace DiscordChatExporter.Core.Discord.Dump;
+
+public partial class DataDump
+{
+    public IReadOnlyList<DataDumpChannel> Channels { get; }
+
+    public DataDump(IReadOnlyList<DataDumpChannel> channels) => Channels = channels;
+}
+
+public partial class DataDump
+{
+    public static DataDump Parse(JsonElement json)
+    {
+        var channels = new List<DataDumpChannel>();
+
+        foreach (var property in json.EnumerateObjectOrEmpty())
+        {
+            var channelId = Snowflake.Parse(property.Name);
+            var channelName = property.Value.GetString();
+
+            // Null items refer to deleted channels
+            if (channelName is null)
+                continue;
+
+            var channel = new DataDumpChannel(channelId, channelName);
+            channels.Add(channel);
+        }
+
+        return new DataDump(channels);
+    }
+
+    public static async ValueTask<DataDump> LoadAsync(
+        string zipFilePath,
+        CancellationToken cancellationToken = default
+    )
+    {
+        using var archive = ZipFile.OpenRead(zipFilePath);
+
+        var entry = archive.GetEntry("messages/index.json");
+        if (entry is null)
+        {
+            throw new InvalidOperationException(
+                "Could not find the channel index inside the data package."
+            );
+        }
+
+        await using var stream = entry.Open();
+        using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken);
+
+        return Parse(document.RootElement);
+    }
+}

DiscordChatExporter.Core/Discord/Dump/DataDumpChannel.cs

@@ -0,0 +1,3 @@
+namespace DiscordChatExporter.Core.Discord.Dump;
+
+public record DataDumpChannel(Snowflake Id, string Name);

DiscordChatExporter.Core/Exporting/ExportAssetDownloader.cs

@@ -7,6 +7,7 @@ using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Web;
 using AsyncKeyedLock;
 using DiscordChatExporter.Core.Utils;
 using DiscordChatExporter.Core.Utils.Extensions;
@@ -101,12 +102,29 @@ internal partial class ExportAssetDownloader
 
 internal partial class ExportAssetDownloader
 {
-    private static string GetUrlHash(string url) =>
+    private static string GetUrlHash(string url)
-        SHA256
+    {
-            .HashData(Encoding.UTF8.GetBytes(url))
+        // Remove signature parameters from Discord CDN URLs to normalize them
+        static string NormalizeUrl(string url)
+        {
+            var uri = new Uri(url);
+            if (!string.Equals(uri.Host, "cdn.discordapp.com", StringComparison.OrdinalIgnoreCase))
+                return url;
+
+            var query = HttpUtility.ParseQueryString(uri.Query);
+            query.Remove("ex");
+            query.Remove("is");
+            query.Remove("hm");
+
+            return uri.GetLeftPart(UriPartial.Path) + query;
+        }
+
+        return SHA256
+            .HashData(Encoding.UTF8.GetBytes(NormalizeUrl(url)))
             .ToHex()
             // 5 chars ought to be enough for anybody
             .Truncate(5);
+    }
 
     private static string GetFileNameFromUrl(string url)
     {