Add parallel exporting

Closes #264

DiscordChatExporter.Cli/Commands/ExportCommandBase.cs

@@ -53,6 +53,7 @@ namespace DiscordChatExporter.Cli.Commands
                 After, Before, progress);
 
             console.Output.WriteLine();
+            console.Output.WriteLine("Done.");
         }
 
         protected async ValueTask ExportAsync(IConsole console, Channel channel)

DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs

@@ -1,16 +1,13 @@
 using System.Linq;
-using System.Net;
 using System.Threading.Tasks;
 using CliFx;
 using CliFx.Attributes;
-using DiscordChatExporter.Core.Models.Exceptions;
 using DiscordChatExporter.Core.Services;
-using DiscordChatExporter.Core.Services.Exceptions;
 
 namespace DiscordChatExporter.Cli.Commands
 {
     [Command("exportdm", Description = "Export all direct message channels.")]
-    public class ExportDirectMessagesCommand : ExportCommandBase
+    public class ExportDirectMessagesCommand : ExportMultipleCommandBase
     {
         public ExportDirectMessagesCommand(SettingsService settingsService, DataService dataService, ExportService exportService)
             : base(settingsService, dataService, exportService)
@@ -19,32 +16,10 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get channels
-            var channels = await DataService.GetDirectMessageChannelsAsync(Token);
+            var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token);
+            var channels = directMessageChannels.OrderBy(c => c.Name).ToArray();
 
-            // Order channels
-            channels = channels.OrderBy(c => c.Name).ToArray();
-
-            // Loop through channels
-            foreach (var channel in channels)
-            {
-                try
-                {
-                    await ExportAsync(console, channel);
-                }
-                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
-                {
-                    console.Error.WriteLine("You don't have access to this channel.");
-                }
-                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
-                {
-                    console.Error.WriteLine("This channel doesn't exist.");
-                }
-                catch (DomainException ex)
-                {
-                    console.Error.WriteLine(ex.Message);
-                }
-            }
+            await ExportMultipleAsync(console, channels);
         }
     }
 }

DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs

@@ -1,17 +1,14 @@
 using System.Linq;
-using System.Net;
 using System.Threading.Tasks;
 using CliFx;
 using CliFx.Attributes;
 using DiscordChatExporter.Core.Models;
-using DiscordChatExporter.Core.Models.Exceptions;
 using DiscordChatExporter.Core.Services;
-using DiscordChatExporter.Core.Services.Exceptions;
 
 namespace DiscordChatExporter.Cli.Commands
 {
     [Command("exportguild", Description = "Export all channels within specified guild.")]
-    public class ExportGuildCommand : ExportCommandBase
+    public class ExportGuildCommand : ExportMultipleCommandBase
     {
         [CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")]
         public string GuildId { get; set; } = "";
@@ -23,32 +20,14 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get channels
-            var channels = await DataService.GetGuildChannelsAsync(Token, GuildId);
+            var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId);
 
-            // Filter and order channels
-            channels = channels.Where(c => c.Type.IsExportable()).OrderBy(c => c.Name).ToArray();
+            var channels = guildChannels
+                .Where(c => c.Type.IsExportable())
+                .OrderBy(c => c.Name)
+                .ToArray();
 
-            // Loop through channels
-            foreach (var channel in channels)
-            {
-                try
-                {
-                    await ExportAsync(console, channel);
-                }
-                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
-                {
-                    console.Error.WriteLine("You don't have access to this channel.");
-                }
-                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
-                {
-                    console.Error.WriteLine("This channel doesn't exist.");
-                }
-                catch (DomainException ex)
-                {
-                    console.Error.WriteLine(ex.Message);
-                }
-            }
+            await ExportMultipleAsync(console, channels);
         }
     }
 }

DiscordChatExporter.Cli/Commands/ExportMultipleCommandBase.cs

@@ -0,0 +1,92 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using CliFx;
+using CliFx.Attributes;
+using CliFx.Utilities;
+using DiscordChatExporter.Core.Models;
+using DiscordChatExporter.Core.Models.Exceptions;
+using DiscordChatExporter.Core.Services;
+using DiscordChatExporter.Core.Services.Exceptions;
+using Gress;
+using Tyrrrz.Extensions;
+
+namespace DiscordChatExporter.Cli.Commands
+{
+    public abstract class ExportMultipleCommandBase : ExportCommandBase
+    {
+        [CommandOption("parallel", Description = "Export this number of separate channels in parallel.")]
+        public int ParallelLimit { get; set; } = 1;
+
+        protected ExportMultipleCommandBase(SettingsService settingsService, DataService dataService, ExportService exportService)
+            : base(settingsService, dataService, exportService)
+        {
+        }
+
+        protected async ValueTask ExportMultipleAsync(IConsole console, IReadOnlyList<Channel> channels)
+        {
+            // This uses a separate route from ExportCommandBase because the progress ticker is not thread-safe
+            // Ugly code ahead. Will need to refactor.
+
+            if (!string.IsNullOrWhiteSpace(DateFormat))
+                SettingsService.DateFormat = DateFormat;
+
+            // Progress
+            console.Output.Write($"Exporting {channels.Count} channels... ");
+            var ticker = console.CreateProgressTicker();
+
+            // TODO: refactor this after improving Gress
+            var progressManager = new ProgressManager();
+            progressManager.PropertyChanged += (sender, args) => ticker.Report(progressManager.Progress);
+
+            var operations = progressManager.CreateOperations(channels.Count);
+
+            // Export channels
+            using var semaphore = new SemaphoreSlim(ParallelLimit.ClampMin(1));
+
+            var errors = new List<string>();
+
+            await Task.WhenAll(channels.Select(async (channel, i) =>
+            {
+                var operation = operations[i];
+                await semaphore.WaitAsync();
+
+                var guild = await DataService.GetGuildAsync(Token, channel.GuildId);
+
+                try
+                {
+                    await ExportService.ExportChatLogAsync(Token, guild, channel,
+                        OutputPath, ExportFormat, PartitionLimit,
+                        After, Before, operation);
+                }
+                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
+                {
+                    errors.Add("You don't have access to this channel.");
+                }
+                catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+                {
+                    errors.Add("This channel doesn't exist.");
+                }
+                catch (DomainException ex)
+                {
+                    errors.Add(ex.Message);
+                }
+                finally
+                {
+                    semaphore.Release();
+                    operation.Dispose();
+                }
+            }));
+
+            ticker.Report(1);
+            console.Output.WriteLine();
+
+            foreach (var error in errors)
+                console.Error.WriteLine(error);
+
+            console.Output.WriteLine("Done.");
+        }
+    }
+}

DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs

@@ -20,13 +20,13 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get channels
-            var channels = await DataService.GetGuildChannelsAsync(Token, GuildId);
+            var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId);
 
-            // Filter and order channels
-            channels = channels.Where(c => c.Type.IsExportable()).OrderBy(c => c.Name).ToArray();
+            var channels = guildChannels
+                .Where(c => c.Type.IsExportable())
+                .OrderBy(c => c.Name)
+                .ToArray();
 
-            // Print result
             foreach (var channel in channels)
                 console.Output.WriteLine($"{channel.Id} | {channel.Name}");
         }

DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs

@@ -16,13 +16,9 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get channels
-            var channels = await DataService.GetDirectMessageChannelsAsync(Token);
+            var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token);
+            var channels = directMessageChannels.OrderBy(c => c.Name).ToArray();
 
-            // Order channels
-            channels = channels.OrderBy(c => c.Name).ToArray();
-
-            // Print result
             foreach (var channel in channels)
                 console.Output.WriteLine($"{channel.Id} | {channel.Name}");
         }

DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs

@@ -16,14 +16,9 @@ namespace DiscordChatExporter.Cli.Commands
 
         public override async ValueTask ExecuteAsync(IConsole console)
         {
-            // Get guilds
             var guilds = await DataService.GetUserGuildsAsync(Token);
 
-            // Order guilds
-            guilds = guilds.OrderBy(g => g.Name).ToArray();
-
-            // Print result
-            foreach (var guild in guilds)
+            foreach (var guild in guilds.OrderBy(g => g.Name))
                 console.Output.WriteLine($"{guild.Id} | {guild.Name}");
         }
     }

DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj

@@ -8,6 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="CliFx" Version="1.0.0" />
+    <PackageReference Include="Gress" Version="1.1.1" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.1" />
     <PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
   </ItemGroup>