Compare commits

...

8 Commits

Author SHA1 Message Date
tyrrrz 05f8df51e9 Refactor 2026-06-26 13:31:15 +03:00
Jacob Pfundstein e2c633b004 Include starter message in thread exports and skip placeholder (#1557)
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-26 12:59:20 +03:00
tyrrrz b11a57a825 Fix formatting 2026-06-20 14:33:04 +03:00
Nick f502e577c2 Check for media.discordapp.net in addition to cdn.discordapp.com when stripping signature parameters from media URLs (#1554)
Co-authored-by: Oleksii Holub <1935960+Tyrrrz@users.noreply.github.com>
2026-06-20 14:30:34 +03:00
Nick 8bc9fe7c72 Catch both old uppercase 5-char hash and newer lowercase 5-char hash when migrating to new hash (#1552)
Co-authored-by: Oleksii Holub <1935960+Tyrrrz@users.noreply.github.com>
2026-06-20 14:16:48 +03:00
Jacob Pfundstein 97485c280b Fix output path detection to correctly identify directories and files (#1556) 2026-06-20 13:55:49 +03:00
Jacob Pfundstein 5a57c4202d Validate output path early to prevent errors when exporting multiple channels (#1555)
Co-authored-by: Oleksii Holub <1935960+Tyrrrz@users.noreply.github.com>
2026-06-18 15:37:58 +03:00
tyrrrz acac8f7bbb Update packages 2026-06-08 15:52:10 +03:00
10 changed files with 163 additions and 75 deletions
+2 -2
View File
@@ -14,7 +14,7 @@
<PackageVersion Include="CommunityToolkit.Mvvm" Version="8.4.2" /> <PackageVersion Include="CommunityToolkit.Mvvm" Version="8.4.2" />
<PackageVersion Include="coverlet.collector" Version="10.0.1" /> <PackageVersion Include="coverlet.collector" Version="10.0.1" />
<PackageVersion Include="CSharpier.MsBuild" Version="1.2.6" /> <PackageVersion Include="CSharpier.MsBuild" Version="1.2.6" />
<PackageVersion Include="Deorcify" Version="1.1.0" /> <PackageVersion Include="Deorcify" Version="2.0.1" />
<PackageVersion Include="DialogHost.Avalonia" Version="0.12.2" /> <PackageVersion Include="DialogHost.Avalonia" Version="0.12.2" />
<PackageVersion Include="FluentAssertions" Version="8.10.0" /> <PackageVersion Include="FluentAssertions" Version="8.10.0" />
<PackageVersion Include="GitHubActionsTestLogger" Version="3.0.4" /> <PackageVersion Include="GitHubActionsTestLogger" Version="3.0.4" />
@@ -33,7 +33,7 @@
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.6.0" /> <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.6.0" />
<PackageVersion Include="Onova" Version="2.6.13" /> <PackageVersion Include="Onova" Version="2.6.13" />
<PackageVersion Include="Polly" Version="8.6.6" /> <PackageVersion Include="Polly" Version="8.6.6" />
<PackageVersion Include="PowerKit" Version="1.2.0" /> <PackageVersion Include="PowerKit" Version="2.0.1" />
<PackageVersion Include="RazorBlade" Version="1.0.0" /> <PackageVersion Include="RazorBlade" Version="1.0.0" />
<PackageVersion Include="Spectre.Console" Version="0.55.2" /> <PackageVersion Include="Spectre.Console" Version="0.55.2" />
<PackageVersion Include="Superpower" Version="3.1.0" /> <PackageVersion Include="Superpower" Version="3.1.0" />
@@ -14,13 +14,13 @@
<PackageReference Include="coverlet.collector" PrivateAssets="all" /> <PackageReference Include="coverlet.collector" PrivateAssets="all" />
<PackageReference Include="CSharpier.MsBuild" PrivateAssets="all" /> <PackageReference Include="CSharpier.MsBuild" PrivateAssets="all" />
<PackageReference Include="FluentAssertions" /> <PackageReference Include="FluentAssertions" />
<PackageReference Include="GitHubActionsTestLogger" PrivateAssets="all" /> <PackageReference Include="GitHubActionsTestLogger" />
<PackageReference Include="JsonExtensions" /> <PackageReference Include="JsonExtensions" />
<PackageReference Include="Microsoft.Extensions.Configuration" /> <PackageReference Include="Microsoft.Extensions.Configuration" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" /> <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" /> <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
<PackageReference Include="Microsoft.NET.Test.Sdk" /> <PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="PowerKit" PrivateAssets="all" /> <PackageReference Include="PowerKit" />
<PackageReference Include="xunit" /> <PackageReference Include="xunit" />
<PackageReference Include="xunit.runner.visualstudio" PrivateAssets="all" /> <PackageReference Include="xunit.runner.visualstudio" PrivateAssets="all" />
</ItemGroup> </ItemGroup>
@@ -165,6 +165,37 @@ public abstract class ExportCommandBase : DiscordCommandBase
throw new CommandException("Option --media-dir cannot be used without --media."); throw new CommandException("Option --media-dir cannot be used without --media.");
} }
// Make sure the user does not try to export multiple channels into one file.
// Output path must either be a directory or contain template tokens for this to work.
// Validate this up-front, before fetching threads, because thread fetching can take a
// long time and it's frustrating to fail only after it completes.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/799
// https://github.com/Tyrrrz/DiscordChatExporter/issues/917
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1549
var mayExportMultipleChannels =
// Multiple channels were provided explicitly
channels.Count > 1
// Thread inclusion can add more channels to the export
|| ThreadInclusionMode != ThreadInclusionMode.None;
var isValidOutputPath =
// Anything is valid when exporting a single channel
!mayExportMultipleChannels
// When using template tokens, assume the user knows what they're doing
|| OutputPath.Contains('%')
// Otherwise, require an existing directory or an unambiguous directory path
|| Directory.Exists(OutputPath)
|| Path.EndsInDirectorySeparator(OutputPath);
if (!isValidOutputPath)
{
throw new CommandException(
"Attempted to export multiple channels, but the output path is neither a directory nor a template. "
+ "If the provided output path is meant to be treated as a directory, make sure it ends with a slash. "
+ $"Provided output path: '{OutputPath}'."
);
}
var unwrappedChannels = new List<Channel>(channels); var unwrappedChannels = new List<Channel>(channels);
// Unwrap threads // Unwrap threads
@@ -205,28 +236,6 @@ public abstract class ExportCommandBase : DiscordCommandBase
await console.Output.WriteLineAsync($"Fetched {fetchedThreadsCount} thread(s)."); await console.Output.WriteLineAsync($"Fetched {fetchedThreadsCount} thread(s).");
} }
// Make sure the user does not try to export multiple channels into one file.
// Output path must either be a directory or contain template tokens for this to work.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/799
// https://github.com/Tyrrrz/DiscordChatExporter/issues/917
var isValidOutputPath =
// Anything is valid when exporting a single channel
unwrappedChannels.Count <= 1
// When using template tokens, assume the user knows what they're doing
|| OutputPath.Contains('%')
// Otherwise, require an existing directory or an unambiguous directory path
|| Directory.Exists(OutputPath)
|| Path.EndsInDirectorySeparator(OutputPath);
if (!isValidOutputPath)
{
throw new CommandException(
"Attempted to export multiple channels, but the output path is neither a directory nor a template. "
+ "If the provided output path is meant to be treated as a directory, make sure it ends with a slash. "
+ $"Provided output path: '{OutputPath}'."
);
}
// Export // Export
var errorsByChannel = new ConcurrentDictionary<Channel, string>(); var errorsByChannel = new ConcurrentDictionary<Channel, string>();
var warningsByChannel = new ConcurrentDictionary<Channel, string>(); var warningsByChannel = new ConcurrentDictionary<Channel, string>();
@@ -9,9 +9,9 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="CliFx" /> <PackageReference Include="CliFx" />
<PackageReference Include="CSharpier.MsBuild" PrivateAssets="all" /> <PackageReference Include="CSharpier.MsBuild" PrivateAssets="all" />
<PackageReference Include="Deorcify" PrivateAssets="all" /> <PackageReference Include="Deorcify" />
<PackageReference Include="Gress" /> <PackageReference Include="Gress" />
<PackageReference Include="PowerKit" PrivateAssets="all" /> <PackageReference Include="PowerKit" />
<PackageReference Include="Spectre.Console" /> <PackageReference Include="Spectre.Console" />
</ItemGroup> </ItemGroup>
@@ -13,4 +13,5 @@ public enum MessageKind
GuildMemberJoin = 7, GuildMemberJoin = 7,
ThreadCreated = 18, ThreadCreated = 18,
Reply = 19, Reply = 19,
ThreadStarterMessage = 21,
} }
@@ -30,9 +30,8 @@ public class DiscordClient(
string url, string url,
TokenKind tokenKind, TokenKind tokenKind,
CancellationToken cancellationToken = default CancellationToken cancellationToken = default
) ) =>
{ await Http.ResponseResiliencePipeline.ExecuteAsync(
return await Http.ResponseResiliencePipeline.ExecuteAsync(
async innerCancellationToken => async innerCancellationToken =>
{ {
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url)); using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
@@ -91,7 +90,6 @@ public class DiscordClient(
}, },
cancellationToken cancellationToken
); );
}
private async ValueTask<TokenKind> ResolveTokenKindAsync( private async ValueTask<TokenKind> ResolveTokenKindAsync(
CancellationToken cancellationToken = default CancellationToken cancellationToken = default
@@ -364,6 +362,7 @@ public class DiscordClient(
$"guilds/{guildId}/members/{memberId}", $"guilds/{guildId}/members/{memberId}",
cancellationToken cancellationToken
); );
return response?.Pipe(j => Member.Parse(j, guildId)); return response?.Pipe(j => Member.Parse(j, guildId));
} }
@@ -412,14 +411,12 @@ public class DiscordClient(
?.GetNonWhiteSpaceStringOrNull() ?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse); ?.Pipe(Snowflake.Parse);
Channel? parent = null; // It's possible for the parent channel to be inaccessible, despite the
if (parentId is not null) // child channel being accessible.
{ // https://github.com/Tyrrrz/DiscordChatExporter/issues/1108
// It's possible for the parent channel to be inaccessible, despite the var parent = parentId is not null
// child channel being accessible. ? await TryGetChannelAsync(parentId.Value, cancellationToken)
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1108 : null;
parent = await TryGetChannelAsync(parentId.Value, cancellationToken);
}
return Channel.Parse(response.Value, parent); return Channel.Parse(response.Value, parent);
} }
@@ -607,8 +604,12 @@ public class DiscordClient(
.SetQueryParameter("after", (after ?? Snowflake.Zero).ToString()) .SetQueryParameter("after", (after ?? Snowflake.Zero).ToString())
.Build(); .Build();
var response = await GetJsonResponseAsync(url, cancellationToken); // Can be null on channels that the user cannot access
var message = response.EnumerateArray().Select(Message.Parse).FirstOrDefault(); var response = await TryGetJsonResponseAsync(url, cancellationToken);
if (response is null)
return null;
var message = response.Value.EnumerateArray().Select(Message.Parse).FirstOrDefault();
return message; return message;
} }
@@ -625,8 +626,39 @@ public class DiscordClient(
.SetQueryParameter("before", before?.ToString()) .SetQueryParameter("before", before?.ToString())
.Build(); .Build();
var response = await GetJsonResponseAsync(url, cancellationToken); // Can be null on channels that the user cannot access
return response.EnumerateArray().Select(Message.Parse).LastOrDefault(); var response = await TryGetJsonResponseAsync(url, cancellationToken);
if (response is null)
return null;
return response.Value.EnumerateArray().Select(Message.Parse).LastOrDefault();
}
public async ValueTask<Message?> TryGetMessageAsync(
Snowflake channelId,
Snowflake messageId,
CancellationToken cancellationToken = default
)
{
// Use the regular message listing endpoint with the 'around' parameter instead of the
// dedicated single-message endpoint, because the latter is not accessible to user tokens.
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
.SetQueryParameter("around", messageId.ToString())
.SetQueryParameter("limit", "1")
.Build();
// Can be null on channels that the user cannot access
var response = await TryGetJsonResponseAsync(url, cancellationToken);
if (response is null)
return null;
// The endpoint returns messages around the requested ID, so make sure to only return
// the message that exactly matches it (it may be absent if it has been deleted).
return response
.Value.EnumerateArray()
.Select(Message.Parse)
.FirstOrDefault(m => m.Id == messageId);
} }
public async IAsyncEnumerable<Message> GetMessagesAsync( public async IAsyncEnumerable<Message> GetMessagesAsync(
@@ -701,7 +733,21 @@ public class DiscordClient(
); );
} }
yield return message; // Some messages, for example thread starter messages, are returned by the API as content-less references.
// Try to resolve them to the actual message so that they appear as they do in the Discord client.
var actualMessage =
message.Kind == MessageKind.ThreadStarterMessage
&& message.Reference?.ChannelId is { } referencedChannelId
&& message.Reference?.MessageId is { } referencedMessageId
? await TryGetMessageAsync(
referencedChannelId,
referencedMessageId,
cancellationToken
)
: null;
yield return actualMessage ?? message;
currentAfter = message.Id; currentAfter = message.Id;
} }
} }
@@ -769,7 +815,20 @@ public class DiscordClient(
); );
} }
yield return message; // Some messages, for example thread starter messages, are returned by the API as content-less references.
// Try to resolve them to the actual message so that they appear as they do in the Discord client.
var actualMessage =
message.Kind == MessageKind.ThreadStarterMessage
&& message.Reference?.ChannelId is { } referencedChannelId
&& message.Reference?.MessageId is { } referencedMessageId
? await TryGetMessageAsync(
referencedChannelId,
referencedMessageId,
cancellationToken
)
: null;
yield return actualMessage ?? message;
} }
currentBefore = messages.Last().Id; currentBefore = messages.Last().Id;
@@ -6,7 +6,7 @@
<PackageReference Include="Gress" /> <PackageReference Include="Gress" />
<PackageReference Include="JsonExtensions" /> <PackageReference Include="JsonExtensions" />
<PackageReference Include="Polly" /> <PackageReference Include="Polly" />
<PackageReference Include="PowerKit" PrivateAssets="all" /> <PackageReference Include="PowerKit" />
<PackageReference Include="RazorBlade" /> <PackageReference Include="RazorBlade" />
<PackageReference Include="Superpower" /> <PackageReference Include="Superpower" />
<PackageReference Include="WebMarkupMin.Core" /> <PackageReference Include="WebMarkupMin.Core" />
@@ -38,23 +38,28 @@ internal partial class ExportAssetDownloader(string workingDirPath, bool reuse)
return _previousPathsByUrl[url] = filePath; return _previousPathsByUrl[url] = filePath;
// Check for a file cached by the legacy naming scheme (5-char hash) and rename it // Check for a file cached by the legacy naming scheme (5-char hash) and rename it
// to the new naming scheme to preserve backwards compatibility with existing exports // to the new naming scheme to preserve backwards compatibility with existing exports.
// This will catch both the 5-char lowercase hash and the 5-char uppercase hash variants.
if (reuse) if (reuse)
{ {
var legacyFilePath = Path.Combine(workingDirPath, GetLegacyFileNameFromUrl(url)); var legacyFileNames = GetLegacyFileNamesFromUrl(url);
if (File.Exists(legacyFilePath)) foreach (var legacyFileName in legacyFileNames)
{ {
// Overwrite in case the destination file was created concurrently between our var legacyFilePath = Path.Combine(workingDirPath, legacyFileName);
// earlier existence check and this move operation if (File.Exists(legacyFilePath))
try
{ {
File.Move(legacyFilePath, filePath, overwrite: true); // Overwrite in case the destination file was created concurrently between our
return _previousPathsByUrl[url] = filePath; // earlier existence check and this move operation
} try
catch (IOException) {
{ File.Move(legacyFilePath, filePath, true);
// The legacy file was moved or deleted concurrently or something else happened. return _previousPathsByUrl[url] = filePath;
// Upgrading old files is not crucial, so we can just move on. }
catch (IOException)
{
// The legacy file was moved or deleted concurrently or something else happened.
// Upgrading old files is not crucial, so we can just move on.
}
} }
} }
} }
@@ -87,10 +92,16 @@ internal partial class ExportAssetDownloader
{ {
private static string NormalizeUrl(string url) private static string NormalizeUrl(string url)
{ {
// Remove signature parameters from Discord CDN URLs to normalize them // Remove signature parameters from Discord CDN/media URLs to normalize them
var uri = new Uri(url); var uri = new Uri(url);
if (!string.Equals(uri.Host, "cdn.discordapp.com", StringComparison.OrdinalIgnoreCase))
if (
!string.Equals(uri.Host, "cdn.discordapp.com", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(uri.Host, "media.discordapp.net", StringComparison.OrdinalIgnoreCase)
)
{
return url; return url;
}
var query = HttpUtility.ParseQueryString(uri.Query); var query = HttpUtility.ParseQueryString(uri.Query);
query.Remove("ex"); query.Remove("ex");
@@ -137,13 +148,16 @@ internal partial class ExportAssetDownloader
); );
// Legacy naming used a 5-char hash, kept for backwards compatibility with existing exports // Legacy naming used a 5-char hash, kept for backwards compatibility with existing exports
private static string GetLegacyFileNameFromUrl(string url) => private static IReadOnlyList<string> GetLegacyFileNamesFromUrl(string url)
GetFileNameFromUrl( {
url, var hashData = SHA256.HashData(Encoding.UTF8.GetBytes(NormalizeUrl(url)));
SHA256
.HashData(Encoding.UTF8.GetBytes(NormalizeUrl(url))) return
.Pipe(Convert.ToHexStringLower) [
// 5 chars = 20 bits, reaches 1% collision probability at ~145 files // Lowercase variant (introduced in 2.46.1)
.Truncate(5) GetFileNameFromUrl(url, Convert.ToHexStringLower(hashData).Truncate(5)),
); // Uppercase variant (original)
GetFileNameFromUrl(url, Convert.ToHexString(hashData).Truncate(5)),
];
}
} }
@@ -206,10 +206,15 @@ public partial class ExportRequest
{ {
var actualOutputPath = FormatPath(outputPath, guild, channel, after, before); var actualOutputPath = FormatPath(outputPath, guild, channel, after, before);
// Output is a directory // Determine whether the output path refers to a directory or a file.
// The extension-based heuristic is evaluated on the original, unsubstituted path,
// because the value of a template token (e.g. a guild or channel name) may contain
// a period that would otherwise be mistaken for a file extension, incorrectly causing
// a directory path to be treated as a file.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1502
if ( if (
Directory.Exists(actualOutputPath) Directory.Exists(actualOutputPath)
|| string.IsNullOrWhiteSpace(Path.GetExtension(actualOutputPath)) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath))
) )
{ {
var fileName = GetDefaultOutputFileName(guild, channel, format, after, before); var fileName = GetDefaultOutputFileName(guild, channel, format, after, before);
@@ -31,7 +31,7 @@
<PackageReference Include="Cogwheel" /> <PackageReference Include="Cogwheel" />
<PackageReference Include="CommunityToolkit.Mvvm" /> <PackageReference Include="CommunityToolkit.Mvvm" />
<PackageReference Include="CSharpier.MsBuild" PrivateAssets="all" /> <PackageReference Include="CSharpier.MsBuild" PrivateAssets="all" />
<PackageReference Include="Deorcify" PrivateAssets="all" /> <PackageReference Include="Deorcify" />
<PackageReference Include="DialogHost.Avalonia" /> <PackageReference Include="DialogHost.Avalonia" />
<PackageReference Include="Gress" /> <PackageReference Include="Gress" />
<PackageReference Include="Markdig" /> <PackageReference Include="Markdig" />
@@ -39,7 +39,7 @@
<PackageReference Include="Material.Icons.Avalonia" /> <PackageReference Include="Material.Icons.Avalonia" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
<PackageReference Include="Onova" /> <PackageReference Include="Onova" />
<PackageReference Include="PowerKit" PrivateAssets="all" /> <PackageReference Include="PowerKit" />
<PackageReference Include="ThisAssembly.Project" PrivateAssets="all" /> <PackageReference Include="ThisAssembly.Project" PrivateAssets="all" />
</ItemGroup> </ItemGroup>