Compare commits

...

24 Commits

Author SHA1 Message Date
Tyrrrz 52a23828ce Update version 2023-10-26 16:06:53 +03:00
Tyrrrz 2edd781e32 Update NuGet packages 2023-10-26 15:59:56 +03:00
Tyrrrz 19cdf47c6e Consolidate some docker jobs 2023-10-26 03:11:07 +03:00
Tyrrrz 65e0a63bda Consistency with other projects 2023-10-26 02:49:26 +03:00
Tyrrrz 692438d10c Reduce repetition in workflows using matrices 2023-10-26 02:45:20 +03:00
Tyrrrz c607c6f307 Fix deploy job 2023-10-26 00:03:55 +03:00
Tyrrrz aae43821e7 Simplify CI workflow 2023-10-24 15:13:30 +03:00
Tyrrrz 7b9a4e1aff Fix closing backticks in docs 2023-10-24 14:32:29 +03:00
LostXOR d8e57f94a8 Add command to fix broken permissions to Linux installation instructions (#1148)
Co-authored-by: Oleksii Holub <1935960+Tyrrrz@users.noreply.github.com>
2023-10-24 00:26:52 +03:00
Tyrrrz 7b7df564ed Allow empty names in interactions 2023-10-22 19:22:51 +03:00
Tyrrrz 80b59d1254 Update version 2023-10-19 18:52:19 +03:00
Tyrrrz 6579563816 Don't unfurl URLs in notify job 2023-10-17 16:18:44 +03:00
Tyrrrz e2971ed3a6 Improve notify job message 2023-10-17 15:01:59 +03:00
Tyrrrz a0a263f074 Improve error message for forum channels
Related to #1144
2023-10-16 17:22:28 +03:00
Tyrrrz 1438fd9f77 Temp fix for the docs link in the readme 2023-10-16 17:12:40 +03:00
Tyrrrz b298b45cf7 Escape markup in CLI 2023-10-13 01:17:13 +03:00
Tyrrrz 8d01c6be35 Improve notify message format 2023-10-12 16:00:47 +03:00
Tyrrrz ce4dbb2616 Update version 2023-10-12 15:43:31 +03:00
Tyrrrz 033d83bc4f More cleanup 2023-10-11 01:20:30 +03:00
Tyrrrz fb6cde38b6 Clean up after last commit 2023-10-10 00:04:57 +03:00
Adam Slatinský 46ede471a9 Remove CDN signature before hashing asset URL (#1138) 2023-10-09 23:58:31 +03:00
Tyrrrz 09f8937d99 Refactor 2023-10-09 16:18:56 +03:00
Adam Slatinský ad2dab2157 Add progress to fetching channels step (#1131) 2023-10-08 23:56:39 +03:00
Tyrrrz c5e426289f Include release link in notify 2023-10-05 22:52:31 +03:00
18 changed files with 380 additions and 181 deletions
+6
View File
@@ -15,6 +15,12 @@ Please follow the [instructions provided here](https://github.com/Tyrrrz/Discord
dotnet DiscordChatExporter.Cli.dll export -t TOKEN -c CHANNEL
```
If the above command throws a "Permission denied" error, execute this command to fix the permissions:
```console
chmod 644 *.dll DiscordChatExporter.*
```
> [How to get Token and Channel IDs](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/.docs/Token-and-IDs.md).
There's much more you can do with DCE.CLI! Read the [CLI explained](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/.docs/Getting-started.md#dcecli-commands-) page to get started.
+7 -30
View File
@@ -3,6 +3,8 @@ name: docker
on: [push, pull_request]
jobs:
# Outputs from this job aren't really used, but it's here to verify that
# the Dockerfile builds correctly on pull requests.
build:
runs-on: ubuntu-latest
permissions:
@@ -30,8 +32,9 @@ jobs:
name: DiscordChatExporter.Cli.Docker
path: DiscordChatExporter.Cli.Docker.tar
deploy-latest:
if: ${{ github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master' }}
deploy:
# Deploy to DockerHub only on tag push or master branch push
if: ${{ github.event_name == 'push' && (github.ref_type == 'tag' || github.ref_type == 'branch' && github.ref_name == 'master') }}
runs-on: ubuntu-latest
permissions:
contents: read
@@ -55,32 +58,6 @@ jobs:
--platform linux/amd64,linux/arm64
--push
--tag tyrrrz/discordchatexporter:latest
${{ github.ref_type == 'tag' && '--tag tyrrrz/discordchatexporter:$GITHUB_REF_NAME' || '' }}
${{ github.ref_type == 'tag' && '--tag tyrrrz/discordchatexporter:stable' || '' }}
.
deploy-stable:
if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # 4.1.0
- name: Install Docker Buildx
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # 3.0.0
- name: Login to DockerHub
run: >
echo ${{ secrets.DOCKER_TOKEN }} |
docker login --username tyrrrz --password-stdin
- name: Build & push image
run: >
docker buildx build
--file DiscordChatExporter.Cli.dockerfile
--platform linux/amd64,linux/arm64
--push
--tag tyrrrz/discordchatexporter:stable
--tag tyrrrz/discordchatexporter:${{ github.ref_name }}
.
+56 -52
View File
@@ -25,6 +25,8 @@ jobs:
- name: Run tests
# Tests need access to secrets, so we can't run them against PRs because of limited trust
if: ${{ github.event_name != 'pull_request' }}
env:
DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
run: >
dotnet test
--configuration Release
@@ -33,8 +35,6 @@ jobs:
--
RunConfiguration.CollectSourceInformation=true
DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=opencover
env:
DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
- name: Upload coverage
uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 3.1.4
@@ -42,6 +42,12 @@ jobs:
token: ${{ secrets.CODECOV_TOKEN }}
pack:
strategy:
matrix:
app:
- DiscordChatExporter.Cli
- DiscordChatExporter.Gui
runs-on: windows-latest
permissions:
actions: write
@@ -56,80 +62,78 @@ jobs:
with:
dotnet-version: 7.0.x
- name: Publish (CLI)
- name: Publish app
run: >
dotnet publish DiscordChatExporter.Cli
--output DiscordChatExporter.Cli/publish/
dotnet publish ${{ matrix.app }}
--output ${{ matrix.app }}/bin/publish/
--configuration Release
- name: Publish (GUI)
run: >
dotnet publish DiscordChatExporter.Gui
--output DiscordChatExporter.Gui/publish/
--configuration Release
- name: Upload artifacts (CLI)
- name: Upload artifacts
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # 3.1.3
with:
name: DiscordChatExporter.Cli
path: DiscordChatExporter.Cli/publish/
name: ${{ matrix.app }}
path: ${{ matrix.app }}/bin/publish/
- name: Upload artifacts (GUI)
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # 3.1.3
with:
name: DiscordChatExporter
path: DiscordChatExporter.Gui/publish/
deploy:
release:
if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
needs:
- test
- pack
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Create release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: >
gh release create "${{ github.ref_name }}"
--repo "${{ github.event.repository.full_name }}"
--title "${{ github.ref_name }}"
--notes "[Changelog](${{ github.event.repository.html_url }}/blob/${{ github.ref_name }}/Changelog.md)"
--verify-tag
deploy:
strategy:
matrix:
app:
- DiscordChatExporter.Cli
- DiscordChatExporter.Gui
include:
- app: DiscordChatExporter.Cli
asset: DiscordChatExporter.Cli
- app: DiscordChatExporter.Gui
# GUI asset isn't suffixed, unlike the CLI asset
asset: DiscordChatExporter
needs: release
runs-on: ubuntu-latest
permissions:
actions: read
contents: write
steps:
- name: Download artifacts (CLI)
- name: Download artifacts
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # 3.0.2
with:
name: DiscordChatExporter.Cli
path: DiscordChatExporter.Cli
name: ${{ matrix.app }}
path: ${{ matrix.app }}/
- name: Download artifacts (GUI)
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # 3.0.2
with:
name: DiscordChatExporter
path: DiscordChatExporter.Gui
- name: Create package (CLI)
shell: pwsh
run: >
Compress-Archive
-Path DiscordChatExporter.Cli/*
-DestinationPath DiscordChatExporter.Cli.zip
-Force
- name: Create package (GUI)
shell: pwsh
run: >
Compress-Archive
-Path DiscordChatExporter.Gui/*
-DestinationPath DiscordChatExporter.zip
-Force
- name: Create package
# Change into the artifacts directory to avoid including the directory itself in the zip archive
working-directory: ${{ matrix.app }}/
run: zip -r ../${{ matrix.asset }}.zip .
- name: Create release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: >
gh release create "${{ github.ref_name }}"
"DiscordChatExporter.Cli.zip"
"DiscordChatExporter.zip"
gh release upload "${{ github.ref_name }}"
"${{ matrix.asset }}.zip"
--repo "${{ github.event.repository.full_name }}"
--title "${{ github.ref_name }}"
--notes "[Changelog](${{ github.event.repository.html_url }}/blob/${{ github.ref_name }}/Changelog.md)"
--verify-tag
notify:
needs: deploy
@@ -148,6 +152,6 @@ jobs:
body: |
{
"avatar_url": "https://raw.githubusercontent.com/${{ github.event.repository.full_name }}/${{ github.ref_name }}/favicon.png",
"content": "**${{ github.event.repository.name }}** new version released!\nVersion: `${{ github.ref_name }}`\nChangelog: <${{ github.event.repository.html_url }}/blob/${{ github.ref_name }}/Changelog.md>"
"content": "**${{ github.event.repository.name }}** v${{ github.ref_name }} has been released!\n🔗 [Download](<${{ github.event.repository.html_url }}/releases/tag/${{ github.ref_name }}>) • [Changelog](<${{ github.event.repository.html_url }}/blob/${{ github.ref_name }}/Changelog.md>)"
}
retry-count: 5
+19
View File
@@ -1,5 +1,24 @@
# Changelog
## v2.42.2 (26-Oct-2023)
- General changes:
- Fixed an issue where trying to export a channel that contained application interactions with empty command names crashed the application.
## v2.42.1 (19-Oct-2023)
- General changes:
- Improved the error message shown when trying to export a forum channel. Such channels cannot be exported directly, you need to fetch and export their individual threads instead.
- CLI changes:
- Fixed an issue where fetching threads with `[` or `]` in their name crashed the application.
## v2.42 (12-Oct-2023)
- General changes:
- Fixed an issue where the "reuse assets" option did not work as intended due to Discord recently introducing signatures in the CDN URLs. (Thanks [@slatinsky](https://github.com/slatinsky))
- CLI changes:
- Added progress reporting for the "fetching channels" stage of the `exportguild` and `exportall` commands. (Thanks [@slatinsky](https://github.com/slatinsky))
## v2.41.2 (05-Oct-2023)
- General changes:
+1 -1
View File
@@ -2,7 +2,7 @@
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<Version>2.41.2</Version>
<Version>2.42.2</Version>
<Company>Tyrrrz</Company>
<Copyright>Copyright (c) Oleksii Holub</Copyright>
<LangVersion>preview</LangVersion>
@@ -11,7 +11,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="AngleSharp" Version="1.0.4" />
<PackageReference Include="AngleSharp" Version="1.0.5" />
<PackageReference Include="coverlet.collector" Version="6.0.0" PrivateAssets="all" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="FluentAssertions" Version="6.12.0" />
@@ -22,8 +22,8 @@
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="7.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
<PackageReference Include="ReflectionMagic" Version="5.0.0" />
<PackageReference Include="xunit" Version="2.5.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.1" PrivateAssets="all" />
<PackageReference Include="xunit" Version="2.5.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" PrivateAssets="all" />
</ItemGroup>
<ItemGroup>
@@ -186,7 +186,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1124
ParallelLimit > 1
)
.StartAsync(async progressContext =>
.StartAsync(async ctx =>
{
await Parallel.ForEachAsync(
channels,
@@ -199,8 +199,8 @@ public abstract class ExportCommandBase : DiscordCommandBase
{
try
{
await progressContext.StartTaskAsync(
channel.GetHierarchicalName(),
await ctx.StartTaskAsync(
Markup.Escape(channel.GetHierarchicalName()),
async progress =>
{
var guild = await Discord.GetGuildAsync(
@@ -257,7 +257,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
using (console.WithForegroundColor(ConsoleColor.Red))
{
await console.Error.WriteLineAsync(
$"Failed to export {errorsByChannel.Count} channel(s):"
$"Failed to export {errorsByChannel.Count} the following channel(s):"
);
}
@@ -1,17 +1,17 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Text.Json;
using System.Linq;
using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Exceptions;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Cli.Commands.Converters;
using DiscordChatExporter.Cli.Commands.Shared;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Cli.Utils.Extensions;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Dump;
using DiscordChatExporter.Core.Exceptions;
using JsonExtensions.Reading;
using Spectre.Console;
namespace DiscordChatExporter.Cli.Commands;
@@ -51,39 +51,84 @@ public class ExportAllCommand : ExportCommandBase
// Pull from the API
if (string.IsNullOrWhiteSpace(DataPackageFilePath))
{
await console.Output.WriteLineAsync("Fetching channels...");
await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
{
// Regular channels
await foreach (
var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken)
)
{
if (channel.IsCategory)
continue;
await console.Output.WriteLineAsync(
$"Fetching channels for guild '{guild.Name}'..."
);
if (!IncludeVoiceChannels && channel.IsVoice)
continue;
var fetchedChannelsCount = 0;
await console
.CreateStatusTicker()
.StartAsync(
"...",
async ctx =>
{
await foreach (
var channel in Discord.GetGuildChannelsAsync(
guild.Id,
cancellationToken
)
)
{
if (channel.IsCategory)
continue;
channels.Add(channel);
}
if (!IncludeVoiceChannels && channel.IsVoice)
continue;
channels.Add(channel);
ctx.Status(
Markup.Escape($"Fetched '{channel.GetHierarchicalName()}'.")
);
fetchedChannelsCount++;
}
}
);
await console.Output.WriteLineAsync($"Fetched {fetchedChannelsCount} channel(s).");
// Threads
if (ThreadInclusionMode != ThreadInclusionMode.None)
{
await foreach (
var thread in Discord.GetGuildThreadsAsync(
guild.Id,
ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken
)
)
{
channels.Add(thread);
}
await console.Output.WriteLineAsync(
$"Fetching threads for guild '{guild.Name}'..."
);
var fetchedThreadsCount = 0;
await console
.CreateStatusTicker()
.StartAsync(
"...",
async ctx =>
{
await foreach (
var thread in Discord.GetGuildThreadsAsync(
guild.Id,
ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken
)
)
{
channels.Add(thread);
ctx.Status(
Markup.Escape($"Fetched '{thread.GetHierarchicalName()}'.")
);
fetchedThreadsCount++;
}
}
);
await console.Output.WriteLineAsync(
$"Fetched {fetchedThreadsCount} thread(s)."
);
}
}
}
@@ -91,39 +136,59 @@ public class ExportAllCommand : ExportCommandBase
else
{
await console.Output.WriteLineAsync("Extracting channels...");
using var archive = ZipFile.OpenRead(DataPackageFilePath);
var entry = archive.GetEntry("messages/index.json");
if (entry is null)
throw new CommandException("Could not find channel index inside the data package.");
var dump = await DataDump.LoadAsync(DataPackageFilePath, cancellationToken);
var inaccessibleChannels = new List<DataDumpChannel>();
await using var stream = entry.Open();
using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken);
await console
.CreateStatusTicker()
.StartAsync(
"...",
async ctx =>
{
foreach (var dumpChannel in dump.Channels)
{
ctx.Status(
Markup.Escape(
$"Fetching '{dumpChannel.Name}' ({dumpChannel.Id})..."
)
);
foreach (var property in document.RootElement.EnumerateObjectOrEmpty())
{
var channelId = Snowflake.Parse(property.Name);
var channelName = property.Value.GetString();
try
{
var channel = await Discord.GetChannelAsync(
dumpChannel.Id,
cancellationToken
);
// Null items refer to deleted channels
if (channelName is null)
continue;
await console.Output.WriteLineAsync(
$"Fetching channel '{channelName}' ({channelId})..."
channels.Add(channel);
}
catch (DiscordChatExporterException)
{
inaccessibleChannels.Add(dumpChannel);
}
}
}
);
try
{
var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
channels.Add(channel);
}
catch (DiscordChatExporterException)
await console.Output.WriteLineAsync($"Fetched {channels} channel(s).");
// Print inaccessible channels
if (inaccessibleChannels.Any())
{
await console.Output.WriteLineAsync();
using (console.WithForegroundColor(ConsoleColor.Red))
{
await console.Error.WriteLineAsync(
$"Channel '{channelName}' ({channelId}) is inaccessible."
"Failed to access the following channel(s):"
);
}
foreach (var dumpChannel in inaccessibleChannels)
await console.Error.WriteLineAsync($"{dumpChannel.Name} ({dumpChannel.Id})");
await console.Error.WriteLineAsync();
}
}
@@ -1,12 +1,15 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Cli.Commands.Converters;
using DiscordChatExporter.Cli.Commands.Shared;
using DiscordChatExporter.Cli.Utils.Extensions;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using Spectre.Console;
namespace DiscordChatExporter.Cli.Commands;
@@ -33,35 +36,69 @@ public class ExportGuildCommand : ExportCommandBase
var cancellationToken = console.RegisterCancellationHandler();
var channels = new List<Channel>();
// Regular channels
await console.Output.WriteLineAsync("Fetching channels...");
// Regular channels
await foreach (var channel in Discord.GetGuildChannelsAsync(GuildId, cancellationToken))
{
if (channel.IsCategory)
continue;
var fetchedChannelsCount = 0;
await console
.CreateStatusTicker()
.StartAsync(
"...",
async ctx =>
{
await foreach (
var channel in Discord.GetGuildChannelsAsync(GuildId, cancellationToken)
)
{
if (channel.IsCategory)
continue;
if (!IncludeVoiceChannels && channel.IsVoice)
continue;
if (!IncludeVoiceChannels && channel.IsVoice)
continue;
channels.Add(channel);
}
channels.Add(channel);
ctx.Status(Markup.Escape($"Fetched '{channel.GetHierarchicalName()}'."));
fetchedChannelsCount++;
}
}
);
await console.Output.WriteLineAsync($"Fetched {fetchedChannelsCount} channel(s).");
// Threads
if (ThreadInclusionMode != ThreadInclusionMode.None)
{
await foreach (
var thread in Discord.GetGuildThreadsAsync(
GuildId,
ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken
)
)
{
channels.Add(thread);
}
await console.Output.WriteLineAsync("Fetching threads...");
var fetchedThreadsCount = 0;
await console
.CreateStatusTicker()
.StartAsync(
"...",
async ctx =>
{
await foreach (
var thread in Discord.GetGuildThreadsAsync(
GuildId,
ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken
)
)
{
channels.Add(thread);
ctx.Status(Markup.Escape($"Fetched '{thread.GetHierarchicalName()}'."));
fetchedThreadsCount++;
}
}
);
await console.Output.WriteLineAsync($"Fetched {fetchedThreadsCount} thread(s).");
}
await ExportAsync(console, channels);
@@ -17,6 +17,9 @@ internal static class ConsoleExtensions
}
);
public static Status CreateStatusTicker(this IConsole console) =>
console.CreateAnsiConsole().Status().AutoRefresh(true);
public static Progress CreateProgressTicker(this IConsole console) =>
console
.CreateAnsiConsole()
@@ -31,18 +34,17 @@ internal static class ConsoleExtensions
);
public static async ValueTask StartTaskAsync(
this ProgressContext progressContext,
this ProgressContext context,
string description,
Func<ProgressTask, ValueTask> performOperationAsync
)
{
// Description cannot be empty
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1133
var actualDescription = !string.IsNullOrWhiteSpace(description) ? description : "?";
var actualDescription = !string.IsNullOrWhiteSpace(description) ? description : "...";
var progressTask = progressContext.AddTask(
// Don't recognize random square brackets as style tags
Markup.Escape(actualDescription),
var progressTask = context.AddTask(
actualDescription,
new ProgressTaskSettings { MaxValue = 1 }
);
@@ -10,7 +10,7 @@ public record Interaction(Snowflake Id, string Name, User User)
public static Interaction Parse(JsonElement json)
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var name = json.GetProperty("name").GetNonWhiteSpaceString();
var name = json.GetProperty("name").GetNonNullString(); // may be empty, but not null
var user = json.GetProperty("user").Pipe(User.Parse);
return new Interaction(id, name, user);
@@ -0,0 +1,60 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Dump;
public partial class DataDump
{
public IReadOnlyList<DataDumpChannel> Channels { get; }
public DataDump(IReadOnlyList<DataDumpChannel> channels) => Channels = channels;
}
public partial class DataDump
{
public static DataDump Parse(JsonElement json)
{
var channels = new List<DataDumpChannel>();
foreach (var property in json.EnumerateObjectOrEmpty())
{
var channelId = Snowflake.Parse(property.Name);
var channelName = property.Value.GetString();
// Null items refer to deleted channels
if (channelName is null)
continue;
var channel = new DataDumpChannel(channelId, channelName);
channels.Add(channel);
}
return new DataDump(channels);
}
public static async ValueTask<DataDump> LoadAsync(
string zipFilePath,
CancellationToken cancellationToken = default
)
{
using var archive = ZipFile.OpenRead(zipFilePath);
var entry = archive.GetEntry("messages/index.json");
if (entry is null)
{
throw new InvalidOperationException(
"Could not find the channel index inside the data package."
);
}
await using var stream = entry.Open();
using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken);
return Parse(document.RootElement);
}
}
@@ -0,0 +1,3 @@
namespace DiscordChatExporter.Core.Discord.Dump;
public record DataDumpChannel(Snowflake Id, string Name);
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<PackageReference Include="AsyncKeyedLock" Version="6.2.1" />
<PackageReference Include="AsyncKeyedLock" Version="6.2.2" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="Gress" Version="2.1.1" />
<PackageReference Include="JsonExtensions" Version="1.2.0" />
@@ -9,7 +9,7 @@
<PackageReference Include="RazorBlade" Version="0.4.4" />
<PackageReference Include="Superpower" Version="3.0.0" />
<PackageReference Include="WebMarkupMin.Core" Version="2.14.0" />
<PackageReference Include="YoutubeExplode" Version="6.3.4" />
<PackageReference Include="YoutubeExplode" Version="6.3.6" />
</ItemGroup>
</Project>
@@ -22,11 +22,18 @@ public class ChannelExporter
{
// Forum channels don't have messages, they are just a list of threads
if (request.Channel.Kind == ChannelKind.GuildForum)
throw new DiscordChatExporterException("Channel is a forum.");
{
throw new DiscordChatExporterException(
"Channel is a forum and cannot be exported directly. "
+ "You need to pull its threads and export them individually."
);
}
// Check if the channel is empty
if (request.Channel.IsEmpty)
{
throw new DiscordChatExporterException("Channel does not contain any messages.");
}
// Check if the 'after' boundary is valid
if (request.After is not null && !request.Channel.MayHaveMessagesAfter(request.After.Value))
@@ -7,6 +7,7 @@ using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using AsyncKeyedLock;
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions;
@@ -101,12 +102,29 @@ internal partial class ExportAssetDownloader
internal partial class ExportAssetDownloader
{
private static string GetUrlHash(string url) =>
SHA256
.HashData(Encoding.UTF8.GetBytes(url))
private static string GetUrlHash(string url)
{
// Remove signature parameters from Discord CDN URLs to normalize them
static string NormalizeUrl(string url)
{
var uri = new Uri(url);
if (!string.Equals(uri.Host, "cdn.discordapp.com", StringComparison.OrdinalIgnoreCase))
return url;
var query = HttpUtility.ParseQueryString(uri.Query);
query.Remove("ex");
query.Remove("is");
query.Remove("hm");
return uri.GetLeftPart(UriPartial.Path) + query;
}
return SHA256
.HashData(Encoding.UTF8.GetBytes(NormalizeUrl(url)))
.ToHex()
// 5 chars ought to be enough for anybody
.Truncate(5);
}
private static string GetFileNameFromUrl(string url)
{
@@ -13,14 +13,14 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Cogwheel" Version="2.0.3" />
<PackageReference Include="Cogwheel" Version="2.0.4" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="Deorcify" Version="1.0.2" PrivateAssets="all" />
<PackageReference Include="DotnetRuntimeBootstrapper" Version="2.5.1" PrivateAssets="all" />
<PackageReference Include="Gress" Version="2.1.1" />
<PackageReference Include="MaterialDesignColors" Version="2.1.4" />
<PackageReference Include="MaterialDesignThemes" Version="4.9.0" />
<PackageReference Include="Microsoft.Xaml.Behaviors.Wpf" Version="1.1.39" />
<PackageReference Include="Microsoft.Xaml.Behaviors.Wpf" Version="1.1.75" />
<PackageReference Include="Ookii.Dialogs.Wpf" Version="5.0.1" />
<PackageReference Include="Onova" Version="2.6.10" />
<PackageReference Include="PropertyChanged.Fody" Version="4.1.0" PrivateAssets="all" />
+5 -4
View File
@@ -23,7 +23,8 @@
**DiscordChatExporter** is an application that can be used to export message history from any [Discord](https://discord.com) channel to a file.
It works with direct messages, group messages, and server channels, and supports Discord's dialect of markdown as well as most other rich media features.
> ❔ If you have questions or issues, **please refer to the [docs](.docs)**.
<!-- Can't use a relative link here due to a bug in markdown parsing -->
> ❔ If you have questions or issues, **please refer to the [docs](https://github.com/Tyrrrz/DiscordChatExporter/tree/master/.docs)**.
> 💬 If you want to chat, **join my [Discord server](https://discord.gg/2SUWKFnHSm)**.
@@ -57,7 +58,7 @@ The following table lists all available download options:
<td>
<ul>
<li>🟢 <b><a href="https://github.com/Tyrrrz/DiscordChatExporter/releases/latest">Stable release</a></b> (<code>DiscordChatExporter.zip</code>)</li>
<li>🟠 <a href="https://github.com/Tyrrrz/DiscordChatExporter/actions/workflows/main.yml">CI build</a> (<code>DiscordChatExporter.zip</code>)</li>
<li>🟠 <a href="https://github.com/Tyrrrz/DiscordChatExporter/actions/workflows/main.yml">CI build</a> (<code>DiscordChatExporter.Gui.zip</code>)</li>
</ul>
</td>
<td>
@@ -70,8 +71,8 @@ The following table lists all available download options:
<td><b>CLI</b></td>
<td>
<ul>
<li>🟢 <b><a href="https://github.com/Tyrrrz/DiscordChatExporter/releases/latest">Stable release</a></b> (<code>DiscordChatExporter.CLI.zip</code></li>
<li>🟠 <a href="https://github.com/Tyrrrz/DiscordChatExporter/actions/workflows/main.yml">CI build</a> (<code>DiscordChatExporter.CLI.zip</code>)</li>
<li>🟢 <b><a href="https://github.com/Tyrrrz/DiscordChatExporter/releases/latest">Stable release</a></b> (<code>DiscordChatExporter.Cli.zip</code></li>
<li>🟠 <a href="https://github.com/Tyrrrz/DiscordChatExporter/actions/workflows/main.yml">CI build</a> (<code>DiscordChatExporter.Cli.zip</code>)</li>
<li>🐋 <a href="https://hub.docker.com/r/tyrrrz/discordchatexporter">Docker</a> (<code>tyrrrz/discordchatexporter</code>)</li>
<li>📦 <a href="https://aur.archlinux.org/packages/discord-chat-exporter-cli">AUR</a> (<code>discord-chat-exporter-cli</code>)</li>
<li>📦 <a href="https://search.nixos.org/packages?query=discordchatexporter-cli">Nix</a> (<code>discordchatexporter-cli</code>)</li>