Compare commits

...

55 Commits

Author SHA1 Message Date
Tyrrrz 9180d51e5e Update version 2023-09-28 19:30:21 +03:00
Tyrrrz a58509fda8 Upgrade to Polly 8 usage 2023-09-28 19:30:12 +03:00
Tyrrrz fbfff4e51f Make tests more resilient 2023-09-28 19:18:52 +03:00
Tyrrrz c466c17793 Update NuGet packages 2023-09-28 16:16:20 +03:00
Tyrrrz 1b2bcf771d Add fallback for progress tasks without name in CLI
Closes #1133
2023-09-26 00:13:22 +03:00
Tyrrrz 99b49f0b8f Allow pack job to run in parallel to test 2023-09-22 14:51:42 +03:00
Tyrrrz 850414b017 Clean up workflow 2023-09-21 20:18:16 +03:00
Tyrrrz 92bf6eab2d Treat all warnings as errors 2023-09-20 21:36:43 +03:00
Oleksii Holub 16d44ce654 Update version 2023-09-15 13:02:35 +03:00
Tyrrrz 5abe74894c Remove Channel.ParentNameWithFallback 2023-09-07 16:36:29 +03:00
Oleksii Holub 59344cedbe Replace the date format option with a locale option (#1130) 2023-09-07 14:34:08 +03:00
Oleksii Holub 53b11d6c49 Update docks for Docker usage 2023-09-07 00:18:29 +03:00
Tyrrrz ccdf082689 Clean up formatting 2023-09-03 22:19:03 +03:00
Tyrrrz 9601e0acea Refactor 2023-09-03 21:46:20 +03:00
Adam Slatinský 9583e2684d Optimize fetching threads (#1125) 2023-09-03 21:18:49 +03:00
Tyrrrz d24550f521 Hide completed tasks in CLI when exporting in parallel
Closes #1124
2023-08-30 18:43:31 +03:00
Tyrrrz c422ac2351 Clean up 2023-08-30 18:43:12 +03:00
Tyrrrz 3740d64601 Optimize thread inclusion UX across GUI and CLI
Related to #1119
2023-08-28 22:08:51 +03:00
Tyrrrz d430f77ae1 Replace --include-threads and --include-archived-threads with a single multi-value option
Related to #1119
2023-08-28 21:52:56 +03:00
Tyrrrz 5f6e51f6fb Show threads in GUI at the bottom of the list, in specialized groups
Related to #1119
2023-08-28 21:31:51 +03:00
Tyrrrz f1c094ac14 Implicitly enable threads if archived threads are enabled in GUI 2023-08-28 21:09:42 +03:00
Tyrrrz 6fb6108610 Be safer when pulling threads using a bot token 2023-08-23 15:58:53 +03:00
Tyrrrz e75a1933f8 Bypass CSharpier in docker build 2023-08-22 21:36:50 +03:00
Tyrrrz 20f58963a6 Use CSharpier 2023-08-22 21:17:19 +03:00
Tyrrrz c410e745b1 Add rudimentary thread support in GUI
Related to #1119
2023-08-22 21:14:34 +03:00
Tyrrrz c29de2c77c Fetch threads more efficiently for user tokens 2023-08-22 19:16:32 +03:00
Tyrrrz 67d41fbdc4 Fix jitter in the GUI when switching from DM guild to any other and back 2023-08-22 17:41:36 +03:00
Tyrrrz 041086267c Short-circuit GetGuildThreadsAsync(...) for DMs 2023-08-22 00:10:04 +03:00
Tyrrrz 47588296a9 Use ValueTask for consistency 2023-08-22 00:08:45 +03:00
Tyrrrz d1a8f466f6 Pull channels lazily in the GUI
Closes #1005
2023-08-22 00:03:29 +03:00
Tyrrrz 758fb26dad Add --include-threads and --include-archived-threads to exportall
Related to #1119
2023-08-21 17:45:12 +03:00
Tyrrrz 8236ba7c4d Add defaults to inits 2023-08-21 17:35:10 +03:00
Tyrrrz eac974cfb1 Add --include-threads and --include-archived-threads to exportguild
Related to #1119
2023-08-21 17:33:22 +03:00
Tyrrrz d1d560fb55 Add --include-vc to the channels command 2023-08-21 17:24:05 +03:00
Tyrrrz 6fba60e570 Print archived threads only if explicitly requested in the channels command 2023-08-21 17:22:44 +03:00
Tyrrrz ac1bd16439 Clean up 2023-08-21 01:05:57 +03:00
Tyrrrz 60389ab002 Add warning about IsBotToken deprecation 2023-08-21 01:03:29 +03:00
Tyrrrz 817d54039b Improve naming of dictionaries 2023-08-20 16:48:39 +03:00
Tyrrrz 0ca42f5a14 Update issue forms 2023-08-18 01:40:01 +03:00
Tyrrrz 193cfd7b1e Update issue forms 2023-08-13 13:40:22 +03:00
Tyrrrz 21293dc7af Update bug report form 2023-08-12 21:55:27 +03:00
Tyrrrz 88dee76690 Update issue forms 2023-08-12 21:44:29 +03:00
Tyrrrz cd5afd07ac Update version 2023-08-11 00:06:15 +03:00
Tyrrrz 55270ad6d3 Switch to MIT license 2023-08-08 22:46:07 +03:00
Tyrrrz e82ad2b9c6 Add comment linking an issue that explains why the EscapedSymbolTextNodeMatcher is used 2023-08-05 18:35:22 +03:00
Tyrrrz dd0a35838d Allow empty lines in repeated single-line quote nodes
Closes #1115
2023-08-05 18:30:11 +03:00
Tyrrrz c9b68427f3 Update NuGet packages 2023-08-05 18:17:06 +03:00
Tyrrrz 1ba0057174 Clean up 2023-07-31 20:22:10 +03:00
Adam Slatinský d00ecf0c37 Export reaction users in JSON format (#1107) 2023-07-31 20:11:47 +03:00
Tyrrrz b224fca6c0 ToHex() should be uppercase by default 2023-07-30 14:54:15 +03:00
Tyrrrz c9447c5c8f Clean up 2023-07-29 20:53:00 +03:00
Tyrrrz de67cfae1c Use invariant culture 2023-07-29 20:47:43 +03:00
Tyrrrz 4efa6094dc Round up the "call lasted for X minutes" value 2023-07-29 20:47:36 +03:00
Tyrrrz 088612a78f Update version 2023-07-29 00:58:32 +03:00
Tyrrrz de531d4a97 Fix category not working in GUI
Closes #1112
2023-07-29 00:57:45 +03:00
193 changed files with 12211 additions and 11872 deletions
+15 -53
View File
@@ -16,9 +16,9 @@ Note the `:stable` tag. DiscordChatExporter images are tagged according to the f
- `stable`: Latest stable version release and maps to the latest GitHub release. This tag is updated with release of each new version. Recommended for personal use.
- `latest`: Built from the latest commit and maps to the latest CI build on GitHub. This tag is updated with each new commit to `master` branch. Not recommended for everyday use, but you can use it if you want to try out some features that haven't been officially released yet.
- `x.y.z` (e.g. `2.30.1`): Fixed version release and maps to the corresponding tag on GitHub. This tag is never updated once that version has been released. Recommended for use in various automation scenarios.
- `x.y.z` (e.g. `2.30.1`): Fixed version release and maps to the corresponding tag on GitHub. This tag is never updated once that version has been released. Recommended for use in automation scenarios.
You can see all available tags [here](https://hub.docker.com/r/tyrrrz/discordchatexporter/tags?page=1&ordering=name).
You can see all available tags [here](https://hub.docker.com/r/tyrrrz/discordchatexporter/tags?ordering=name).
## Usage
@@ -34,7 +34,7 @@ To export a channel:
docker run --rm -v /path/on/machine:/out tyrrrz/discordchatexporter:stable export -t TOKEN -c CHANNELID
```
If you want colored output, real-time progress reporting, and some other stuff, pass `-it` (interactive) option:
If you want colored output and real-time progress reporting, pass the `-it` (interactive) option:
```
docker run --rm -it -v /path/on/machine:/out tyrrrz/discordchatexporter:stable export -t TOKEN -c CHANNELID
@@ -42,68 +42,30 @@ docker run --rm -it -v /path/on/machine:/out tyrrrz/discordchatexporter:stable e
Note the `-v /path/on/machine:/out` option, which instructs Docker to bind the `/out` directory inside the container to a path on your host machine. Replace `/path/on/machine` with the directory you want the files to be saved at.
**If you are running SELinux, you will need to add the `:z` option after `/out`, e.g.:**
```
docker run --rm -v /path/on/machine:/out:z tyrrrz/discordchatexporter:stable export -t TOKEN -c CHANNELID
```
For more information, refer to the [Docker docs SELinux labels for bind mounts page](https://docs.docker.com/storage/bind-mounts/#configure-the-selinux-label).
> **Note**:
> If you are running SELinux, you will need to add the `:z` option after `/out`, e.g.:
>
> ```
> docker run --rm -v /path/on/machine:/out:z tyrrrz/discordchatexporter:stable export -t TOKEN -c CHANNELID
> ```
>
> For more information, refer to the [Docker docs SELinux labels for bind mounts page](https://docs.docker.com/storage/bind-mounts/#configure-the-selinux-label).
You can also use the current working directory as the output directory by specifying:
- `-v $PWD:/out` in Mac/Linux
- `-v $PWD:/out` in Bash
- `-v $pwd.Path:/out` in PowerShell
For more information, please refer to the [Dockerfile](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/Dockerfile) and [Docker documentation](https://docs.docker.com/engine/reference/run/).
For more information, please refer to the [Dockerfile](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/DiscordChatExporter.Cli.dockerfile) and [Docker documentation](https://docs.docker.com/engine/reference/run).
To get your Token and Channel IDs, please refer to [this page](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/.docs/Token-and-IDs.md).
## Environment variables
DiscordChatExpoter CLI accepts environment variables as fallbacks for the `--token` and `--bot` options.
DiscordChatExpoter CLI accepts the `DISCORD_TOKEN` environment variable as a fallback for the `--token` option. You can set this variable either with the `--env` Docker option or with a combination of the `--env-file` Docker option and a `.env` file.
- `DISCORD_TOKEN` to set the token
- `DISCORD_TOKEN_BOT` to set whether it's a bot token (`true` or `false`)
You can use these variables either with the `--env` Docker option or with a combination of the `--env-file` Docker option and a `.env` file.
Please refer to the Docker documentation for more information:
- [Docker run - Set environment variables](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file)
Please refer to the [Docker documentation](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file) for more information.
## Permission issues with Linux hosts
When bounding volumes between the container and the host, files mirrored from the Docker container will be owned by the Docker user (in most setups this is `root`). Please refer to this [issue comment](https://github.com/Tyrrrz/DiscordChatExporter/issues/800#issuecomment-1030471970) for instructions on how to override this behavior.
## Build Docker images for arm on Raspberry Pi
If you use the Docker image on arm (for example Raspberry Pi), you'd build the image as below.
```bash
$ git clone https://github.com/Tyrrrz/DiscordChatExporter.git
$ cd DiscordChatExporter
$ docker build -f DiscordChatExporter.Cli.dockerfile -t tyrrrz/discordchatexporter:latest .
```
After a while, some images are built like below.
```bash
$ docker image ls
REPOSITORY TAG IMAGE ID CREATED SIZE
tyrrrz/discordchatexporter latest 7c9ead725177 49 seconds ago 199MB
<none> <none> 57f9f49d05af About a minute ago 678MB
mcr.microsoft.com/dotnet/sdk 5.0 72af6071941e 4 days ago 641MB
mcr.microsoft.com/dotnet/runtime 3.1 6229bebd5a11 4 days ago 197MB
```
Among these, two images (`tyrrrz/discordchatexporter:latest` and `mcr.microsoft.com/dotnet/runtime:3.1`) are required, so you may delete the other two images.
Now you can execute DiscordChatExporter:
```bash
$ docker run --rm tyrrrz/discordchatexporter:latest
```
---
Special thanks to [@simnalamburt](https://github.com/simnalamburt) (dockerize) and [@Nimja](https://github.com/nimja) (better instructions)
+63 -20
View File
@@ -1,4 +1,4 @@
name: 🐞 Bug report
name: 🐛 Bug report
description: Report broken functionality.
labels: [bug]
@@ -6,26 +6,25 @@ body:
- type: markdown
attributes:
value: |
🧐 **Guidelines:**
- Avoid generic or vague titles such as "Something's not working" or "A couple of problems" — be as descriptive as possible.
- Keep your issue focused on one single problem. If you have multiple bug reports, please create a separate issue for each of them.
- Issues should represent **complete and actionable** work items. If you are unsure about something or have a question, please start a [discussion](https://github.com/Tyrrrz/DiscordChatExporter/discussions/new) instead.
- Remember that **DiscordChatExporter** is an open-source project funded by the community. If you find it useful, **please consider [donating](https://tyrrrz.me/donate) to support its development**.
- Search through [existing issues](https://github.com/Tyrrrz/DiscordChatExporter/issues?q=is%3Aissue) first to ensure that this bug has not been reported before.
- Write a descriptive title for your issue. Avoid generic or vague titles such as "Something's not working" or "A couple of problems".
- Keep your issue focused on one single problem. If you have multiple bug reports, please create separate issues for each of them.
- Provide as much context as possible in the details section. Include screenshots, screen recordings, links, references, or anything else you may consider relevant.
- If you want to ask a question instead of reporting a bug, please use [discussions](https://github.com/Tyrrrz/DiscordChatExporter/discussions/new) instead.
___
- type: input
attributes:
label: Version
description: Which version of DiscordChatExporter does this bug affect?
placeholder: ver X.Y.Z
description: Which version of the application does this bug affect? Make sure you're not using an outdated version.
placeholder: v1.0.0
validations:
required: true
- type: dropdown
attributes:
label: Flavor
description: Does this bug affect the GUI version, the CLI version, or both?
description: Which flavor(s) of the application does this bug affect?
multiple: true
options:
- GUI (Graphical User Interface)
@@ -33,10 +32,18 @@ body:
validations:
required: true
- type: input
attributes:
label: Platform
description: Which platform do you experience this bug on?
placeholder: Docker / Windows 11
validations:
required: true
- type: dropdown
attributes:
label: Export format
description: Does this bug affect specific export format(s)? If not applicable, leave this field blank.
description: Which export format(s) do you experience this bug with, if applicable?
multiple: true
options:
- HTML
@@ -44,21 +51,57 @@ body:
- JSON
- CSV
- type: textarea
attributes:
label: Details
description: Clear and thorough explanation of the bug.
placeholder: I was doing X expecting Y to happen, but Z happened instead.
validations:
required: true
- type: textarea
attributes:
label: Steps to reproduce
description: Minimum steps required to reproduce the bug.
description: >
Minimum steps required to reproduce the bug, including prerequisites, export settings, or other relevant items.
The information provided in this field must be readily actionable, meaning that anyone should be able to reproduce the bug by following these steps.
If the bug depends on external factors (such as a specific server, channel, or message), then this field must include the server invite and the corresponding link.
placeholder: |
Server invite: https://discord.gg/...
Channel or message link: https://discord.com/channels/.../...
Export settings:
- ...
Application settings:
- ...
Steps:
- Step 1
- Step 2
- Step 3
validations:
required: true
- type: textarea
attributes:
label: Details
description: Clear and thorough explanation of the bug, including any additional information you may find relevant.
placeholder: |
- Expected behavior: ...
- Actual behavior: ...
validations:
required: true
- type: checkboxes
attributes:
label: Checklist
description: Quick list of checks to ensure that everything is in order.
options:
- label: I have looked through existing issues to make sure that this bug has not been reported before
required: true
- label: I have provided a descriptive title for this issue
required: true
- label: I have made sure that that this bug is reproducible on the latest version of the application
required: true
- label: I have provided all the information needed to reproduce this bug as efficiently as possible
required: true
- label: I have sponsored this project
required: false
- type: markdown
attributes:
value: |
If you are struggling to provide actionable reproduction steps, or if something else is preventing you from creating a complete bug report, please start a [discussion](https://github.com/Tyrrrz/DiscordChatExporter/discussions/new) instead.
+4 -4
View File
@@ -3,12 +3,12 @@ contact_links:
- name: ⚠ Feature request
url: https://github.com/Tyrrrz/.github/blob/master/docs/project-status.md
about: Sorry, but this project is in maintenance mode and no longer accepts new feature requests.
- name: 💬 Discord server
url: https://discord.gg/2SUWKFnHSm
about: Chat with the project community.
- name: 📖 Documentation
url: https://github.com/Tyrrrz/DiscordChatExporter/blob/master/.docs
about: Find usage guides and frequently asked questions.
- name: 🗨 Discussions
url: https://github.com/Tyrrrz/DiscordChatExporter/discussions/new
about: Ask and answer questions.
about: Ask and answer questions.
- name: 💬 Discord server
url: https://discord.gg/2SUWKFnHSm
about: Chat with the project community.
+8 -12
View File
@@ -2,17 +2,17 @@ name: main
on: [push, pull_request]
env:
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
DOTNET_NOLOGO: true
DOTNET_CLI_TELEMETRY_OPTOUT: true
jobs:
test:
runs-on: windows-latest
permissions:
contents: read
env:
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
DOTNET_NOLOGO: true
DOTNET_CLI_TELEMETRY_OPTOUT: true
steps:
- name: Checkout
uses: actions/checkout@v3
@@ -42,17 +42,11 @@ jobs:
token: ${{ secrets.CODECOV_TOKEN }}
pack:
needs: test
runs-on: windows-latest
permissions:
actions: write
contents: read
env:
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
DOTNET_NOLOGO: true
DOTNET_CLI_TELEMETRY_OPTOUT: true
steps:
- name: Checkout
uses: actions/checkout@v3
@@ -88,7 +82,9 @@ jobs:
deploy:
if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
needs: pack
needs:
- test
- pack
runs-on: ubuntu-latest
permissions:
actions: read
+37 -4
View File
@@ -1,5 +1,38 @@
# Changelog
## v2.41.1 (28-Sep-2023)
- CLI changes:
- Fixed an issue where the export failed to export channels with an empty name.
## v2.41 (15-Sep-2023)
- General changes:
- Replaced the "date format" option with a "locale" option. Now, you can choose one of the locales supported by the Discord app, and DiscordChatExporter will infer all the required date, time, and number formats automatically. In GUI, the locale can be selected from the settings window. In CLI, the locale can be specified using the `--locale` option (e.g. `--locale "en-US"`). By default, the current system locale is used.
- Optimized the algorithm for fetching threads to improve performance. (Thanks [@slatinsky](https://github.com/slatinsky))
- GUI changes:
- Improved the initial load time after submitting the token. Now, only the guilds are loaded initially, and the channels are loaded on-demand when the user selects a guild. This should significantly improve performance for accounts with many guilds.
- Added an option to include threads in the channel list, which is configurable in settings. When enabled, threads will appear at the bottom of the list, grouped by the parent channel. Note that pulling threads (especially archived threads) can be a very time-consuming process, depending on the guild size. By default, threads are not included.
- CLI changes:
- Added a warning when using the `--bot` option, informing that it's obsolete and shouldn't be used.
- Changed the `--include-threads` option on the `channels` command from a switch (boolean) option, to an enum (choice) option. It now has possible values of `none`, `active`, or `all`, indicating which threads should be included. To include both active and archived threads, use `--include-threads all`. For backwards compatibility, the `--include-threads` option can still be used as a switch, in which case it will be treated as `--include-threads active`. Note that pulling threads (especially archived threads) can be a very time-consuming process, depending on the guild size.
- Added the `--include-threads` option to the `exportguild` and `exportall` commands. Using this option, you can specify whether to include threads (and which type) in the export. By default, threads are not included.
- Added the `--include-vc` option to the `channels` command. Using this option, you can specify whether to include voice-channel-accompanying text channels in the output. By default, these channels are included.
- Changed the progress reporting behavior of export commands such that tasks are now hidden on completion if parallelization is enabled (i.e. `--parallel` is set to `2` or higher). This should help improve visual clarity when exporting many channels in parallel.
## v2.40.4 (11-Aug-2023)
- HTML changes:
- Fixed an issue where the "call lasted for X minutes" system message was unnecessarily too precise. Now, it's rounded to the nearest minute.
- Fixed an issue where multi-line quote blocks with empty lines in the middle were rendered as separate blocks.
- JSON changes:
- Added the `users` field to the reaction object, containing an array with limited information about the users that reacted with that particular emoji. (Thanks [@slatinsky](https://github.com/slatinsky))
## v2.40.3 (29-Jul-2023)
- GUI changes:
- Fixed an issue where guild categories were rendered with blank names.
## v2.40.2 (27-Jul-2023)
- General changes:
@@ -12,7 +45,7 @@
- General changes:
- Fixed an issue where the fallback avatar (i.e. the avatar used for when the user doesn't have a custom one set) was not resolved properly for users that don't have a discriminator.
- CLI changes:
- Added the `--include-vc` option to `exportall` and `exportguild` commands, which instructs whether to include voice-channel-accompanying text channels in the export. By default, this is set to `true` to match the previous behavior. To disable the option, use `--include-vc false`.
- Added the `--include-vc` option to the `exportall` and `exportguild` commands, which instructs whether to include voice-channel-accompanying text channels in the export. By default, this is set to `true` to match the previous behavior. To disable the option, use `--include-vc false`.
- GUI changes:
- Replaced the blur effect applied to the token text box with a password mask.
- JSON format changes:
@@ -59,7 +92,7 @@
- Updated the usage guide with additional steps required to retrieve the user token.
- Fixed an issue where certain user mentions were incorrectly rendered as `@Unknown`.
- Fixed an issue where some embed images were rendered using their actual URLs instead of Discord proxy URLs.
- GUI changes:
- GUI changes:
- Added a text box for specifying the output path when exporting channels. It can be used to set an output path that uses template tokens, which previously required hacky workarounds. This text box can be left empty, in which case clicking on the "Export" button will open the file or folder picker dialog, matching the previous behavior.
- Added a notification when the application is updated to a new version, containing the changelog link.
- Moved the "reuse assets" option from the settings dialog to the export dialog.
@@ -67,7 +100,7 @@
- Minor cosmetic improvements.
- CLI changes:
- Changed the behavior of the `export` command when providing IDs of category channels. Now, instead of displaying an error, the command will export all channels contained within the corresponding categories.
- HTML format changes:
- HTML format changes:
- Added proper support for more system notification messages.
- Added support for rendering server invites. Now, if the message contains an invite link, the export will show the server icon and include additional information, such as the name of the server and the target channel.
- Changed the preamble part of the export to show the channel icon when available, instead of the default Discord logo. This is only available for group DM channels. (Thanks [@CanePlayz](https://github.com/CanePlayz))
@@ -562,4 +595,4 @@ Please note that CSV export is most likely not going to receive future updates a
- Added support for user and role mentions.
- Added support for channel mentions.
- Fixed text in pre blocks not being wrapped correctly.
- Added workaround for non-default message types.
- Added workaround for non-default message types.
+2 -2
View File
@@ -2,12 +2,12 @@
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<Version>2.40.2</Version>
<Version>2.41.1</Version>
<Company>Tyrrrz</Company>
<Copyright>Copyright (c) Oleksii Holub</Copyright>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<WarningsAsErrors>nullable</WarningsAsErrors>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
</Project>
@@ -8,22 +8,22 @@
<ItemGroup>
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
<None Include="*.secret" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AngleSharp" Version="1.0.4" />
<PackageReference Include="FluentAssertions" Version="6.11.0" />
<PackageReference Include="GitHubActionsTestLogger" Version="2.3.2" PrivateAssets="all" />
<PackageReference Include="coverlet.collector" Version="6.0.0" PrivateAssets="all" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="GitHubActionsTestLogger" Version="2.3.3" PrivateAssets="all" />
<PackageReference Include="JsonExtensions" Version="1.2.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="7.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.3" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
<PackageReference Include="ReflectionMagic" Version="5.0.0" />
<PackageReference Include="xunit" Version="2.5.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.0" PrivateAssets="all" />
<PackageReference Include="coverlet.collector" Version="6.0.0" PrivateAssets="all" />
<PackageReference Include="xunit" Version="2.5.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.1" PrivateAssets="all" />
</ItemGroup>
<ItemGroup>
@@ -23,4 +23,4 @@ public static class ChannelIds
public static Snowflake SelfContainedTestCases { get; } = Snowflake.Parse("887441432678379560");
public static Snowflake StickerTestCases { get; } = Snowflake.Parse("939668868253769729");
}
}
@@ -19,14 +19,16 @@ namespace DiscordChatExporter.Cli.Tests.Infra;
public static class ExportWrapper
{
private static readonly AsyncKeyedLocker<string> Locker = new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private static readonly AsyncKeyedLocker<string> Locker =
new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private static readonly string DirPath = Path.Combine(
Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? Directory.GetCurrentDirectory(),
Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)
?? Directory.GetCurrentDirectory(),
"ExportCache"
);
@@ -36,9 +38,7 @@ public static class ExportWrapper
{
Directory.Delete(DirPath, true);
}
catch (DirectoryNotFoundException)
{
}
catch (DirectoryNotFoundException) { }
Directory.CreateDirectory(DirPath);
}
@@ -59,20 +59,20 @@ public static class ExportWrapper
Token = Secrets.DiscordToken,
ChannelIds = new[] { channelId },
ExportFormat = format,
OutputPath = filePath
OutputPath = filePath,
Locale = "en-US",
IsUtcNormalizationEnabled = true
}.ExecuteAsync(console);
}
return await File.ReadAllTextAsync(filePath);
}
public static async ValueTask<IHtmlDocument> ExportAsHtmlAsync(Snowflake channelId) => Html.Parse(
await ExportAsync(channelId, ExportFormat.HtmlDark)
);
public static async ValueTask<IHtmlDocument> ExportAsHtmlAsync(Snowflake channelId) =>
Html.Parse(await ExportAsync(channelId, ExportFormat.HtmlDark));
public static async ValueTask<JsonElement> ExportAsJsonAsync(Snowflake channelId) => Json.Parse(
await ExportAsync(channelId, ExportFormat.Json)
);
public static async ValueTask<JsonElement> ExportAsJsonAsync(Snowflake channelId) =>
Json.Parse(await ExportAsync(channelId, ExportFormat.Json));
public static async ValueTask<string> ExportAsPlainTextAsync(Snowflake channelId) =>
await ExportAsync(channelId, ExportFormat.PlainText);
@@ -80,25 +80,26 @@ public static class ExportWrapper
public static async ValueTask<string> ExportAsCsvAsync(Snowflake channelId) =>
await ExportAsync(channelId, ExportFormat.Csv);
public static async ValueTask<IReadOnlyList<IElement>> GetMessagesAsHtmlAsync(Snowflake channelId) =>
(await ExportAsHtmlAsync(channelId))
.QuerySelectorAll("[data-message-id]")
.ToArray();
public static async ValueTask<IReadOnlyList<IElement>> GetMessagesAsHtmlAsync(
Snowflake channelId
) => (await ExportAsHtmlAsync(channelId)).QuerySelectorAll("[data-message-id]").ToArray();
public static async ValueTask<IReadOnlyList<JsonElement>> GetMessagesAsJsonAsync(Snowflake channelId) =>
(await ExportAsJsonAsync(channelId))
.GetProperty("messages")
.EnumerateArray()
.ToArray();
public static async ValueTask<IReadOnlyList<JsonElement>> GetMessagesAsJsonAsync(
Snowflake channelId
) => (await ExportAsJsonAsync(channelId)).GetProperty("messages").EnumerateArray().ToArray();
public static async ValueTask<IElement> GetMessageAsHtmlAsync(Snowflake channelId, Snowflake messageId)
public static async ValueTask<IElement> GetMessageAsHtmlAsync(
Snowflake channelId,
Snowflake messageId
)
{
var message = (await GetMessagesAsHtmlAsync(channelId)).SingleOrDefault(e =>
string.Equals(
e.GetAttribute("data-message-id"),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
var message = (await GetMessagesAsHtmlAsync(channelId)).SingleOrDefault(
e =>
string.Equals(
e.GetAttribute("data-message-id"),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
);
if (message is null)
@@ -111,14 +112,18 @@ public static class ExportWrapper
return message;
}
public static async ValueTask<JsonElement> GetMessageAsJsonAsync(Snowflake channelId, Snowflake messageId)
public static async ValueTask<JsonElement> GetMessageAsJsonAsync(
Snowflake channelId,
Snowflake messageId
)
{
var message = (await GetMessagesAsJsonAsync(channelId)).SingleOrDefault(j =>
string.Equals(
j.GetProperty("id").GetString(),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
var message = (await GetMessagesAsJsonAsync(channelId)).SingleOrDefault(
j =>
string.Equals(
j.GetProperty("id").GetString(),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
);
if (message.ValueKind == JsonValueKind.Undefined)
@@ -130,4 +135,4 @@ public static class ExportWrapper
return message;
}
}
}
@@ -12,6 +12,6 @@ internal static class Secrets
.Build();
public static string DiscordToken =>
Configuration["DISCORD_TOKEN"] ??
throw new InvalidOperationException("Discord token not provided for tests.");
}
Configuration["DISCORD_TOKEN"]
?? throw new InvalidOperationException("Discord token not provided for tests.");
}
@@ -14,16 +14,18 @@ public class CsvContentSpecs
var document = await ExportWrapper.ExportAsCsvAsync(ChannelIds.DateRangeTestCases);
// Assert
document.Should().ContainAll(
"tyrrrz",
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
document
.Should()
.ContainAll(
"tyrrrz",
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
}
}
}
@@ -34,8 +34,7 @@ public class DateRangeSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
var timestamps = Json
.Parse(await File.ReadAllTextAsync(file.Path))
var timestamps = Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("timestamp").GetDateTimeOffset())
@@ -43,21 +42,28 @@ public class DateRangeSpecs
timestamps.All(t => t > after).Should().BeTrue();
timestamps.Should().BeEquivalentTo(new[]
{
new DateTimeOffset(2021, 07, 24, 13, 49, 13, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 38, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 39, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 40, TimeSpan.Zero),
new DateTimeOffset(2021, 09, 08, 14, 26, 35, TimeSpan.Zero)
}, o =>
{
return o
.Using<DateTimeOffset>(ctx =>
ctx.Subject.Should().BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
});
timestamps
.Should()
.BeEquivalentTo(
new[]
{
new DateTimeOffset(2021, 07, 24, 13, 49, 13, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 38, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 39, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 40, TimeSpan.Zero),
new DateTimeOffset(2021, 09, 08, 14, 26, 35, TimeSpan.Zero)
},
o =>
{
return o.Using<DateTimeOffset>(
ctx =>
ctx.Subject
.Should()
.BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
}
);
}
[Fact]
@@ -78,8 +84,7 @@ public class DateRangeSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
var timestamps = Json
.Parse(await File.ReadAllTextAsync(file.Path))
var timestamps = Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("timestamp").GetDateTimeOffset())
@@ -87,19 +92,26 @@ public class DateRangeSpecs
timestamps.All(t => t < before).Should().BeTrue();
timestamps.Should().BeEquivalentTo(new[]
{
new DateTimeOffset(2021, 07, 19, 13, 34, 18, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 19, 15, 58, 48, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 19, 17, 23, 58, TimeSpan.Zero)
}, o =>
{
return o
.Using<DateTimeOffset>(ctx =>
ctx.Subject.Should().BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
});
timestamps
.Should()
.BeEquivalentTo(
new[]
{
new DateTimeOffset(2021, 07, 19, 13, 34, 18, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 19, 15, 58, 48, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 19, 17, 23, 58, TimeSpan.Zero)
},
o =>
{
return o.Using<DateTimeOffset>(
ctx =>
ctx.Subject
.Should()
.BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
}
);
}
[Fact]
@@ -122,8 +134,7 @@ public class DateRangeSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
var timestamps = Json
.Parse(await File.ReadAllTextAsync(file.Path))
var timestamps = Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("timestamp").GetDateTimeOffset())
@@ -131,19 +142,26 @@ public class DateRangeSpecs
timestamps.All(t => t < before && t > after).Should().BeTrue();
timestamps.Should().BeEquivalentTo(new[]
{
new DateTimeOffset(2021, 07, 24, 13, 49, 13, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 38, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 39, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 40, TimeSpan.Zero)
}, o =>
{
return o
.Using<DateTimeOffset>(ctx =>
ctx.Subject.Should().BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
});
timestamps
.Should()
.BeEquivalentTo(
new[]
{
new DateTimeOffset(2021, 07, 24, 13, 49, 13, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 38, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 39, TimeSpan.Zero),
new DateTimeOffset(2021, 07, 24, 14, 52, 40, TimeSpan.Zero)
},
o =>
{
return o.Using<DateTimeOffset>(
ctx =>
ctx.Subject
.Should()
.BeCloseTo(ctx.Expectation, TimeSpan.FromSeconds(1))
)
.WhenTypeIs<DateTimeOffset>();
}
);
}
}
}
@@ -32,8 +32,7 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
@@ -58,8 +57,7 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("author").GetProperty("name").GetString())
@@ -84,8 +82,7 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
@@ -110,8 +107,7 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
@@ -136,12 +132,11 @@ public class FilterSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(file.Path))
Json.Parse(await File.ReadAllTextAsync(file.Path))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
.Should()
.ContainSingle("This has mention");
}
}
}
@@ -1,4 +1,5 @@
using System.Linq;
using System;
using System.Linq;
using System.Threading.Tasks;
using AngleSharp.Dom;
using DiscordChatExporter.Cli.Tests.Infra;
@@ -20,18 +21,18 @@ public class HtmlAttachmentSpecs
);
// Assert
message.Text().Should().ContainAll(
"Generic file attachment",
"Test.txt",
"11 bytes"
);
message.Text().Should().ContainAll("Generic file attachment", "Test.txt", "11 bytes");
message
.QuerySelectorAll("a")
.Select(e => e.GetAttribute("href"))
.Should()
.Contain(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
u =>
u.StartsWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt",
StringComparison.Ordinal
)
);
}
@@ -52,7 +53,11 @@ public class HtmlAttachmentSpecs
.Select(e => e.GetAttribute("src"))
.Should()
.Contain(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
u =>
u.StartsWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png",
StringComparison.Ordinal
)
);
}
@@ -71,9 +76,11 @@ public class HtmlAttachmentSpecs
message.Text().Should().Contain("Video attachment");
var videoUrl = message.QuerySelector("video source")?.GetAttribute("src");
videoUrl.Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
videoUrl
.Should()
.StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
}
[Fact]
@@ -91,8 +98,10 @@ public class HtmlAttachmentSpecs
message.Text().Should().Contain("Audio attachment");
var audioUrl = message.QuerySelector("audio source")?.GetAttribute("src");
audioUrl.Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
audioUrl
.Should()
.StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
}
}
}
@@ -16,26 +16,32 @@ public class HtmlContentSpecs
var messages = await ExportWrapper.GetMessagesAsHtmlAsync(ChannelIds.DateRangeTestCases);
// Assert
messages.Select(e => e.GetAttribute("data-message-id")).Should().Equal(
"866674314627121232",
"866710679758045195",
"866732113319428096",
"868490009366396958",
"868505966528835604",
"868505969821364245",
"868505973294268457",
"885169254029213696"
);
messages
.Select(e => e.GetAttribute("data-message-id"))
.Should()
.Equal(
"866674314627121232",
"866710679758045195",
"866732113319428096",
"868490009366396958",
"868505966528835604",
"868505969821364245",
"868505973294268457",
"885169254029213696"
);
messages.SelectMany(e => e.Text()).Should().ContainInOrder(
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
messages
.SelectMany(e => e.Text())
.Should()
.ContainInOrder(
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
}
}
}
@@ -21,15 +21,21 @@ public class HtmlEmbedSpecs
);
// Assert
message.Text().Should().ContainAll(
"Embed author",
"Embed title",
"Embed description",
"Field 1", "Value 1",
"Field 2", "Value 2",
"Field 3", "Value 3",
"Embed footer"
);
message
.Text()
.Should()
.ContainAll(
"Embed author",
"Embed title",
"Embed description",
"Field 1",
"Value 1",
"Field 2",
"Value 2",
"Field 3",
"Value 3",
"Embed footer"
);
}
[Fact]
@@ -83,7 +89,12 @@ public class HtmlEmbedSpecs
.QuerySelectorAll("source")
.Select(e => e.GetAttribute("src"))
.WhereNotNull()
.Where(s => s.EndsWith("i_am_currently_feeling_slight_displeasure_of_what_you_have_just_sent_lqrem.mp4"))
.Where(
s =>
s.EndsWith(
"i_am_currently_feeling_slight_displeasure_of_what_you_have_just_sent_lqrem.mp4"
)
)
.Should()
.ContainSingle();
}
@@ -134,7 +145,7 @@ public class HtmlEmbedSpecs
// Assert
var iframeUrl = message.QuerySelector("iframe")?.GetAttribute("src");
iframeUrl.Should().Be("https://open.spotify.com/embed/track/1LHZMWefF9502NPfArRfvP");
iframeUrl.Should().StartWith("https://open.spotify.com/embed/track/1LHZMWefF9502NPfArRfvP");
}
[Fact]
@@ -150,7 +161,7 @@ public class HtmlEmbedSpecs
// Assert
var iframeUrl = message.QuerySelector("iframe")?.GetAttribute("src");
iframeUrl.Should().Be("https://www.youtube.com/embed/qOWW4OlgbvE");
iframeUrl.Should().StartWith("https://www.youtube.com/embed/qOWW4OlgbvE");
}
[Fact]
@@ -193,4 +204,4 @@ public class HtmlEmbedSpecs
// Assert
message.Text().Should().Contain("DiscordChatExporter TestServer");
}
}
}
@@ -32,8 +32,7 @@ public class HtmlGroupingSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
var messageGroups = Html
.Parse(await File.ReadAllTextAsync(file.Path))
var messageGroups = Html.Parse(await File.ReadAllTextAsync(file.Path))
.QuerySelectorAll(".chatlog__message-group");
messageGroups.Should().HaveCount(2);
@@ -59,12 +58,6 @@ public class HtmlGroupingSpecs
.QuerySelectorAll(".chatlog__content")
.Select(e => e.Text())
.Should()
.ContainInOrder(
"Eleventh",
"Twelveth",
"Thirteenth",
"Fourteenth",
"Fifteenth"
);
.ContainInOrder("Eleventh", "Twelveth", "Thirteenth", "Fourteenth", "Fifteenth");
}
}
}
@@ -1,8 +1,6 @@
using System;
using System.Threading.Tasks;
using System.Threading.Tasks;
using AngleSharp.Dom;
using DiscordChatExporter.Cli.Tests.Infra;
using DiscordChatExporter.Cli.Tests.Utils;
using DiscordChatExporter.Core.Discord;
using FluentAssertions;
using Xunit;
@@ -14,215 +12,128 @@ public class HtmlMarkdownSpecs
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_a_timestamp_marker()
{
// Date formatting code relies on the local time zone, so we need to set it to a fixed value
TimeZoneInfoEx.SetLocal(TimeSpan.FromHours(+2));
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323136411078787")
);
try
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323136411078787")
);
// Assert
message.Text().Should().Contain("Default timestamp: 02/12/2023 3:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 3:36 PM");
}
finally
{
TimeZoneInfo.ClearCachedData();
}
// Assert
message.Text().Should().Contain("Default timestamp: 2/12/2023 1:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 1:36 PM");
}
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_a_timestamp_marker_in_the_short_format()
{
// Date formatting code relies on the local time zone, so we need to set it to a fixed value
TimeZoneInfoEx.SetLocal(TimeSpan.FromHours(+2));
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323205268967596")
);
try
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323205268967596")
);
// Assert
message.Text().Should().Contain("Short time timestamp: 3:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 3:36 PM");
}
finally
{
TimeZoneInfo.ClearCachedData();
}
// Assert
message.Text().Should().Contain("Short time timestamp: 1:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 1:36 PM");
}
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_a_timestamp_marker_in_the_long_format()
{
// Date formatting code relies on the local time zone, so we need to set it to a fixed value
TimeZoneInfoEx.SetLocal(TimeSpan.FromHours(+2));
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323235342139483")
);
try
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323235342139483")
);
// Assert
message.Text().Should().Contain("Long time timestamp: 3:36:12 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 3:36 PM");
}
finally
{
TimeZoneInfo.ClearCachedData();
}
// Assert
message.Text().Should().Contain("Long time timestamp: 1:36:12 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 1:36 PM");
}
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_a_timestamp_marker_in_the_short_date_format()
{
// Date formatting code relies on the local time zone, so we need to set it to a fixed value
TimeZoneInfoEx.SetLocal(TimeSpan.FromHours(+2));
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323326727634984")
);
try
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323326727634984")
);
// Assert
message.Text().Should().Contain("Short date timestamp: 02/12/2023");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 3:36 PM");
}
finally
{
TimeZoneInfo.ClearCachedData();
}
// Assert
message.Text().Should().Contain("Short date timestamp: 2/12/2023");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 1:36 PM");
}
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_a_timestamp_marker_in_the_long_date_format()
{
// Date formatting code relies on the local time zone, so we need to set it to a fixed value
TimeZoneInfoEx.SetLocal(TimeSpan.FromHours(+2));
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323350731640863")
);
try
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323350731640863")
);
// Assert
message.Text().Should().Contain("Long date timestamp: February 12, 2023");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 3:36 PM");
}
finally
{
TimeZoneInfo.ClearCachedData();
}
// Assert
message.Text().Should().Contain("Long date timestamp: Sunday, February 12, 2023");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 1:36 PM");
}
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_a_timestamp_marker_in_the_full_format()
{
// Date formatting code relies on the local time zone, so we need to set it to a fixed value
TimeZoneInfoEx.SetLocal(TimeSpan.FromHours(+2));
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323374379118593")
);
try
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323374379118593")
);
// Assert
message.Text().Should().Contain("Full timestamp: February 12, 2023 3:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 3:36 PM");
}
finally
{
TimeZoneInfo.ClearCachedData();
}
// Assert
message.Text().Should().Contain("Full timestamp: Sunday, February 12, 2023 1:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 1:36 PM");
}
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_a_timestamp_marker_in_the_full_long_format()
{
// Date formatting code relies on the local time zone, so we need to set it to a fixed value
TimeZoneInfoEx.SetLocal(TimeSpan.FromHours(+2));
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323409095376947")
);
try
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323409095376947")
);
// Assert
message.Text().Should().Contain("Full long timestamp: Sunday, February 12, 2023 3:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 3:36 PM");
}
finally
{
TimeZoneInfo.ClearCachedData();
}
// Assert
message
.Text()
.Should()
.Contain("Full long timestamp: Sunday, February 12, 2023 1:36:12 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 1:36 PM");
}
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_a_timestamp_marker_in_the_relative_format()
{
// Date formatting code relies on the local time zone, so we need to set it to a fixed value
TimeZoneInfoEx.SetLocal(TimeSpan.FromHours(+2));
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323436853285004")
);
try
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074323436853285004")
);
// Assert
message.Text().Should().Contain("Relative timestamp: 02/12/2023 3:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 3:36 PM");
}
finally
{
TimeZoneInfo.ClearCachedData();
}
// Assert
message.Text().Should().Contain("Relative timestamp: 2/12/2023 1:36 PM");
message.InnerHtml.Should().Contain("Sunday, February 12, 2023 1:36 PM");
}
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_an_invalid_timestamp_marker()
{
// Date formatting code relies on the local time zone, so we need to set it to a fixed value
TimeZoneInfoEx.SetLocal(TimeSpan.FromHours(+2));
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074328534409019563")
);
try
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MarkdownTestCases,
Snowflake.Parse("1074328534409019563")
);
// Assert
message.Text().Should().Contain("Invalid timestamp: Invalid date");
}
finally
{
TimeZoneInfo.ClearCachedData();
}
// Assert
message.Text().Should().Contain("Invalid timestamp: Invalid date");
}
}
}
@@ -61,4 +61,4 @@ public class HtmlMentionSpecs
// Assert
message.Text().Should().Contain("Role mention: @Role 1");
}
}
}
@@ -36,9 +36,11 @@ public class HtmlReplySpecs
// Assert
message.Text().Should().Contain("reply to deleted");
message.QuerySelector(".chatlog__reply-link")?.Text().Should().Contain(
"Original message was deleted or could not be loaded."
);
message
.QuerySelector(".chatlog__reply-link")
?.Text()
.Should()
.Contain("Original message was deleted or could not be loaded.");
}
[Fact]
@@ -54,7 +56,11 @@ public class HtmlReplySpecs
// Assert
message.Text().Should().Contain("reply to attachment");
message.QuerySelector(".chatlog__reply-link")?.Text().Should().Contain("Click to see attachment");
message
.QuerySelector(".chatlog__reply-link")
?.Text()
.Should()
.Contain("Click to see attachment");
}
[Fact]
@@ -84,8 +90,11 @@ public class HtmlReplySpecs
);
// Assert
message.Text().Should().Contain("This is a test message from an announcement channel on another server");
message
.Text()
.Should()
.Contain("This is a test message from an announcement channel on another server");
message.Text().Should().Contain("SERVER");
message.QuerySelector(".chatlog__reply-link").Should().BeNull();
}
}
}
@@ -19,7 +19,7 @@ public class HtmlStickerSpecs
// Assert
var stickerUrl = message.QuerySelector("[title='rock'] img")?.GetAttribute("src");
stickerUrl.Should().Be("https://cdn.discordapp.com/stickers/904215665597120572.png");
stickerUrl.Should().StartWith("https://cdn.discordapp.com/stickers/904215665597120572.png");
}
[Fact]
@@ -32,7 +32,12 @@ public class HtmlStickerSpecs
);
// Assert
var stickerUrl = message.QuerySelector("[title='Yikes'] [data-source]")?.GetAttribute("data-source");
stickerUrl.Should().Be("https://cdn.discordapp.com/stickers/816087132447178774.json");
var stickerUrl = message
.QuerySelector("[title='Yikes'] [data-source]")
?.GetAttribute("data-source");
stickerUrl
.Should()
.StartWith("https://cdn.discordapp.com/stickers/816087132447178774.json");
}
}
}
@@ -24,9 +24,14 @@ public class JsonAttachmentSpecs
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments[0].GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
);
attachments[0]
.GetProperty("url")
.GetString()
.Should()
.StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
);
attachments[0].GetProperty("fileName").GetString().Should().Be("Test.txt");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(11);
}
@@ -46,9 +51,14 @@ public class JsonAttachmentSpecs
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments[0].GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
);
attachments[0]
.GetProperty("url")
.GetString()
.Should()
.StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
);
attachments[0].GetProperty("fileName").GetString().Should().Be("bird-thumbnail.png");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(466335);
}
@@ -68,10 +78,20 @@ public class JsonAttachmentSpecs
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments[0].GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
attachments[0].GetProperty("fileName").GetString().Should().Be("file_example_MP4_640_3MG.mp4");
attachments[0]
.GetProperty("url")
.GetString()
.Should()
.StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
attachments[0]
.GetProperty("fileName")
.GetString()
.Should()
.Be("file_example_MP4_640_3MG.mp4");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(3114374);
}
@@ -90,10 +110,15 @@ public class JsonAttachmentSpecs
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments[0].GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
attachments[0]
.GetProperty("url")
.GetString()
.Should()
.StartWith(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
attachments[0].GetProperty("fileName").GetString().Should().Be("file_example_MP3_1MG.mp3");
attachments[0].GetProperty("fileSizeBytes").GetInt64().Should().Be(1087849);
}
}
}
@@ -15,26 +15,32 @@ public class JsonContentSpecs
var messages = await ExportWrapper.GetMessagesAsJsonAsync(ChannelIds.DateRangeTestCases);
// Assert
messages.Select(j => j.GetProperty("id").GetString()).Should().Equal(
"866674314627121232",
"866710679758045195",
"866732113319428096",
"868490009366396958",
"868505966528835604",
"868505969821364245",
"868505973294268457",
"885169254029213696"
);
messages
.Select(j => j.GetProperty("id").GetString())
.Should()
.Equal(
"866674314627121232",
"866710679758045195",
"866732113319428096",
"868490009366396958",
"868505966528835604",
"868505969821364245",
"868505973294268457",
"885169254029213696"
);
messages.Select(j => j.GetProperty("content").GetString()).Should().Equal(
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
messages
.Select(j => j.GetProperty("content").GetString())
.Should()
.Equal(
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
}
}
}
@@ -52,4 +52,4 @@ public class JsonEmbedSpecs
embedFields[2].GetProperty("value").GetString().Should().Be("Value 3");
embedFields[2].GetProperty("isInline").GetBoolean().Should().BeTrue();
}
}
}
@@ -39,7 +39,11 @@ public class JsonMentionSpecs
);
// Assert
message.GetProperty("content").GetString().Should().Be("Text channel mention: #mention-tests");
message
.GetProperty("content")
.GetString()
.Should()
.Be("Text channel mention: #mention-tests");
}
[Fact]
@@ -52,7 +56,11 @@ public class JsonMentionSpecs
);
// Assert
message.GetProperty("content").GetString().Should().Be("Voice channel mention: #general [voice]");
message
.GetProperty("content")
.GetString()
.Should()
.Be("Voice channel mention: #general [voice]");
}
[Fact]
@@ -67,4 +75,4 @@ public class JsonMentionSpecs
// Assert
message.GetProperty("content").GetString().Should().Be("Role mention: @Role 1");
}
}
}
@@ -19,15 +19,16 @@ public class JsonStickerSpecs
);
// Assert
var sticker = message
.GetProperty("stickers")
.EnumerateArray()
.Single();
var sticker = message.GetProperty("stickers").EnumerateArray().Single();
sticker.GetProperty("id").GetString().Should().Be("904215665597120572");
sticker.GetProperty("name").GetString().Should().Be("rock");
sticker.GetProperty("format").GetString().Should().Be("Apng");
sticker.GetProperty("sourceUrl").GetString().Should().Be("https://cdn.discordapp.com/stickers/904215665597120572.png");
sticker
.GetProperty("sourceUrl")
.GetString()
.Should()
.StartWith("https://cdn.discordapp.com/stickers/904215665597120572.png");
}
[Fact]
@@ -40,14 +41,15 @@ public class JsonStickerSpecs
);
// Assert
var sticker = message
.GetProperty("stickers")
.EnumerateArray()
.Single();
var sticker = message.GetProperty("stickers").EnumerateArray().Single();
sticker.GetProperty("id").GetString().Should().Be("816087132447178774");
sticker.GetProperty("name").GetString().Should().Be("Yikes");
sticker.GetProperty("format").GetString().Should().Be("Lottie");
sticker.GetProperty("sourceUrl").GetString().Should().Be("https://cdn.discordapp.com/stickers/816087132447178774.json");
sticker
.GetProperty("sourceUrl")
.GetString()
.Should()
.StartWith("https://cdn.discordapp.com/stickers/816087132447178774.json");
}
}
}
@@ -31,9 +31,7 @@ public class PartitioningSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Directory.EnumerateFiles(dir.Path, "output*")
.Should()
.HaveCount(3);
Directory.EnumerateFiles(dir.Path, "output*").Should().HaveCount(3);
}
[Fact]
@@ -54,8 +52,6 @@ public class PartitioningSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Directory.EnumerateFiles(dir.Path, "output*")
.Should()
.HaveCount(8);
Directory.EnumerateFiles(dir.Path, "output*").Should().HaveCount(8);
}
}
}
@@ -14,16 +14,18 @@ public class PlainTextContentSpecs
var document = await ExportWrapper.ExportAsPlainTextAsync(ChannelIds.DateRangeTestCases);
// Assert
document.Should().ContainAll(
"tyrrrz",
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
document
.Should()
.ContainAll(
"tyrrrz",
"Hello world",
"Goodbye world",
"Foo bar",
"Hurdle Durdle",
"One",
"Two",
"Three",
"Yeet"
);
}
}
}
@@ -31,8 +31,7 @@ public class SelfContainedSpecs
}.ExecuteAsync(new FakeConsole());
// Assert
Html
.Parse(await File.ReadAllTextAsync(filePath))
Html.Parse(await File.ReadAllTextAsync(filePath))
.QuerySelectorAll("body [src]")
.Select(e => e.GetAttribute("src")!)
.Select(f => Path.GetFullPath(f, dir.Path))
@@ -40,4 +39,4 @@ public class SelfContainedSpecs
.Should()
.BeTrue();
}
}
}
+1 -1
View File
@@ -8,4 +8,4 @@ internal static class Html
private static readonly IHtmlParser Parser = new HtmlParser();
public static IHtmlDocument Parse(string source) => Parser.ParseDocument(source);
}
}
@@ -9,8 +9,7 @@ internal partial class TempDir : IDisposable
{
public string Path { get; }
public TempDir(string path) =>
Path = path;
public TempDir(string path) => Path = path;
public void Dispose()
{
@@ -18,9 +17,7 @@ internal partial class TempDir : IDisposable
{
Directory.Delete(Path, true);
}
catch (DirectoryNotFoundException)
{
}
catch (DirectoryNotFoundException) { }
}
}
@@ -29,7 +26,8 @@ internal partial class TempDir
public static TempDir Create()
{
var dirPath = PathEx.Combine(
PathEx.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? Directory.GetCurrentDirectory(),
PathEx.GetDirectoryName(Assembly.GetExecutingAssembly().Location)
?? Directory.GetCurrentDirectory(),
"Temp",
Guid.NewGuid().ToString()
);
@@ -38,4 +36,4 @@ internal partial class TempDir
return new TempDir(dirPath);
}
}
}
@@ -9,8 +9,7 @@ internal partial class TempFile : IDisposable
{
public string Path { get; }
public TempFile(string path) =>
Path = path;
public TempFile(string path) => Path = path;
public void Dispose()
{
@@ -18,9 +17,7 @@ internal partial class TempFile : IDisposable
{
File.Delete(Path);
}
catch (FileNotFoundException)
{
}
catch (FileNotFoundException) { }
}
}
@@ -29,17 +26,15 @@ internal partial class TempFile
public static TempFile Create()
{
var dirPath = PathEx.Combine(
PathEx.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? Directory.GetCurrentDirectory(),
PathEx.GetDirectoryName(Assembly.GetExecutingAssembly().Location)
?? Directory.GetCurrentDirectory(),
"Temp"
);
Directory.CreateDirectory(dirPath);
var filePath = PathEx.Combine(
dirPath,
Guid.NewGuid() + ".tmp"
);
var filePath = PathEx.Combine(dirPath, Guid.NewGuid() + ".tmp");
return new TempFile(filePath);
}
}
}
@@ -1,14 +0,0 @@
using System;
using ReflectionMagic;
namespace DiscordChatExporter.Cli.Tests.Utils;
internal static class TimeZoneInfoEx
{
// https://stackoverflow.com/a/63700512/2205454
public static void SetLocal(TimeZoneInfo timeZone) =>
typeof(TimeZoneInfo).AsDynamicType().s_cachedData._localTimeZone = timeZone;
public static void SetLocal(TimeSpan offset) =>
SetLocal(TimeZoneInfo.CreateCustomTimeZone("test-tz", offset, "test-tz", "test-tz"));
}
+1
View File
@@ -23,6 +23,7 @@ COPY DiscordChatExporter.Cli DiscordChatExporter.Cli
# Publish a self-contained assembly so we can use a slimmer runtime image
RUN dotnet publish DiscordChatExporter.Cli \
--configuration Release \
-p:CSharpier_Bypass=true \
--self-contained \
--use-current-runtime \
--arch $TARGETARCH \
@@ -24,10 +24,28 @@ public abstract class DiscordCommandBase : ICommand
EnvironmentVariable = "DISCORD_TOKEN_BOT",
Description = "This option doesn't do anything. Kept for backwards compatibility."
)]
public bool IsBotToken { get; init; }
public bool IsBotToken { get; init; } = false;
private DiscordClient? _discordClient;
protected DiscordClient Discord => _discordClient ??= new DiscordClient(Token);
public abstract ValueTask ExecuteAsync(IConsole console);
}
public virtual ValueTask ExecuteAsync(IConsole console)
{
#pragma warning disable CS0618
// Warn if the bot option is used
if (IsBotToken)
{
using (console.WithForegroundColor(ConsoleColor.DarkYellow))
{
console.Error.WriteLine(
"Warning: Option --bot is deprecated and should not be used. "
+ "The type of the provided token is now inferred automatically. "
+ "Please update your workflows as this option may be completely removed in a future version."
);
}
}
#pragma warning restore CS0618
return default;
}
}
@@ -1,6 +1,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
@@ -18,6 +19,7 @@ using DiscordChatExporter.Core.Exporting.Partitioning;
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions;
using Gress;
using Spectre.Console;
namespace DiscordChatExporter.Cli.Commands.Base;
@@ -28,11 +30,10 @@ public abstract class ExportCommandBase : DiscordCommandBase
[CommandOption(
"output",
'o',
Description =
"Output file or directory path. " +
"Directory path must end with a slash to avoid ambiguity. " +
"If a directory is specified, file names will be generated automatically. " +
"Supports template tokens, see the documentation for more info."
Description = "Output file or directory path. "
+ "Directory path must end with a slash to avoid ambiguity. "
+ "If a directory is specified, file names will be generated automatically. "
+ "Supports template tokens, see the documentation for more info."
)]
public string OutputPath
{
@@ -42,11 +43,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
init => _outputPath = Path.GetFullPath(value);
}
[CommandOption(
"format",
'f',
Description = "Export format."
)]
[CommandOption("format", 'f', Description = "Export format.")]
public ExportFormat ExportFormat { get; init; } = ExportFormat.HtmlDark;
[CommandOption(
@@ -64,17 +61,15 @@ public abstract class ExportCommandBase : DiscordCommandBase
[CommandOption(
"partition",
'p',
Description =
"Split the output into partitions, each limited to the specified " +
"number of messages (e.g. '100') or file size (e.g. '10mb')."
Description = "Split the output into partitions, each limited to the specified "
+ "number of messages (e.g. '100') or file size (e.g. '10mb')."
)]
public PartitionLimit PartitionLimit { get; init; } = PartitionLimit.Null;
[CommandOption(
"filter",
Description =
"Only include messages that satisfy this filter. " +
"See the documentation for more info."
Description = "Only include messages that satisfy this filter. "
+ "See the documentation for more info."
)]
public MessageFilter MessageFilter { get; init; } = MessageFilter.Null;
@@ -100,15 +95,14 @@ public abstract class ExportCommandBase : DiscordCommandBase
"reuse-media",
Description = "Reuse previously downloaded assets to avoid redundant requests."
)]
public bool ShouldReuseAssets { get; init; }
public bool ShouldReuseAssets { get; init; } = false;
private readonly string? _assetsDirPath;
[CommandOption(
"media-dir",
Description =
"Download assets to this directory. " +
"If not specified, the asset directory path will be derived from the output path."
Description = "Download assets to this directory. "
+ "If not specified, the asset directory path will be derived from the output path."
)]
public string? AssetsDirPath
{
@@ -118,12 +112,19 @@ public abstract class ExportCommandBase : DiscordCommandBase
init => _assetsDirPath = value is not null ? Path.GetFullPath(value) : null;
}
[Obsolete("This option doesn't do anything. Kept for backwards compatibility.")]
[CommandOption(
"dateformat",
Description = "Format used when writing dates."
Description = "This option doesn't do anything. Kept for backwards compatibility."
)]
public string DateFormat { get; init; } = "MM/dd/yyyy h:mm tt";
[CommandOption("locale", Description = "Locale to use when formatting dates and numbers.")]
public string Locale { get; init; } = CultureInfo.CurrentCulture.Name;
[CommandOption("utc", Description = "Normalize all timestamps to UTC+0.")]
public bool IsUtcNormalizationEnabled { get; init; } = false;
[CommandOption(
"fuck-russia",
EnvironmentVariable = "FUCK_RUSSIA",
@@ -131,28 +132,24 @@ public abstract class ExportCommandBase : DiscordCommandBase
// Use a converter to accept '1' as 'true' to reuse the existing environment variable
Converter = typeof(TruthyBooleanBindingConverter)
)]
public bool IsUkraineSupportMessageDisabled { get; init; }
public bool IsUkraineSupportMessageDisabled { get; init; } = false;
private ChannelExporter? _channelExporter;
protected ChannelExporter Exporter => _channelExporter ??= new ChannelExporter(Discord);
protected async ValueTask ExecuteAsync(IConsole console, IReadOnlyList<Channel> channels)
protected async ValueTask ExportAsync(IConsole console, IReadOnlyList<Channel> channels)
{
// Asset reuse can only be enabled if the download assets option is set
// https://github.com/Tyrrrz/DiscordChatExporter/issues/425
if (ShouldReuseAssets && !ShouldDownloadAssets)
{
throw new CommandException(
"Option --reuse-media cannot be used without --media."
);
throw new CommandException("Option --reuse-media cannot be used without --media.");
}
// Assets directory can only be specified if the download assets option is set
if (!string.IsNullOrWhiteSpace(AssetsDirPath) && !ShouldDownloadAssets)
{
throw new CommandException(
"Option --media-dir cannot be used without --media."
);
throw new CommandException("Option --media-dir cannot be used without --media.");
}
// Make sure the user does not try to export multiple channels into one file.
@@ -161,100 +158,112 @@ public abstract class ExportCommandBase : DiscordCommandBase
// https://github.com/Tyrrrz/DiscordChatExporter/issues/917
var isValidOutputPath =
// Anything is valid when exporting a single channel
channels.Count <= 1 ||
channels.Count <= 1
// When using template tokens, assume the user knows what they're doing
OutputPath.Contains('%') ||
|| OutputPath.Contains('%')
// Otherwise, require an existing directory or an unambiguous directory path
Directory.Exists(OutputPath) || PathEx.IsDirectoryPath(OutputPath);
|| Directory.Exists(OutputPath)
|| PathEx.IsDirectoryPath(OutputPath);
if (!isValidOutputPath)
{
throw new CommandException(
"Attempted to export multiple channels, but the output path is neither a directory nor a template. " +
"If the provided output path is meant to be treated as a directory, make sure it ends with a slash."
"Attempted to export multiple channels, but the output path is neither a directory nor a template. "
+ "If the provided output path is meant to be treated as a directory, make sure it ends with a slash."
);
}
// Export
var cancellationToken = console.RegisterCancellationHandler();
var errors = new ConcurrentDictionary<Channel, string>();
var errorsByChannel = new ConcurrentDictionary<Channel, string>();
await console.Output.WriteLineAsync($"Exporting {channels.Count} channel(s)...");
await console.CreateProgressTicker().StartAsync(async progressContext =>
{
await Parallel.ForEachAsync(
channels,
new ParallelOptions
{
MaxDegreeOfParallelism = Math.Max(1, ParallelLimit),
CancellationToken = cancellationToken
},
async (channel, innerCancellationToken) =>
{
try
await console
.CreateProgressTicker()
.HideCompleted(
// When exporting multiple channels in parallel, hide the completed tasks
// because it gets hard to visually parse them as they complete out of order.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1124
ParallelLimit > 1
)
.StartAsync(async progressContext =>
{
await Parallel.ForEachAsync(
channels,
new ParallelOptions
{
await progressContext.StartTaskAsync(
$"{channel.Category} / {channel.Name}",
async progress =>
{
var guild = await Discord.GetGuildAsync(channel.GuildId, innerCancellationToken);
var request = new ExportRequest(
guild,
channel,
OutputPath,
AssetsDirPath,
ExportFormat,
After,
Before,
PartitionLimit,
MessageFilter,
ShouldFormatMarkdown,
ShouldDownloadAssets,
ShouldReuseAssets,
DateFormat
);
await Exporter.ExportChannelAsync(
request,
progress.ToPercentageBased(),
innerCancellationToken
);
}
);
}
catch (DiscordChatExporterException ex) when (!ex.IsFatal)
MaxDegreeOfParallelism = Math.Max(1, ParallelLimit),
CancellationToken = cancellationToken
},
async (channel, innerCancellationToken) =>
{
errors[channel] = ex.Message;
try
{
await progressContext.StartTaskAsync(
channel.GetHierarchicalName(),
async progress =>
{
var guild = await Discord.GetGuildAsync(
channel.GuildId,
innerCancellationToken
);
var request = new ExportRequest(
guild,
channel,
OutputPath,
AssetsDirPath,
ExportFormat,
After,
Before,
PartitionLimit,
MessageFilter,
ShouldFormatMarkdown,
ShouldDownloadAssets,
ShouldReuseAssets,
Locale,
IsUtcNormalizationEnabled
);
await Exporter.ExportChannelAsync(
request,
progress.ToPercentageBased(),
innerCancellationToken
);
}
);
}
catch (DiscordChatExporterException ex) when (!ex.IsFatal)
{
errorsByChannel[channel] = ex.Message;
}
}
}
);
});
);
});
// Print the result
using (console.WithForegroundColor(ConsoleColor.White))
{
await console.Output.WriteLineAsync(
$"Successfully exported {channels.Count - errors.Count} channel(s)."
$"Successfully exported {channels.Count - errorsByChannel.Count} channel(s)."
);
}
// Print errors
if (errors.Any())
if (errorsByChannel.Any())
{
await console.Output.WriteLineAsync();
using (console.WithForegroundColor(ConsoleColor.Red))
{
await console.Error.WriteLineAsync(
$"Failed to export {errors.Count} channel(s):"
$"Failed to export {errorsByChannel.Count} channel(s):"
);
}
foreach (var (channel, error) in errors)
foreach (var (channel, error) in errorsByChannel)
{
await console.Error.WriteAsync($"{channel.Category} / {channel.Name}: ");
await console.Error.WriteAsync($"{channel.GetHierarchicalName()}: ");
using (console.WithForegroundColor(ConsoleColor.Red))
await console.Error.WriteLineAsync(error);
}
@@ -264,11 +273,11 @@ public abstract class ExportCommandBase : DiscordCommandBase
// Fail the command only if ALL channels failed to export.
// If only some channels failed to export, it's okay.
if (errors.Count >= channels.Count)
if (errorsByChannel.Count >= channels.Count)
throw new CommandException("Export failed.");
}
protected async ValueTask ExecuteAsync(IConsole console, IReadOnlyList<Snowflake> channelIds)
protected async ValueTask ExportAsync(IConsole console, IReadOnlyList<Snowflake> channelIds)
{
var cancellationToken = console.RegisterCancellationHandler();
@@ -282,11 +291,11 @@ public abstract class ExportCommandBase : DiscordCommandBase
var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
// Unwrap categories
if (channel.Kind == ChannelKind.GuildCategory)
if (channel.IsCategory)
{
var guildChannels =
channelsByGuild.GetValueOrDefault(channel.GuildId) ??
await Discord.GetGuildChannelsAsync(channel.GuildId, cancellationToken);
channelsByGuild.GetValueOrDefault(channel.GuildId)
?? await Discord.GetGuildChannelsAsync(channel.GuildId, cancellationToken);
foreach (var guildChannel in guildChannels)
{
@@ -303,26 +312,44 @@ public abstract class ExportCommandBase : DiscordCommandBase
}
}
await ExecuteAsync(console, channels);
await ExportAsync(console, channels);
}
public override ValueTask ExecuteAsync(IConsole console)
public override async ValueTask ExecuteAsync(IConsole console)
{
// Support Ukraine callout
if (!IsUkraineSupportMessageDisabled)
{
console.Output.WriteLine("┌────────────────────────────────────────────────────────────────────┐");
console.Output.WriteLine("│ Thank you for supporting Ukraine <3 │");
console.Output.WriteLine("│ │");
console.Output.WriteLine("│ As Russia wages a genocidal war against my country, │");
console.Output.WriteLine("│ I'm grateful to everyone who continues to │");
console.Output.WriteLine("│ stand with Ukraine in our fight for freedom. │");
console.Output.WriteLine("│ │");
console.Output.WriteLine("│ Learn more: https://tyrrrz.me/ukraine │");
console.Output.WriteLine("└────────────────────────────────────────────────────────────────────┘");
console.Output.WriteLine(
"┌────────────────────────────────────────────────────────────────────┐"
);
console.Output.WriteLine(
"│ Thank you for supporting Ukraine <3 │"
);
console.Output.WriteLine(
"│ │"
);
console.Output.WriteLine(
"│ As Russia wages a genocidal war against my country, │"
);
console.Output.WriteLine(
"│ I'm grateful to everyone who continues to │"
);
console.Output.WriteLine(
"│ stand with Ukraine in our fight for freedom. │"
);
console.Output.WriteLine(
"│ │"
);
console.Output.WriteLine(
"│ Learn more: https://tyrrrz.me/ukraine │"
);
console.Output.WriteLine(
"└────────────────────────────────────────────────────────────────────┘"
);
console.Output.WriteLine("");
}
return default;
await base.ExecuteAsync(console);
}
}
}
@@ -0,0 +1,22 @@
using System;
using CliFx.Extensibility;
using DiscordChatExporter.Cli.Commands.Shared;
namespace DiscordChatExporter.Cli.Commands.Converters;
internal class ThreadInclusionModeBindingConverter : BindingConverter<ThreadInclusionMode>
{
public override ThreadInclusionMode Convert(string? rawValue)
{
// Empty or unset value is treated as 'active' to match the previous behavior
if (string.IsNullOrWhiteSpace(rawValue))
return ThreadInclusionMode.Active;
// Boolean 'true' is treated as 'active', boolean 'false' is treated as 'none'
if (bool.TryParse(rawValue, out var boolValue))
return boolValue ? ThreadInclusionMode.Active : ThreadInclusionMode.None;
// Otherwise, fall back to regular enum parsing
return Enum.Parse<ThreadInclusionMode>(rawValue, true);
}
}
@@ -7,19 +7,15 @@ internal class TruthyBooleanBindingConverter : BindingConverter<bool>
{
public override bool Convert(string? rawValue)
{
// Null is still considered true, to match the base behavior
if (rawValue is null)
// Empty or unset value is treated as 'true', to match the regular boolean behavior
if (string.IsNullOrWhiteSpace(rawValue))
return true;
if (string.IsNullOrWhiteSpace(rawValue))
return false;
// Number '1' is treated as 'true', other numbers are treated as 'false'
if (int.TryParse(rawValue, CultureInfo.InvariantCulture, out var intValue))
return intValue == 1;
if (bool.TryParse(rawValue, out var boolValue))
return boolValue;
if (int.TryParse(rawValue, CultureInfo.InvariantCulture, out var intValue) && intValue == 0)
return false;
return true;
// Otherwise, fall back to regular boolean parsing
return bool.Parse(rawValue);
}
}
}
@@ -6,6 +6,8 @@ using CliFx.Attributes;
using CliFx.Exceptions;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Cli.Commands.Converters;
using DiscordChatExporter.Cli.Commands.Shared;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions;
@@ -16,29 +18,26 @@ namespace DiscordChatExporter.Cli.Commands;
[Command("exportall", Description = "Exports all accessible channels.")]
public class ExportAllCommand : ExportCommandBase
{
[CommandOption(
"include-dm",
Description = "Include direct message channels."
)]
[CommandOption("include-dm", Description = "Include direct message channels.")]
public bool IncludeDirectChannels { get; init; } = true;
[CommandOption(
"include-guilds",
Description = "Include guild channels."
)]
[CommandOption("include-guilds", Description = "Include guild channels.")]
public bool IncludeGuildChannels { get; init; } = true;
[CommandOption(
"include-vc",
Description = "Include voice channels."
)]
[CommandOption("include-vc", Description = "Include voice channels.")]
public bool IncludeVoiceChannels { get; init; } = true;
[CommandOption(
"include-threads",
Description = "Which types of threads should be included.",
Converter = typeof(ThreadInclusionModeBindingConverter)
)]
public ThreadInclusionMode ThreadInclusionMode { get; init; } = ThreadInclusionMode.None;
[CommandOption(
"data-package",
Description =
"Path to the personal data package (ZIP file) requested from Discord. " +
"If provided, only channels referenced in the dump will be exported."
Description = "Path to the personal data package (ZIP file) requested from Discord. "
+ "If provided, only channels referenced in the dump will be exported."
)]
public string? DataPackageFilePath { get; init; }
@@ -56,10 +55,36 @@ public class ExportAllCommand : ExportCommandBase
await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
{
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
// Regular channels
await foreach (
var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken)
)
{
if (channel.IsCategory)
continue;
if (!IncludeVoiceChannels && channel.IsVoice)
continue;
channels.Add(channel);
}
// Threads
if (ThreadInclusionMode != ThreadInclusionMode.None)
{
await foreach (
var thread in Discord.GetGuildThreadsAsync(
guild.Id,
ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken
)
)
{
channels.Add(thread);
}
}
}
}
// Pull from the data package
@@ -84,7 +109,9 @@ public class ExportAllCommand : ExportCommandBase
if (channelName is null)
continue;
await console.Output.WriteLineAsync($"Fetching channel '{channelName}' ({channelId})...");
await console.Output.WriteLineAsync(
$"Fetching channel '{channelName}' ({channelId})..."
);
try
{
@@ -93,19 +120,25 @@ public class ExportAllCommand : ExportCommandBase
}
catch (DiscordChatExporterException)
{
await console.Error.WriteLineAsync($"Channel '{channelName}' ({channelId}) is inaccessible.");
await console.Error.WriteLineAsync(
$"Channel '{channelName}' ({channelId}) is inaccessible."
);
}
}
}
// Filter out unwanted channels
if (!IncludeDirectChannels)
channels.RemoveAll(c => c.Kind.IsDirect());
channels.RemoveAll(c => c.IsDirect);
if (!IncludeGuildChannels)
channels.RemoveAll(c => c.Kind.IsGuild());
channels.RemoveAll(c => c.IsGuild);
if (!IncludeVoiceChannels)
channels.RemoveAll(c => c.Kind.IsVoice());
channels.RemoveAll(c => c.IsVoice);
if (ThreadInclusionMode == ThreadInclusionMode.None)
channels.RemoveAll(c => c.IsThread);
if (ThreadInclusionMode != ThreadInclusionMode.All)
channels.RemoveAll(c => c is { IsThread: true, IsArchived: true });
await base.ExecuteAsync(console, channels);
await ExportAsync(console, channels);
}
}
}
@@ -14,15 +14,14 @@ public class ExportChannelsCommand : ExportCommandBase
[CommandOption(
"channel",
'c',
Description =
"Channel ID(s). " +
"If provided with category ID(s), all channels inside those categories will be exported."
Description = "Channel ID(s). "
+ "If provided with category ID(s), all channels inside those categories will be exported."
)]
public required IReadOnlyList<Snowflake> ChannelIds { get; init; }
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
await base.ExecuteAsync(console, ChannelIds);
await ExportAsync(console, ChannelIds);
}
}
}
@@ -17,8 +17,11 @@ public class ExportDirectMessagesCommand : ExportCommandBase
var cancellationToken = console.RegisterCancellationHandler();
await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
var channels = await Discord.GetGuildChannelsAsync(
Guild.DirectMessages.Id,
cancellationToken
);
await base.ExecuteAsync(console, channels);
await ExportAsync(console, channels);
}
}
}
@@ -1,43 +1,69 @@
using System.Linq;
using System.Collections.Generic;
using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Cli.Commands.Converters;
using DiscordChatExporter.Cli.Commands.Shared;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands;
[Command("exportguild", Description = "Exports all channels within the specified guild.")]
public class ExportGuildCommand : ExportCommandBase
{
[CommandOption(
"guild",
'g',
Description = "Guild ID."
)]
[CommandOption("guild", 'g', Description = "Guild ID.")]
public required Snowflake GuildId { get; init; }
[CommandOption(
"include-vc",
Description = "Include voice channels."
)]
[CommandOption("include-vc", Description = "Include voice channels.")]
public bool IncludeVoiceChannels { get; init; } = true;
[CommandOption(
"include-threads",
Description = "Which types of threads should be included.",
Converter = typeof(ThreadInclusionModeBindingConverter)
)]
public ThreadInclusionMode ThreadInclusionMode { get; init; } = ThreadInclusionMode.None;
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
var cancellationToken = console.RegisterCancellationHandler();
var channels = new List<Channel>();
await console.Output.WriteLineAsync("Fetching channels...");
var channels = (await Discord.GetGuildChannelsAsync(GuildId, cancellationToken))
.Where(c => c.Kind != ChannelKind.GuildCategory)
.Where(c => IncludeVoiceChannels || !c.Kind.IsVoice())
.ToArray();
// Regular channels
await foreach (var channel in Discord.GetGuildChannelsAsync(GuildId, cancellationToken))
{
if (channel.IsCategory)
continue;
await base.ExecuteAsync(console, channels);
if (!IncludeVoiceChannels && channel.IsVoice)
continue;
channels.Add(channel);
}
// Threads
if (ThreadInclusionMode != ThreadInclusionMode.None)
{
await foreach (
var thread in Discord.GetGuildThreadsAsync(
GuildId,
ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken
)
)
{
channels.Add(thread);
}
}
await ExportAsync(console, channels);
}
}
}
@@ -4,6 +4,8 @@ using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Cli.Commands.Converters;
using DiscordChatExporter.Cli.Commands.Shared;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions;
@@ -13,25 +15,28 @@ namespace DiscordChatExporter.Cli.Commands;
[Command("channels", Description = "Get the list of channels in a guild.")]
public class GetChannelsCommand : DiscordCommandBase
{
[CommandOption(
"guild",
'g',
Description = "Guild ID."
)]
[CommandOption("guild", 'g', Description = "Guild ID.")]
public required Snowflake GuildId { get; init; }
[CommandOption("include-vc", Description = "Include voice channels.")]
public bool IncludeVoiceChannels { get; init; } = true;
[CommandOption(
"include-threads",
Description = "Include threads in the output."
Description = "Which types of threads should be included.",
Converter = typeof(ThreadInclusionModeBindingConverter)
)]
public bool IncludeThreads { get; init; }
public ThreadInclusionMode ThreadInclusionMode { get; init; } = ThreadInclusionMode.None;
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
var cancellationToken = console.RegisterCancellationHandler();
var channels = (await Discord.GetGuildChannelsAsync(GuildId, cancellationToken))
.Where(c => c.Kind != ChannelKind.GuildCategory)
.Where(c => !c.IsCategory)
.Where(c => IncludeVoiceChannels || !c.IsVoice)
.OrderBy(c => c.Parent?.Position)
.ThenBy(c => c.Name)
.ToArray();
@@ -41,9 +46,20 @@ public class GetChannelsCommand : DiscordCommandBase
.OrderDescending()
.FirstOrDefault();
var threads = IncludeThreads
? (await Discord.GetGuildThreadsAsync(GuildId, cancellationToken)).OrderBy(c => c.Name).ToArray()
: Array.Empty<Channel>();
var threads =
ThreadInclusionMode != ThreadInclusionMode.None
? (
await Discord.GetGuildThreadsAsync(
GuildId,
ThreadInclusionMode == ThreadInclusionMode.All,
null,
null,
cancellationToken
)
)
.OrderBy(c => c.Name)
.ToArray()
: Array.Empty<Channel>();
foreach (var channel in channels)
{
@@ -56,9 +72,9 @@ public class GetChannelsCommand : DiscordCommandBase
using (console.WithForegroundColor(ConsoleColor.DarkGray))
await console.Output.WriteAsync(" | ");
// Channel category / name
// Channel name
using (console.WithForegroundColor(ConsoleColor.White))
await console.Output.WriteLineAsync($"{channel.Category} / {channel.Name}");
await console.Output.WriteLineAsync(channel.GetHierarchicalName());
var channelThreads = threads.Where(t => t.Parent?.Id == channel.Id).ToArray();
var channelThreadIdMaxLength = channelThreads
@@ -90,8 +106,10 @@ public class GetChannelsCommand : DiscordCommandBase
// Thread status
using (console.WithForegroundColor(ConsoleColor.White))
await console.Output.WriteLineAsync(channelThread.IsActive ? "Active" : "Archived");
await console.Output.WriteLineAsync(
channelThread.IsArchived ? "Archived" : "Active"
);
}
}
}
}
}
@@ -14,10 +14,13 @@ public class GetDirectChannelsCommand : DiscordCommandBase
{
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
var cancellationToken = console.RegisterCancellationHandler();
var channels = (await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken))
.Where(c => c.Kind != ChannelKind.GuildCategory)
var channels = (
await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken)
)
.OrderByDescending(c => c.LastMessageId)
.ThenBy(c => c.Name)
.ToArray();
@@ -38,9 +41,9 @@ public class GetDirectChannelsCommand : DiscordCommandBase
using (console.WithForegroundColor(ConsoleColor.DarkGray))
await console.Output.WriteAsync(" | ");
// Channel category / name
// Channel name
using (console.WithForegroundColor(ConsoleColor.White))
await console.Output.WriteLineAsync($"{channel.Category} / {channel.Name}");
await console.Output.WriteLineAsync(channel.GetHierarchicalName());
}
}
}
}
@@ -14,6 +14,8 @@ public class GetGuildsCommand : DiscordCommandBase
{
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
var cancellationToken = console.RegisterCancellationHandler();
var guilds = (await Discord.GetUserGuildsAsync(cancellationToken))
@@ -30,9 +32,7 @@ public class GetGuildsCommand : DiscordCommandBase
foreach (var guild in guilds)
{
// Guild ID
await console.Output.WriteAsync(
guild.Id.ToString().PadRight(guildIdMaxLength, ' ')
);
await console.Output.WriteAsync(guild.Id.ToString().PadRight(guildIdMaxLength, ' '));
// Separator
using (console.WithForegroundColor(ConsoleColor.DarkGray))
@@ -43,4 +43,4 @@ public class GetGuildsCommand : DiscordCommandBase
await console.Output.WriteLineAsync(guild.Name);
}
}
}
}
@@ -15,14 +15,18 @@ public class GuideCommand : ICommand
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("To get user token:");
console.Output.WriteLine(" * Automating user accounts is technically against TOS — USE AT YOUR OWN RISK!");
console.Output.WriteLine(
" * Automating user accounts is technically against TOS — USE AT YOUR OWN RISK!"
);
console.Output.WriteLine(" 1. Open Discord in your web browser and login");
console.Output.WriteLine(" 2. Open any server or direct message channel");
console.Output.WriteLine(" 3. Press Ctrl+Shift+I to show developer tools");
console.Output.WriteLine(" 4. Navigate to the Network tab");
console.Output.WriteLine(" 5. Press Ctrl+R to reload");
console.Output.WriteLine(" 6. Switch between random channels to trigger network requests");
console.Output.WriteLine(" 7. Search for a request containing \"messages?limit=50\" or similar");
console.Output.WriteLine(
" 7. Search for a request containing \"messages?limit=50\" or similar"
);
console.Output.WriteLine(" 8. Select the Headers tab on the right");
console.Output.WriteLine(" 9. Scroll down to the Request Headers section");
console.Output.WriteLine(" 10. Copy the value of the \"authorization\" header");
@@ -36,7 +40,9 @@ public class GuideCommand : ICommand
console.Output.WriteLine(" 2. Open your application's settings");
console.Output.WriteLine(" 3. Navigate to the Bot section on the left");
console.Output.WriteLine(" 4. Under Token click Copy");
console.Output.WriteLine(" * Your bot needs to have Message Content Intent enabled to read messages");
console.Output.WriteLine(
" * Your bot needs to have Message Content Intent enabled to read messages"
);
console.Output.WriteLine();
// Guild or channel ID
@@ -47,15 +53,21 @@ public class GuideCommand : ICommand
console.Output.WriteLine(" 2. Open Settings");
console.Output.WriteLine(" 3. Go to Advanced section");
console.Output.WriteLine(" 4. Enable Developer Mode");
console.Output.WriteLine(" 5. Right-click on the desired guild or channel and click Copy Server ID or Copy Channel ID");
console.Output.WriteLine(
" 5. Right-click on the desired guild or channel and click Copy Server ID or Copy Channel ID"
);
console.Output.WriteLine();
// Docs link
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("If you have questions or issues, please refer to the documentation:");
console.Output.WriteLine(
"If you have questions or issues, please refer to the documentation:"
);
using (console.WithForegroundColor(ConsoleColor.DarkCyan))
console.Output.WriteLine("https://github.com/Tyrrrz/DiscordChatExporter/blob/master/.docs");
console.Output.WriteLine(
"https://github.com/Tyrrrz/DiscordChatExporter/blob/master/.docs"
);
return default;
}
}
}
@@ -0,0 +1,8 @@
namespace DiscordChatExporter.Cli.Commands.Shared;
public enum ThreadInclusionMode
{
None,
Active,
All
}
@@ -7,6 +7,7 @@
<ItemGroup>
<PackageReference Include="CliFx" Version="2.3.4" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="Deorcify" Version="1.0.2" PrivateAssets="all" />
<PackageReference Include="DotnetRuntimeBootstrapper" Version="2.5.1" PrivateAssets="all" />
<PackageReference Include="Gress" Version="2.1.1" />
+1 -4
View File
@@ -1,6 +1,3 @@
using CliFx;
return await new CliApplicationBuilder()
.AddCommandsFromThisAssembly()
.Build()
.RunAsync(args);
return await new CliApplicationBuilder().AddCommandsFromThisAssembly().Build().RunAsync(args);
@@ -8,34 +8,42 @@ namespace DiscordChatExporter.Cli.Utils.Extensions;
internal static class ConsoleExtensions
{
public static IAnsiConsole CreateAnsiConsole(this IConsole console) =>
AnsiConsole.Create(new AnsiConsoleSettings
{
Ansi = AnsiSupport.Detect,
ColorSystem = ColorSystemSupport.Detect,
Out = new AnsiConsoleOutput(console.Output)
});
public static Progress CreateProgressTicker(this IConsole console) => console
.CreateAnsiConsole()
.Progress()
.AutoClear(false)
.AutoRefresh(true)
.HideCompleted(false)
.Columns(
new TaskDescriptionColumn {Alignment = Justify.Left},
new ProgressBarColumn(),
new PercentageColumn()
AnsiConsole.Create(
new AnsiConsoleSettings
{
Ansi = AnsiSupport.Detect,
ColorSystem = ColorSystemSupport.Detect,
Out = new AnsiConsoleOutput(console.Output)
}
);
public static Progress CreateProgressTicker(this IConsole console) =>
console
.CreateAnsiConsole()
.Progress()
.AutoClear(false)
.AutoRefresh(true)
.HideCompleted(false)
.Columns(
new TaskDescriptionColumn { Alignment = Justify.Left },
new ProgressBarColumn(),
new PercentageColumn()
);
public static async ValueTask StartTaskAsync(
this ProgressContext progressContext,
string description,
Func<ProgressTask, ValueTask> performOperationAsync)
Func<ProgressTask, ValueTask> performOperationAsync
)
{
// Description cannot be empty
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1133
var actualDescription = !string.IsNullOrWhiteSpace(description) ? description : "?";
var progressTask = progressContext.AddTask(
// Don't recognize random square brackets as style tags
Markup.Escape(description),
new ProgressTaskSettings {MaxValue = 1}
Markup.Escape(actualDescription),
new ProgressTaskSettings { MaxValue = 1 }
);
try
@@ -48,4 +56,4 @@ internal static class ConsoleExtensions
progressTask.StopTask();
}
}
}
}
@@ -15,30 +15,31 @@ public partial record Attachment(
string? Description,
int? Width,
int? Height,
FileSize FileSize) : IHasId
FileSize FileSize
) : IHasId
{
public string FileExtension => Path.GetExtension(FileName);
public bool IsImage =>
string.Equals(FileExtension, ".jpg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".jpeg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".png", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".gif", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".bmp", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".webp", StringComparison.OrdinalIgnoreCase);
string.Equals(FileExtension, ".jpg", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".jpeg", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".png", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".gif", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".bmp", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".webp", StringComparison.OrdinalIgnoreCase);
public bool IsVideo =>
string.Equals(FileExtension, ".gifv", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".mp4", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".webm", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".mov", StringComparison.OrdinalIgnoreCase);
string.Equals(FileExtension, ".gifv", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".mp4", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".webm", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".mov", StringComparison.OrdinalIgnoreCase);
public bool IsAudio =>
string.Equals(FileExtension, ".mp3", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".wav", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".ogg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".flac", StringComparison.OrdinalIgnoreCase) ||
string.Equals(FileExtension, ".m4a", StringComparison.OrdinalIgnoreCase);
string.Equals(FileExtension, ".mp3", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".wav", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".ogg", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".flac", StringComparison.OrdinalIgnoreCase)
|| string.Equals(FileExtension, ".m4a", StringComparison.OrdinalIgnoreCase);
public bool IsSpoiler => FileName.StartsWith("SPOILER_", StringComparison.Ordinal);
}
@@ -57,4 +58,4 @@ public partial record Attachment
return new Attachment(id, url, fileName, description, width, height, fileSize);
}
}
}
@@ -1,4 +1,5 @@
using System.Linq;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
@@ -16,28 +17,44 @@ public partial record Channel(
int? Position,
string? IconUrl,
string? Topic,
bool IsActive,
Snowflake? LastMessageId) : IHasId
bool IsArchived,
Snowflake? LastMessageId
) : IHasId
{
// Used for visual backwards-compatibility with old exports, where
// channels without a parent (i.e. mostly DM channels) or channels
// with an inaccessible parent (i.e. inside private categories) had
// a fallback category created for them.
public string Category => Parent?.Name ?? Kind switch
{
ChannelKind.GuildCategory => "Category",
ChannelKind.GuildTextChat => "Text",
ChannelKind.DirectTextChat => "Private",
ChannelKind.DirectGroupTextChat => "Group",
ChannelKind.GuildPrivateThread => "Private Thread",
ChannelKind.GuildPublicThread => "Public Thread",
ChannelKind.GuildNews => "News",
ChannelKind.GuildNewsThread => "News Thread",
_ => "Default"
};
public bool IsDirect => Kind is ChannelKind.DirectTextChat or ChannelKind.DirectGroupTextChat;
// Only needed for WPF data binding. Don't use anywhere else.
public bool IsVoice => Kind.IsVoice();
public bool IsGuild => !IsDirect;
public bool IsCategory => Kind == ChannelKind.GuildCategory;
public bool IsVoice => Kind is ChannelKind.GuildVoiceChat or ChannelKind.GuildStageVoice;
public bool IsThread =>
Kind
is ChannelKind.GuildNewsThread
or ChannelKind.GuildPublicThread
or ChannelKind.GuildPrivateThread;
public bool IsEmpty => LastMessageId is null;
public IEnumerable<Channel> GetParents()
{
var current = Parent;
while (current is not null)
{
yield return current;
current = current.Parent;
}
}
public Channel? TryGetRootParent() => GetParents().LastOrDefault();
public string GetHierarchicalName() =>
string.Join(" / ", GetParents().Reverse().Select(c => c.Name).Append(Name));
public bool MayHaveMessagesAfter(Snowflake messageId) => !IsEmpty && messageId < LastMessageId;
public bool MayHaveMessagesBefore(Snowflake messageId) => !IsEmpty && messageId > Id;
}
public partial record Channel
@@ -48,44 +65,39 @@ public partial record Channel
var kind = (ChannelKind)json.GetProperty("type").GetInt32();
var guildId =
json.GetPropertyOrNull("guild_id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse) ??
Guild.DirectMessages.Id;
json.GetPropertyOrNull("guild_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse) ?? Guild.DirectMessages.Id;
var name =
// Guild channel
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ??
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull()
// DM channel
json.GetPropertyOrNull("recipients")?
.EnumerateArrayOrNull()?
.Select(User.Parse)
?? json.GetPropertyOrNull("recipients")
?.EnumerateArrayOrNull()
?.Select(User.Parse)
.Select(u => u.DisplayName)
.Pipe(s => string.Join(", ", s)) ??
.Pipe(s => string.Join(", ", s))
// Fallback
id.ToString();
?? id.ToString();
var position =
positionHint ??
json.GetPropertyOrNull("position")?.GetInt32OrNull();
var position = positionHint ?? json.GetPropertyOrNull("position")?.GetInt32OrNull();
// Icons can only be set for group DM channels
var iconUrl = json
.GetPropertyOrNull("icon")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetChannelIconUrl(id, h));
var iconUrl = json.GetPropertyOrNull("icon")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetChannelIconUrl(id, h));
var topic = json.GetPropertyOrNull("topic")?.GetStringOrNull();
var isActive = !json
.GetPropertyOrNull("thread_metadata")?
.GetPropertyOrNull("archived")?
.GetBooleanOrNull() ?? true;
var isArchived =
json.GetPropertyOrNull("thread_metadata")
?.GetPropertyOrNull("archived")
?.GetBooleanOrNull() ?? false;
var lastMessageId = json
.GetPropertyOrNull("last_message_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var lastMessageId = json.GetPropertyOrNull("last_message_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
return new Channel(
id,
@@ -96,8 +108,8 @@ public partial record Channel
position,
iconUrl,
topic,
isActive,
isArchived,
lastMessageId
);
}
}
}
@@ -16,15 +16,3 @@ public enum ChannelKind
GuildDirectory = 14,
GuildForum = 15
}
public static class ChannelKindExtensions
{
public static bool IsDirect(this ChannelKind kind) =>
kind is ChannelKind.DirectTextChat or ChannelKind.DirectGroupTextChat;
public static bool IsGuild(this ChannelKind kind) =>
!kind.IsDirect();
public static bool IsVoice(this ChannelKind kind) =>
kind is ChannelKind.GuildVoiceChat or ChannelKind.GuildStageVoice;
}
@@ -41,10 +41,13 @@ public readonly partial record struct FileSize(long TotalBytes)
[ExcludeFromCodeCoverage]
public override string ToString() =>
string.Create(CultureInfo.InvariantCulture, $"{GetLargestWholeNumberValue():0.##} {GetLargestWholeNumberSymbol()}");
string.Create(
CultureInfo.InvariantCulture,
$"{GetLargestWholeNumberValue():0.##} {GetLargestWholeNumberSymbol()}"
);
}
public partial record struct FileSize
{
public static FileSize FromBytes(long bytes) => new(bytes);
}
}
@@ -3,4 +3,4 @@
public interface IHasId
{
Snowflake Id { get; }
}
}
@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using System.Linq;
using DiscordChatExporter.Core.Utils.Extensions;
@@ -21,7 +22,7 @@ public static class ImageCdn
var twemojiId = string.Join(
"-",
filteredRunes.Select(r => r.Value.ToString("x"))
filteredRunes.Select(r => r.Value.ToString("x", CultureInfo.InvariantCulture))
);
return $"https://cdn.jsdelivr.net/gh/twitter/twemoji@latest/assets/svg/{twemojiId}.svg";
@@ -50,11 +51,16 @@ public static class ImageCdn
public static string GetFallbackUserAvatarUrl(int index = 0) =>
$"https://cdn.discordapp.com/embed/avatars/{index}.png";
public static string GetMemberAvatarUrl(Snowflake guildId, Snowflake userId, string avatarHash, int size = 512) =>
public static string GetMemberAvatarUrl(
Snowflake guildId,
Snowflake userId,
string avatarHash,
int size = 512
) =>
avatarHash.StartsWith("a_", StringComparison.Ordinal)
? $"https://cdn.discordapp.com/guilds/{guildId}/users/{userId}/avatars/{avatarHash}.gif?size={size}"
: $"https://cdn.discordapp.com/guilds/{guildId}/users/{userId}/avatars/{avatarHash}.png?size={size}";
public static string GetStickerUrl(Snowflake stickerId, string format = "png") =>
$"https://cdn.discordapp.com/stickers/{stickerId}.{format}";
}
}
@@ -21,7 +21,8 @@ public partial record Embed(
EmbedImage? Thumbnail,
IReadOnlyList<EmbedImage> Images,
EmbedVideo? Video,
EmbedFooter? Footer)
EmbedFooter? Footer
)
{
// Embeds can only have one image according to the API model,
// but the client can render multiple images in some cases.
@@ -41,24 +42,25 @@ public partial record Embed
var title = json.GetPropertyOrNull("title")?.GetStringOrNull();
var kind =
json.GetPropertyOrNull("type")?.GetStringOrNull()?.ParseEnumOrNull<EmbedKind>() ??
EmbedKind.Rich;
json.GetPropertyOrNull("type")?.GetStringOrNull()?.ParseEnumOrNull<EmbedKind>()
?? EmbedKind.Rich;
var url = json.GetPropertyOrNull("url")?.GetNonWhiteSpaceStringOrNull();
var timestamp = json.GetPropertyOrNull("timestamp")?.GetDateTimeOffsetOrNull();
var color = json
.GetPropertyOrNull("color")?
.GetInt32OrNull()?
.Pipe(System.Drawing.Color.FromArgb)
var color = json.GetPropertyOrNull("color")
?.GetInt32OrNull()
?.Pipe(System.Drawing.Color.FromArgb)
.ResetAlpha();
var author = json.GetPropertyOrNull("author")?.Pipe(EmbedAuthor.Parse);
var description = json.GetPropertyOrNull("description")?.GetStringOrNull();
var fields =
json.GetPropertyOrNull("fields")?.EnumerateArrayOrNull()?.Select(EmbedField.Parse).ToArray() ??
Array.Empty<EmbedField>();
json.GetPropertyOrNull("fields")
?.EnumerateArrayOrNull()
?.Select(EmbedField.Parse)
.ToArray() ?? Array.Empty<EmbedField>();
var thumbnail = json.GetPropertyOrNull("thumbnail")?.Pipe(EmbedImage.Parse);
@@ -70,8 +72,10 @@ public partial record Embed
// with this by merging related embeds at the end of the message parsing process.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/695
var images =
json.GetPropertyOrNull("image")?.Pipe(EmbedImage.Parse).ToSingletonEnumerable().ToArray() ??
Array.Empty<EmbedImage>();
json.GetPropertyOrNull("image")
?.Pipe(EmbedImage.Parse)
.ToSingletonEnumerable()
.ToArray() ?? Array.Empty<EmbedImage>();
var video = json.GetPropertyOrNull("video")?.Pipe(EmbedVideo.Parse);
@@ -92,4 +96,4 @@ public partial record Embed
footer
);
}
}
}
@@ -4,11 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-author-structure
public record EmbedAuthor(
string? Name,
string? Url,
string? IconUrl,
string? IconProxyUrl)
public record EmbedAuthor(string? Name, string? Url, string? IconUrl, string? IconProxyUrl)
{
public static EmbedAuthor Parse(JsonElement json)
{
@@ -19,4 +15,4 @@ public record EmbedAuthor(
return new EmbedAuthor(name, url, iconUrl, iconProxyUrl);
}
}
}
@@ -4,10 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-field-structure
public record EmbedField(
string Name,
string Value,
bool IsInline)
public record EmbedField(string Name, string Value, bool IsInline)
{
public static EmbedField Parse(JsonElement json)
{
@@ -17,4 +14,4 @@ public record EmbedField(
return new EmbedField(name, value, isInline);
}
}
}
@@ -4,10 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-footer-structure
public record EmbedFooter(
string Text,
string? IconUrl,
string? IconProxyUrl)
public record EmbedFooter(string Text, string? IconUrl, string? IconProxyUrl)
{
public static EmbedFooter Parse(JsonElement json)
{
@@ -17,4 +14,4 @@ public record EmbedFooter(
return new EmbedFooter(text, iconUrl, iconProxyUrl);
}
}
}
@@ -4,11 +4,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure
public record EmbedImage(
string? Url,
string? ProxyUrl,
int? Width,
int? Height)
public record EmbedImage(string? Url, string? ProxyUrl, int? Width, int? Height)
{
public static EmbedImage Parse(JsonElement json)
{
@@ -19,4 +15,4 @@ public record EmbedImage(
return new EmbedImage(url, proxyUrl, width, height);
}
}
}
@@ -8,4 +8,4 @@ public enum EmbedKind
Video,
Gifv,
Link
}
}
@@ -4,11 +4,7 @@ using System.Text.Json;
namespace DiscordChatExporter.Core.Discord.Data.Embeds;
// https://discord.com/developers/docs/resources/channel#embed-object-embed-video-structure
public record EmbedVideo(
string? Url,
string? ProxyUrl,
int? Width,
int? Height)
public record EmbedVideo(string? Url, string? ProxyUrl, int? Width, int? Height)
{
public static EmbedVideo Parse(JsonElement json)
{
@@ -19,4 +15,4 @@ public record EmbedVideo(
return new EmbedVideo(url, proxyUrl, width, height);
}
}
}
@@ -12,7 +12,9 @@ public partial record SpotifyTrackEmbedProjection
private static string? TryParseTrackId(string embedUrl)
{
// https://open.spotify.com/track/1LHZMWefF9502NPfArRfvP?si=3efac6ce9be04f0a
var trackId = Regex.Match(embedUrl, @"spotify\.com/track/(.*?)(?:\?|&|/|$)").Groups[1].Value;
var trackId = Regex.Match(embedUrl, @"spotify\.com/track/(.*?)(?:\?|&|/|$)").Groups[
1
].Value;
if (!string.IsNullOrWhiteSpace(trackId))
return trackId;
@@ -33,4 +35,4 @@ public partial record SpotifyTrackEmbedProjection
return new SpotifyTrackEmbedProjection(trackId);
}
}
}
@@ -21,4 +21,4 @@ public partial record YouTubeVideoEmbedProjection
return new YouTubeVideoEmbedProjection(videoId);
}
}
}
+10 -13
View File
@@ -14,12 +14,11 @@ public partial record Emoji(
// Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂)
string Name,
bool IsAnimated,
string ImageUrl)
string ImageUrl
)
{
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => Id is not null
? Name
: EmojiIndex.TryGetCode(Name) ?? Name;
public string Code => Id is not null ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
}
public partial record Emoji
@@ -39,19 +38,17 @@ public partial record Emoji
public static Emoji Parse(JsonElement json)
{
var id = json.GetPropertyOrNull("id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse);
var id = json.GetPropertyOrNull("id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
// Names may be missing on custom emoji within reactions
var name = json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ?? "Unknown Emoji";
var name =
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ?? "Unknown Emoji";
var isAnimated = json.GetPropertyOrNull("animated")?.GetBooleanOrNull() ?? false;
var imageUrl = GetImageUrl(id, name, isAnimated);
return new Emoji(
id,
name,
isAnimated,
imageUrl
);
return new Emoji(id, name, isAnimated, imageUrl);
}
}
}
File diff suppressed because it is too large Load Diff
+12 -12
View File
@@ -6,14 +6,16 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data;
// https://discord.com/developers/docs/resources/guild#guild-object
public record Guild(Snowflake Id, string Name, string IconUrl) : IHasId
public partial record Guild(Snowflake Id, string Name, string IconUrl) : IHasId
{
public bool IsDirect => Id == DirectMessages.Id;
}
public partial record Guild
{
// Direct messages are encapsulated within a special pseudo-guild for consistency
public static Guild DirectMessages { get; } = new(
Snowflake.Zero,
"Direct Messages",
ImageCdn.GetFallbackUserAvatarUrl()
);
public static Guild DirectMessages { get; } =
new(Snowflake.Zero, "Direct Messages", ImageCdn.GetFallbackUserAvatarUrl());
public static Guild Parse(JsonElement json)
{
@@ -21,12 +23,10 @@ public record Guild(Snowflake Id, string Name, string IconUrl) : IHasId
var name = json.GetProperty("name").GetNonNullString();
var iconUrl =
json
.GetPropertyOrNull("icon")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetGuildIconUrl(id, h)) ??
ImageCdn.GetFallbackUserAvatarUrl();
json.GetPropertyOrNull("icon")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetGuildIconUrl(id, h)) ?? ImageCdn.GetFallbackUserAvatarUrl();
return new Guild(id, name, iconUrl);
}
}
}
@@ -15,4 +15,4 @@ public record Interaction(Snowflake Id, string Name, User User)
return new Interaction(id, name, user);
}
}
}
@@ -6,10 +6,7 @@ using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data;
// https://discord.com/developers/docs/resources/invite#invite-object
public record Invite(
string Code,
Guild Guild,
Channel? Channel)
public record Invite(string Code, Guild Guild, Channel? Channel)
{
public static string? TryGetCodeFromUrl(string url) =>
Regex.Match(url, @"^https?://discord\.gg/(\w+)/?$").Groups[1].Value.NullIfWhiteSpace();
@@ -22,4 +19,4 @@ public record Invite(
return new Invite(code, guild, channel);
}
}
}
+13 -18
View File
@@ -13,7 +13,8 @@ public partial record Member(
User User,
string? DisplayName,
string? AvatarUrl,
IReadOnlyList<Snowflake> RoleIds) : IHasId
IReadOnlyList<Snowflake> RoleIds
) : IHasId
{
public Snowflake Id => User.Id;
}
@@ -28,25 +29,19 @@ public partial record Member
var user = json.GetProperty("user").Pipe(User.Parse);
var displayName = json.GetPropertyOrNull("nick")?.GetNonWhiteSpaceStringOrNull();
var roleIds = json
.GetPropertyOrNull("roles")?
.EnumerateArray()
.Select(j => j.GetNonWhiteSpaceString())
.Select(Snowflake.Parse)
.ToArray() ?? Array.Empty<Snowflake>();
var roleIds =
json.GetPropertyOrNull("roles")
?.EnumerateArray()
.Select(j => j.GetNonWhiteSpaceString())
.Select(Snowflake.Parse)
.ToArray() ?? Array.Empty<Snowflake>();
var avatarUrl = guildId is not null
? json
.GetPropertyOrNull("avatar")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetMemberAvatarUrl(guildId.Value, user.Id, h))
? json.GetPropertyOrNull("avatar")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetMemberAvatarUrl(guildId.Value, user.Id, h))
: null;
return new Member(
user,
displayName,
avatarUrl,
roleIds
);
return new Member(user, displayName, avatarUrl, roleIds);
}
}
}
@@ -27,9 +27,16 @@ public partial record Message(
IReadOnlyList<User> MentionedUsers,
MessageReference? Reference,
Message? ReferencedMessage,
Interaction? Interaction) : IHasId
Interaction? Interaction
) : IHasId
{
public bool IsReplyLike => Kind == MessageKind.Reply || Interaction is not null;
public bool IsSystemNotification =>
Kind is >= MessageKind.RecipientAdd and <= MessageKind.ThreadCreated;
public bool IsReply => Kind == MessageKind.Reply;
// App interactions are rendered as replies in the Discord client, but they are not actually replies
public bool IsReplyLike => IsReply || Interaction is not null;
public IEnumerable<User> GetReferencedUsers()
{
@@ -70,22 +77,26 @@ public partial record Message
// Find embeds with the same URL that only contain a single image and nothing else
var trailingEmbeds = embeds
.Skip(i + 1)
.TakeWhile(e =>
e.Url == embed.Url &&
e.Timestamp is null &&
e.Author is null &&
e.Color is null &&
string.IsNullOrWhiteSpace(e.Description) &&
!e.Fields.Any() &&
e.Images.Count == 1 &&
e.Footer is null
.TakeWhile(
e =>
e.Url == embed.Url
&& e.Timestamp is null
&& e.Author is null
&& e.Color is null
&& string.IsNullOrWhiteSpace(e.Description)
&& !e.Fields.Any()
&& e.Images.Count == 1
&& e.Footer is null
)
.ToArray();
if (trailingEmbeds.Any())
{
// Concatenate all images into one embed
var images = embed.Images.Concat(trailingEmbeds.SelectMany(e => e.Images)).ToArray();
var images = embed.Images
.Concat(trailingEmbeds.SelectMany(e => e.Images))
.ToArray();
normalizedEmbeds.Add(embed with { Images = images });
i += trailingEmbeds.Length;
@@ -108,42 +119,49 @@ public partial record Message
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var kind = (MessageKind)json.GetProperty("type").GetInt32();
var flags = (MessageFlags?)json.GetPropertyOrNull("flags")?.GetInt32OrNull() ?? MessageFlags.None;
var flags =
(MessageFlags?)json.GetPropertyOrNull("flags")?.GetInt32OrNull() ?? MessageFlags.None;
var author = json.GetProperty("author").Pipe(User.Parse);
var timestamp = json.GetProperty("timestamp").GetDateTimeOffset();
var editedTimestamp = json.GetPropertyOrNull("edited_timestamp")?.GetDateTimeOffsetOrNull();
var callEndedTimestamp = json
.GetPropertyOrNull("call")?
.GetPropertyOrNull("ended_timestamp")?
.GetDateTimeOffsetOrNull();
var callEndedTimestamp = json.GetPropertyOrNull("call")
?.GetPropertyOrNull("ended_timestamp")
?.GetDateTimeOffsetOrNull();
var isPinned = json.GetPropertyOrNull("pinned")?.GetBooleanOrNull() ?? false;
var content = json.GetPropertyOrNull("content")?.GetStringOrNull() ?? "";
var attachments =
json.GetPropertyOrNull("attachments")?.EnumerateArrayOrNull()?.Select(Attachment.Parse).ToArray() ??
Array.Empty<Attachment>();
json.GetPropertyOrNull("attachments")
?.EnumerateArrayOrNull()
?.Select(Attachment.Parse)
.ToArray() ?? Array.Empty<Attachment>();
var embeds = NormalizeEmbeds(
json.GetPropertyOrNull("embeds")?.EnumerateArrayOrNull()?.Select(Embed.Parse).ToArray() ??
Array.Empty<Embed>()
json.GetPropertyOrNull("embeds")?.EnumerateArrayOrNull()?.Select(Embed.Parse).ToArray()
?? Array.Empty<Embed>()
);
var stickers =
json.GetPropertyOrNull("sticker_items")?.EnumerateArrayOrNull()?.Select(Sticker.Parse).ToArray() ??
Array.Empty<Sticker>();
json.GetPropertyOrNull("sticker_items")
?.EnumerateArrayOrNull()
?.Select(Sticker.Parse)
.ToArray() ?? Array.Empty<Sticker>();
var reactions =
json.GetPropertyOrNull("reactions")?.EnumerateArrayOrNull()?.Select(Reaction.Parse).ToArray() ??
Array.Empty<Reaction>();
json.GetPropertyOrNull("reactions")
?.EnumerateArrayOrNull()
?.Select(Reaction.Parse)
.ToArray() ?? Array.Empty<Reaction>();
var mentionedUsers =
json.GetPropertyOrNull("mentions")?.EnumerateArrayOrNull()?.Select(User.Parse).ToArray() ??
Array.Empty<User>();
json.GetPropertyOrNull("mentions")?.EnumerateArrayOrNull()?.Select(User.Parse).ToArray()
?? Array.Empty<User>();
var messageReference = json.GetPropertyOrNull("message_reference")?.Pipe(MessageReference.Parse);
var messageReference = json.GetPropertyOrNull("message_reference")
?.Pipe(MessageReference.Parse);
var referencedMessage = json.GetPropertyOrNull("referenced_message")?.Pipe(Parse);
var interaction = json.GetPropertyOrNull("interaction")?.Pipe(Interaction.Parse);
@@ -167,4 +185,4 @@ public partial record Message
interaction
);
}
}
}
@@ -15,4 +15,4 @@ public enum MessageFlags
HasThread = 32,
Ephemeral = 64,
Loading = 128
}
}
@@ -14,8 +14,3 @@ public enum MessageKind
ThreadCreated = 18,
Reply = 19
}
public static class MessageKindExtensions
{
public static bool IsSystemNotification(this MessageKind kind) => (int)kind is >= 1 and <= 18;
}
@@ -9,21 +9,18 @@ public record MessageReference(Snowflake? MessageId, Snowflake? ChannelId, Snowf
{
public static MessageReference Parse(JsonElement json)
{
var messageId = json
.GetPropertyOrNull("message_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var messageId = json.GetPropertyOrNull("message_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
var channelId = json
.GetPropertyOrNull("channel_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var channelId = json.GetPropertyOrNull("channel_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
var guildId = json
.GetPropertyOrNull("guild_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var guildId = json.GetPropertyOrNull("guild_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
return new MessageReference(messageId, channelId, guildId);
}
}
}
@@ -13,4 +13,4 @@ public record Reaction(Emoji Emoji, int Count)
return new Reaction(emoji, count);
}
}
}
@@ -15,13 +15,12 @@ public record Role(Snowflake Id, string Name, int Position, Color? Color) : IHas
var name = json.GetProperty("name").GetNonNullString();
var position = json.GetProperty("position").GetInt32();
var color = json
.GetPropertyOrNull("color")?
.GetInt32OrNull()?
.Pipe(System.Drawing.Color.FromArgb)
var color = json.GetPropertyOrNull("color")
?.GetInt32OrNull()
?.Pipe(System.Drawing.Color.FromArgb)
.ResetAlpha()
.NullIf(c => c.ToRgb() <= 0);
return new Role(id, name, position, color);
}
}
}
@@ -15,14 +15,17 @@ public record Sticker(Snowflake Id, string Name, StickerFormat Format, string So
var name = json.GetProperty("name").GetNonNullString();
var format = (StickerFormat)json.GetProperty("format_type").GetInt32();
var sourceUrl = ImageCdn.GetStickerUrl(id, format switch
{
StickerFormat.Png => "png",
StickerFormat.Apng => "png",
StickerFormat.Lottie => "json",
_ => throw new InvalidOperationException($"Unknown sticker format '{format}'.")
});
var sourceUrl = ImageCdn.GetStickerUrl(
id,
format switch
{
StickerFormat.Png => "png",
StickerFormat.Apng => "png",
StickerFormat.Lottie => "json",
_ => throw new InvalidOperationException($"Unknown sticker format '{format}'.")
}
);
return new Sticker(id, name, format, sourceUrl);
}
}
}
@@ -5,4 +5,4 @@ public enum StickerFormat
Png = 1,
Apng = 2,
Lottie = 3
}
}
+15 -18
View File
@@ -15,18 +15,16 @@ public partial record User(
int? Discriminator,
string Name,
string DisplayName,
string AvatarUrl) : IHasId
string AvatarUrl
) : IHasId
{
public string DiscriminatorFormatted => Discriminator is not null
? $"{Discriminator:0000}"
: "0000";
public string DiscriminatorFormatted =>
Discriminator is not null ? $"{Discriminator:0000}" : "0000";
// This effectively represents the user's true identity.
// In the old system, this is formed from the username and discriminator.
// In the new system, the username is already the user's unique identifier.
public string FullName => Discriminator is not null
? $"{Name}#{DiscriminatorFormatted}"
: Name;
public string FullName => Discriminator is not null ? $"{Name}#{DiscriminatorFormatted}" : Name;
}
public partial record User
@@ -36,24 +34,23 @@ public partial record User
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var isBot = json.GetPropertyOrNull("bot")?.GetBooleanOrNull() ?? false;
var discriminator = json
.GetPropertyOrNull("discriminator")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(int.Parse)
var discriminator = json.GetPropertyOrNull("discriminator")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(int.Parse)
.NullIfDefault();
var name = json.GetProperty("username").GetNonNullString();
var displayName = json.GetPropertyOrNull("global_name")?.GetNonWhiteSpaceStringOrNull() ?? name;
var displayName =
json.GetPropertyOrNull("global_name")?.GetNonWhiteSpaceStringOrNull() ?? name;
var avatarIndex = discriminator % 5 ?? (int)((id.Value >> 22) % 6);
var avatarUrl =
json
.GetPropertyOrNull("avatar")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(h => ImageCdn.GetUserAvatarUrl(id, h)) ??
ImageCdn.GetFallbackUserAvatarUrl(avatarIndex);
json.GetPropertyOrNull("avatar")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(h => ImageCdn.GetUserAvatarUrl(id, h))
?? ImageCdn.GetFallbackUserAvatarUrl(avatarIndex);
return new User(id, isBot, discriminator, name, displayName, avatarUrl);
}
}
}
+298 -117
View File
@@ -30,48 +30,46 @@ public class DiscordClient
private async ValueTask<HttpResponseMessage> GetResponseAsync(
string url,
TokenKind tokenKind,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
return await Http.ResponseResiliencePolicy.ExecuteAsync(async innerCancellationToken =>
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
// Don't validate because the token can have special characters
// https://github.com/Tyrrrz/DiscordChatExporter/issues/828
request.Headers.TryAddWithoutValidation(
"Authorization",
tokenKind == TokenKind.Bot
? $"Bot {_token}"
: _token
);
var response = await Http.Client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
innerCancellationToken
);
// If this was the last request available before hitting the rate limit,
// wait out the reset time so that future requests can succeed.
// This may add an unnecessary delay in case the user doesn't intend to
// make any more requests, but implementing a smarter solution would
// require properly keeping track of Discord's global/per-route/per-resource
// rate limits and that's just way too much effort.
// https://discord.com/developers/docs/topics/rate-limits
var remainingRequestCount = response
.Headers
.TryGetValue("X-RateLimit-Remaining")?
.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture));
var resetAfterDelay = response
.Headers
.TryGetValue("X-RateLimit-Reset-After")?
.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture))
.Pipe(TimeSpan.FromSeconds);
if (remainingRequestCount <= 0 && resetAfterDelay is not null)
return await Http.ResponseResiliencePipeline.ExecuteAsync(
async innerCancellationToken =>
{
var delay =
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
// Don't validate because the token can have special characters
// https://github.com/Tyrrrz/DiscordChatExporter/issues/828
request.Headers.TryAddWithoutValidation(
"Authorization",
tokenKind == TokenKind.Bot ? $"Bot {_token}" : _token
);
var response = await Http.Client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
innerCancellationToken
);
// If this was the last request available before hitting the rate limit,
// wait out the reset time so that future requests can succeed.
// This may add an unnecessary delay in case the user doesn't intend to
// make any more requests, but implementing a smarter solution would
// require properly keeping track of Discord's global/per-route/per-resource
// rate limits and that's just way too much effort.
// https://discord.com/developers/docs/topics/rate-limits
var remainingRequestCount = response.Headers
.TryGetValue("X-RateLimit-Remaining")
?.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture));
var resetAfterDelay = response.Headers
.TryGetValue("X-RateLimit-Reset-After")
?.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture))
.Pipe(TimeSpan.FromSeconds);
if (remainingRequestCount <= 0 && resetAfterDelay is not null)
{
var delay =
// Adding a small buffer to the reset time reduces the chance of getting
// rate limited again, because it allows for more requests to be released.
(resetAfterDelay.Value + TimeSpan.FromSeconds(1))
@@ -79,14 +77,18 @@ public class DiscordClient
// is not actually enforced by the server. So we cap it at a reasonable value.
.Clamp(TimeSpan.Zero, TimeSpan.FromSeconds(60));
await Task.Delay(delay, innerCancellationToken);
}
await Task.Delay(delay, innerCancellationToken);
}
return response;
}, cancellationToken);
return response;
},
cancellationToken
);
}
private async ValueTask<TokenKind> GetTokenKindAsync(CancellationToken cancellationToken = default)
private async ValueTask<TokenKind> GetTokenKindAsync(
CancellationToken cancellationToken = default
)
{
// Try authenticating as a user
using var userResponse = await GetResponseAsync(
@@ -113,7 +115,8 @@ public class DiscordClient
private async ValueTask<HttpResponseMessage> GetResponseAsync(
string url,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var tokenKind = _resolvedTokenKind ??= await GetTokenKindAsync(cancellationToken);
return await GetResponseAsync(url, tokenKind, cancellationToken);
@@ -121,7 +124,8 @@ public class DiscordClient
private async ValueTask<JsonElement> GetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
using var response = await GetResponseAsync(url, cancellationToken);
@@ -129,26 +133,30 @@ public class DiscordClient
{
throw response.StatusCode switch
{
HttpStatusCode.Unauthorized => throw new DiscordChatExporterException(
"Authentication token is invalid.",
true
),
HttpStatusCode.Unauthorized
=> throw new DiscordChatExporterException(
"Authentication token is invalid.",
true
),
HttpStatusCode.Forbidden => throw new DiscordChatExporterException(
$"Request to '{url}' failed: forbidden."
),
HttpStatusCode.Forbidden
=> throw new DiscordChatExporterException(
$"Request to '{url}' failed: forbidden."
),
HttpStatusCode.NotFound => throw new DiscordChatExporterException(
$"Request to '{url}' failed: not found."
),
HttpStatusCode.NotFound
=> throw new DiscordChatExporterException(
$"Request to '{url}' failed: not found."
),
_ => throw new DiscordChatExporterException(
$"""
_
=> throw new DiscordChatExporterException(
$"""
Request to '{url}' failed: {response.StatusCode.ToString().ToSpaceSeparatedWords().ToLowerInvariant()}.
Response content: {await response.Content.ReadAsStringAsync(cancellationToken)}
""",
true
)
true
)
};
}
@@ -157,7 +165,8 @@ public class DiscordClient
private async ValueTask<JsonElement?> TryGetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
using var response = await GetResponseAsync(url, cancellationToken);
return response.IsSuccessStatusCode
@@ -167,14 +176,16 @@ public class DiscordClient
public async ValueTask<User?> TryGetUserAsync(
Snowflake userId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var response = await TryGetJsonResponseAsync($"users/{userId}", cancellationToken);
return response?.Pipe(User.Parse);
}
public async IAsyncEnumerable<Guild> GetUserGuildsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
yield return Guild.DirectMessages;
@@ -189,24 +200,25 @@ public class DiscordClient
var response = await GetJsonResponseAsync(url, cancellationToken);
var isEmpty = true;
var count = 0;
foreach (var guildJson in response.EnumerateArray())
{
var guild = Guild.Parse(guildJson);
yield return guild;
currentAfter = guild.Id;
isEmpty = false;
count++;
}
if (isEmpty)
if (count <= 0)
yield break;
}
}
public async ValueTask<Guild> GetGuildAsync(
Snowflake guildId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
return Guild.DirectMessages;
@@ -217,7 +229,8 @@ public class DiscordClient
public async IAsyncEnumerable<Channel> GetGuildChannelsAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
{
@@ -227,7 +240,10 @@ public class DiscordClient
}
else
{
var response = await GetJsonResponseAsync($"guilds/{guildId}/channels", cancellationToken);
var response = await GetJsonResponseAsync(
$"guilds/{guildId}/channels",
cancellationToken
);
var channelsJson = response
.EnumerateArray()
@@ -247,9 +263,9 @@ public class DiscordClient
foreach (var channelJson in channelsJson)
{
var parent = channelJson
.GetPropertyOrNull("parent_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse)
.GetPropertyOrNull("parent_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse)
.Pipe(parentsById.GetValueOrDefault);
yield return Channel.Parse(channelJson, parent, position);
@@ -260,14 +276,35 @@ public class DiscordClient
public async IAsyncEnumerable<Channel> GetGuildThreadsAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
bool includeArchived = false,
Snowflake? before = null,
Snowflake? after = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
yield break;
var tokenKind = _resolvedTokenKind ??= await GetTokenKindAsync(cancellationToken);
var channels = await GetGuildChannelsAsync(guildId, cancellationToken);
var channels = (await GetGuildChannelsAsync(guildId, cancellationToken))
// Categories cannot have threads
.Where(c => !c.IsCategory)
// Voice channels cannot have threads
.Where(c => !c.IsVoice)
// Empty channels cannot have threads
.Where(c => !c.IsEmpty)
// If the 'before' boundary is specified, skip channels that don't have messages
// for that range, because thread-start event should always be accompanied by a message.
// Note that we don't perform a similar check for the 'after' boundary, because
// threads may have messages in range, even if the parent channel doesn't.
.Where(c => before is null || c.MayHaveMessagesBefore(before.Value))
.ToArray();
// User accounts can only fetch threads using the search endpoint
if (tokenKind == TokenKind.User)
{
// Active threads
foreach (var channel in channels)
{
var currentOffset = 0;
@@ -275,24 +312,94 @@ public class DiscordClient
{
var url = new UrlBuilder()
.SetPath($"channels/{channel.Id}/threads/search")
.SetQueryParameter("sort_by", "last_message_time")
.SetQueryParameter("sort_order", "desc")
.SetQueryParameter("archived", "false")
.SetQueryParameter("offset", currentOffset.ToString())
.Build();
// Can be null on channels that the user cannot access
// Can be null on channels that the user cannot access or channels without threads
var response = await TryGetJsonResponseAsync(url, cancellationToken);
if (response is null)
break;
foreach (var threadJson in response.Value.GetProperty("threads").EnumerateArray())
var breakOuter = false;
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
{
yield return Channel.Parse(threadJson, channel);
var thread = Channel.Parse(threadJson, channel);
// If the 'after' boundary is specified, we can break early,
// because threads are sorted by last message time.
if (after is not null && !thread.MayHaveMessagesAfter(after.Value))
{
breakOuter = true;
break;
}
yield return thread;
currentOffset++;
}
if (breakOuter)
break;
if (!response.Value.GetProperty("has_more").GetBoolean())
break;
}
}
// Archived threads
if (includeArchived)
{
foreach (var channel in channels)
{
var currentOffset = 0;
while (true)
{
var url = new UrlBuilder()
.SetPath($"channels/{channel.Id}/threads/search")
.SetQueryParameter("sort_by", "last_message_time")
.SetQueryParameter("sort_order", "desc")
.SetQueryParameter("archived", "true")
.SetQueryParameter("offset", currentOffset.ToString())
.Build();
// Can be null on channels that the user cannot access or channels without threads
var response = await TryGetJsonResponseAsync(url, cancellationToken);
if (response is null)
break;
var breakOuter = false;
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
{
var thread = Channel.Parse(threadJson, channel);
// If the 'after' boundary is specified, we can break early,
// because threads are sorted by last message time.
if (after is not null && !thread.MayHaveMessagesAfter(after.Value))
{
breakOuter = true;
break;
}
yield return thread;
currentOffset++;
}
if (breakOuter)
break;
if (!response.Value.GetProperty("has_more").GetBoolean())
break;
}
}
}
}
// Bot accounts can only fetch threads using the threads endpoint
else
@@ -301,41 +408,61 @@ public class DiscordClient
{
var parentsById = channels.ToDictionary(c => c.Id);
var response = await GetJsonResponseAsync($"guilds/{guildId}/threads/active", cancellationToken);
var response = await GetJsonResponseAsync(
$"guilds/{guildId}/threads/active",
cancellationToken
);
foreach (var threadJson in response.GetProperty("threads").EnumerateArray())
{
var parent = threadJson
.GetPropertyOrNull("parent_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse)
.GetPropertyOrNull("parent_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse)
.Pipe(parentsById.GetValueOrDefault);
yield return Channel.Parse(threadJson, parent);
}
}
foreach (var channel in channels)
// Archived threads
if (includeArchived)
{
// Public archived threads
foreach (var channel in channels)
{
var response = await GetJsonResponseAsync(
$"channels/{channel.Id}/threads/archived/public",
cancellationToken
);
// Public archived threads
{
// Can be null on certain channels
var response = await TryGetJsonResponseAsync(
$"channels/{channel.Id}/threads/archived/public",
cancellationToken
);
foreach (var threadJson in response.GetProperty("threads").EnumerateArray())
yield return Channel.Parse(threadJson, channel);
}
if (response is null)
continue;
// Private archived threads
{
var response = await GetJsonResponseAsync(
$"channels/{channel.Id}/threads/archived/private",
cancellationToken
);
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
yield return Channel.Parse(threadJson, channel);
}
foreach (var threadJson in response.GetProperty("threads").EnumerateArray())
yield return Channel.Parse(threadJson, channel);
// Private archived threads
{
// Can be null on certain channels
var response = await TryGetJsonResponseAsync(
$"channels/{channel.Id}/threads/archived/private",
cancellationToken
);
if (response is null)
continue;
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
yield return Channel.Parse(threadJson, channel);
}
}
}
}
@@ -343,7 +470,8 @@ public class DiscordClient
public async IAsyncEnumerable<Role> GetGuildRolesAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
yield break;
@@ -356,18 +484,23 @@ public class DiscordClient
public async ValueTask<Member?> TryGetGuildMemberAsync(
Snowflake guildId,
Snowflake memberId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (guildId == Guild.DirectMessages.Id)
return null;
var response = await TryGetJsonResponseAsync($"guilds/{guildId}/members/{memberId}", cancellationToken);
var response = await TryGetJsonResponseAsync(
$"guilds/{guildId}/members/{memberId}",
cancellationToken
);
return response?.Pipe(j => Member.Parse(j, guildId));
}
public async ValueTask<Invite?> TryGetInviteAsync(
string code,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var response = await TryGetJsonResponseAsync($"invites/{code}", cancellationToken);
return response?.Pipe(Invite.Parse);
@@ -375,14 +508,15 @@ public class DiscordClient
public async ValueTask<Channel> GetChannelAsync(
Snowflake channelId,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var response = await GetJsonResponseAsync($"channels/{channelId}", cancellationToken);
var parentId = response
.GetPropertyOrNull("parent_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
.GetPropertyOrNull("parent_id")
?.GetNonWhiteSpaceStringOrNull()
?.Pipe(Snowflake.Parse);
try
{
@@ -404,7 +538,8 @@ public class DiscordClient
private async ValueTask<Message?> TryGetLastMessageAsync(
Snowflake channelId,
Snowflake? before = null,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
@@ -421,7 +556,8 @@ public class DiscordClient
Snowflake? after = null,
Snowflake? before = null,
IProgress<Percentage>? progress = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
// Get the last message in the specified range, so we can later calculate the
// progress based on the difference between message timestamps.
@@ -470,13 +606,15 @@ public class DiscordClient
var exportedDuration = (message.Timestamp - firstMessage.Timestamp).Duration();
var totalDuration = (lastMessage.Timestamp - firstMessage.Timestamp).Duration();
progress.Report(Percentage.FromFraction(
// Avoid division by zero if all messages have the exact same timestamp
// (which happens when there's only one message in the channel)
totalDuration > TimeSpan.Zero
? exportedDuration / totalDuration
: 1
));
progress.Report(
Percentage.FromFraction(
// Avoid division by zero if all messages have the exact same timestamp
// (which happens when there's only one message in the channel)
totalDuration > TimeSpan.Zero
? exportedDuration / totalDuration
: 1
)
);
}
yield return message;
@@ -484,4 +622,47 @@ public class DiscordClient
}
}
}
}
public async IAsyncEnumerable<User> GetMessageReactionsAsync(
Snowflake channelId,
Snowflake messageId,
Emoji emoji,
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
var reactionName = emoji.Id is not null
// Custom emoji
? emoji.Name + ':' + emoji.Id
// Standard emoji
: emoji.Name;
var currentAfter = Snowflake.Zero;
while (true)
{
var url = new UrlBuilder()
.SetPath(
$"channels/{channelId}/messages/{messageId}/reactions/{Uri.EscapeDataString(reactionName)}"
)
.SetQueryParameter("limit", "100")
.SetQueryParameter("after", currentAfter.ToString())
.Build();
var response = await GetJsonResponseAsync(url, cancellationToken);
var count = 0;
foreach (var userJson in response.EnumerateArray())
{
var user = User.Parse(userJson);
yield return user;
currentAfter = user.Id;
count++;
}
// Each batch can contain up to 100 users.
// If we got fewer, then it's definitely the last batch.
if (count < 100)
yield break;
}
}
}
@@ -6,9 +6,10 @@ namespace DiscordChatExporter.Core.Discord;
public readonly partial record struct Snowflake(ulong Value)
{
public DateTimeOffset ToDate() => DateTimeOffset.FromUnixTimeMilliseconds(
(long)((Value >> 22) + 1420070400000UL)
).ToLocalTime();
public DateTimeOffset ToDate() =>
DateTimeOffset
.FromUnixTimeMilliseconds((long)((Value >> 22) + 1420070400000UL))
.ToLocalTime();
[ExcludeFromCodeCoverage]
public override string ToString() => Value.ToString(CultureInfo.InvariantCulture);
@@ -18,9 +19,8 @@ public partial record struct Snowflake
{
public static Snowflake Zero { get; } = new(0);
public static Snowflake FromDate(DateTimeOffset instant) => new(
((ulong)instant.ToUnixTimeMilliseconds() - 1420070400000UL) << 22
);
public static Snowflake FromDate(DateTimeOffset instant) =>
new(((ulong)instant.ToUnixTimeMilliseconds() - 1420070400000UL) << 22);
public static Snowflake? TryParse(string? str, IFormatProvider? formatProvider = null)
{
@@ -59,4 +59,4 @@ public partial record struct Snowflake : IComparable<Snowflake>, IComparable
public static bool operator >(Snowflake left, Snowflake right) => left.CompareTo(right) > 0;
public static bool operator <(Snowflake left, Snowflake right) => left.CompareTo(right) < 0;
}
}
@@ -4,4 +4,4 @@ public enum TokenKind
{
User,
Bot
}
}
@@ -2,13 +2,14 @@
<ItemGroup>
<PackageReference Include="AsyncKeyedLock" Version="6.2.1" />
<PackageReference Include="CSharpier.MsBuild" Version="0.25.0" PrivateAssets="all" />
<PackageReference Include="Gress" Version="2.1.1" />
<PackageReference Include="JsonExtensions" Version="1.2.0" />
<PackageReference Include="Polly" Version="7.2.4" />
<PackageReference Include="RazorBlade" Version="0.4.3" />
<PackageReference Include="Polly" Version="8.0.0" />
<PackageReference Include="RazorBlade" Version="0.4.4" />
<PackageReference Include="Superpower" Version="3.0.0" />
<PackageReference Include="WebMarkupMin.Core" Version="2.14.0" />
<PackageReference Include="YoutubeExplode" Version="6.3.0" />
<PackageReference Include="YoutubeExplode" Version="6.3.4" />
</ItemGroup>
</Project>
@@ -11,4 +11,4 @@ public class DiscordChatExporterException : Exception
{
IsFatal = isFatal;
}
}
}
@@ -2,6 +2,7 @@
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions;
using Gress;
@@ -16,18 +17,30 @@ public class ChannelExporter
public async ValueTask ExportChannelAsync(
ExportRequest request,
IProgress<Percentage>? progress = null,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
// Forum channels don't have messages, they are just a list of threads
if (request.Channel.Kind == ChannelKind.GuildForum)
throw new DiscordChatExporterException("Channel is a forum.");
// Check if the channel is empty
if (request.Channel.LastMessageId is null)
if (request.Channel.IsEmpty)
throw new DiscordChatExporterException("Channel does not contain any messages.");
// Check if the 'after' boundary is valid
if (request.After is not null && !request.Channel.MayHaveMessagesAfter(request.After.Value))
{
throw new DiscordChatExporterException(
"Channel does not contain any messages."
"Channel does not contain any messages within the specified period."
);
}
// Check if the 'after' boundary is valid
if (request.After is not null && request.Channel.LastMessageId < request.After)
// Check if the 'before' boundary is valid
if (
request.Before is not null
&& !request.Channel.MayHaveMessagesBefore(request.Before.Value)
)
{
throw new DiscordChatExporterException(
"Channel does not contain any messages within the specified period."
@@ -40,12 +53,15 @@ public class ChannelExporter
// Export messages
await using var messageExporter = new MessageExporter(context);
await foreach (var message in _discord.GetMessagesAsync(
request.Channel.Id,
request.After,
request.Before,
progress,
cancellationToken))
await foreach (
var message in _discord.GetMessagesAsync(
request.Channel.Id,
request.After,
request.Before,
progress,
cancellationToken
)
)
{
// Resolve members for referenced users
foreach (var user in message.GetReferencedUsers())
@@ -64,4 +80,4 @@ public class ChannelExporter
);
}
}
}
}
@@ -20,17 +20,20 @@ internal partial class CsvMessageWriter : MessageWriter
private async ValueTask<string> FormatMarkdownAsync(
string markdown,
CancellationToken cancellationToken = default) =>
CancellationToken cancellationToken = default
) =>
Context.Request.ShouldFormatMarkdown
? await PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken)
: markdown;
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
public override async ValueTask WritePreambleAsync(
CancellationToken cancellationToken = default
) => await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
private async ValueTask WriteAttachmentsAsync(
IReadOnlyList<Attachment> attachments,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var buffer = new StringBuilder();
@@ -48,7 +51,8 @@ internal partial class CsvMessageWriter : MessageWriter
private async ValueTask WriteReactionsAsync(
IReadOnlyList<Reaction> reactions,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
var buffer = new StringBuilder();
@@ -70,7 +74,8 @@ internal partial class CsvMessageWriter : MessageWriter
public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
await base.WriteMessageAsync(message, cancellationToken);
@@ -87,17 +92,15 @@ internal partial class CsvMessageWriter : MessageWriter
await _writer.WriteAsync(',');
// Message content
if (message.Kind.IsSystemNotification())
if (message.IsSystemNotification)
{
await _writer.WriteAsync(CsvEncode(
message.GetFallbackContent()
));
await _writer.WriteAsync(CsvEncode(message.GetFallbackContent()));
}
else
{
await _writer.WriteAsync(CsvEncode(
await FormatMarkdownAsync(message.Content, cancellationToken)
));
await _writer.WriteAsync(
CsvEncode(await FormatMarkdownAsync(message.Content, cancellationToken))
);
}
await _writer.WriteAsync(',');
@@ -127,4 +130,4 @@ internal partial class CsvMessageWriter
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
}
}
@@ -15,17 +15,18 @@ namespace DiscordChatExporter.Core.Exporting;
internal partial class ExportAssetDownloader
{
private static readonly AsyncKeyedLocker<string> Locker = new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private static readonly AsyncKeyedLocker<string> Locker =
new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private readonly string _workingDirPath;
private readonly bool _reuse;
// File paths of the previously downloaded assets
private readonly Dictionary<string, string> _pathCache = new(StringComparer.Ordinal);
private readonly Dictionary<string, string> _previousPathsByUrl = new(StringComparer.Ordinal);
public ExportAssetDownloader(string workingDirPath, bool reuse)
{
@@ -33,64 +34,79 @@ internal partial class ExportAssetDownloader
_reuse = reuse;
}
public async ValueTask<string> DownloadAsync(string url, CancellationToken cancellationToken = default)
public async ValueTask<string> DownloadAsync(
string url,
CancellationToken cancellationToken = default
)
{
var fileName = GetFileNameFromUrl(url);
var filePath = Path.Combine(_workingDirPath, fileName);
using var _ = await Locker.LockAsync(filePath, cancellationToken);
if (_pathCache.TryGetValue(url, out var cachedFilePath))
if (_previousPathsByUrl.TryGetValue(url, out var cachedFilePath))
return cachedFilePath;
// Reuse existing files if we're allowed to
if (_reuse && File.Exists(filePath))
return _pathCache[url] = filePath;
return _previousPathsByUrl[url] = filePath;
Directory.CreateDirectory(_workingDirPath);
await Http.ResiliencePolicy.ExecuteAsync(async () =>
{
// Download the file
using var response = await Http.Client.GetAsync(url, cancellationToken);
await using (var output = File.Create(filePath))
await response.Content.CopyToAsync(output, cancellationToken);
// Try to set the file date according to the last-modified header
try
await Http.ResiliencePipeline.ExecuteAsync(
async innerCancellationToken =>
{
var lastModified = response.Content.Headers.TryGetValue("Last-Modified")?.Pipe(s =>
DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.None, out var instant)
? instant
: (DateTimeOffset?)null
);
// Download the file
using var response = await Http.Client.GetAsync(url, innerCancellationToken);
await using (var output = File.Create(filePath))
await response.Content.CopyToAsync(output, innerCancellationToken);
if (lastModified is not null)
// Try to set the file date according to the last-modified header
try
{
File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime);
}
}
catch
{
// This can apparently fail for some reason.
// Updating the file date is not a critical task, so we'll just ignore exceptions thrown here.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/585
}
});
var lastModified = response.Content.Headers
.TryGetValue("Last-Modified")
?.Pipe(
s =>
DateTimeOffset.TryParse(
s,
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out var instant
)
? instant
: (DateTimeOffset?)null
);
return _pathCache[url] = filePath;
if (lastModified is not null)
{
File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime);
}
}
catch
{
// This can apparently fail for some reason.
// Updating the file date is not a critical task, so we'll just ignore exceptions thrown here.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/585
}
},
cancellationToken
);
return _previousPathsByUrl[url] = filePath;
}
}
internal partial class ExportAssetDownloader
{
private static string GetUrlHash(string url) => SHA256
.HashData(Encoding.UTF8.GetBytes(url))
.ToHex()
// 5 chars ought to be enough for anybody
.Truncate(5);
private static string GetUrlHash(string url) =>
SHA256
.HashData(Encoding.UTF8.GetBytes(url))
.ToHex()
// 5 chars ought to be enough for anybody
.Truncate(5);
private static string GetFileNameFromUrl(string url)
{
@@ -115,6 +131,8 @@ internal partial class ExportAssetDownloader
fileExtension = "";
}
return PathEx.EscapeFileName(fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension);
return PathEx.EscapeFileName(
fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension
);
}
}
}
@@ -14,17 +14,16 @@ namespace DiscordChatExporter.Core.Exporting;
internal class ExportContext
{
private readonly Dictionary<Snowflake, Member?> _members = new();
private readonly Dictionary<Snowflake, Channel> _channels = new();
private readonly Dictionary<Snowflake, Role> _roles = new();
private readonly Dictionary<Snowflake, Member?> _membersById = new();
private readonly Dictionary<Snowflake, Channel> _channelsById = new();
private readonly Dictionary<Snowflake, Role> _rolesById = new();
private readonly ExportAssetDownloader _assetDownloader;
public DiscordClient Discord { get; }
public ExportRequest Request { get; }
public ExportContext(DiscordClient discord,
ExportRequest request)
public ExportContext(DiscordClient discord, ExportRequest request)
{
Discord = discord;
Request = request;
@@ -35,22 +34,37 @@ internal class ExportContext
);
}
public async ValueTask PopulateChannelsAndRolesAsync(CancellationToken cancellationToken = default)
public DateTimeOffset NormalizeDate(DateTimeOffset instant) =>
Request.IsUtcNormalizationEnabled ? instant.ToUniversalTime() : instant.ToLocalTime();
public string FormatDate(DateTimeOffset instant, string format = "g") =>
NormalizeDate(instant).ToString(format, Request.CultureInfo);
public async ValueTask PopulateChannelsAndRolesAsync(
CancellationToken cancellationToken = default
)
{
await foreach (var channel in Discord.GetGuildChannelsAsync(Request.Guild.Id, cancellationToken))
_channels[channel.Id] = channel;
await foreach (
var channel in Discord.GetGuildChannelsAsync(Request.Guild.Id, cancellationToken)
)
{
_channelsById[channel.Id] = channel;
}
await foreach (var role in Discord.GetGuildRolesAsync(Request.Guild.Id, cancellationToken))
_roles[role.Id] = role;
{
_rolesById[role.Id] = role;
}
}
// Because members cannot be pulled in bulk, we need to populate them on demand
private async ValueTask PopulateMemberAsync(
Snowflake id,
User? fallbackUser,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (_members.ContainsKey(id))
if (_membersById.ContainsKey(id))
return;
var member = await Discord.TryGetGuildMemberAsync(Request.Guild.Id, id, cancellationToken);
@@ -67,41 +81,39 @@ internal class ExportContext
}
// Store the result even if it's null, to avoid re-fetching non-existing members
_members[id] = member;
_membersById[id] = member;
}
public async ValueTask PopulateMemberAsync(Snowflake id, CancellationToken cancellationToken = default) =>
await PopulateMemberAsync(id, null, cancellationToken);
public async ValueTask PopulateMemberAsync(
Snowflake id,
CancellationToken cancellationToken = default
) => await PopulateMemberAsync(id, null, cancellationToken);
public async ValueTask PopulateMemberAsync(User user, CancellationToken cancellationToken = default) =>
await PopulateMemberAsync(user.Id, user, cancellationToken);
public async ValueTask PopulateMemberAsync(
User user,
CancellationToken cancellationToken = default
) => await PopulateMemberAsync(user.Id, user, cancellationToken);
public string FormatDate(DateTimeOffset instant) => Request.DateFormat switch
{
"unix" => instant.ToUnixTimeSeconds().ToString(),
"unixms" => instant.ToUnixTimeMilliseconds().ToString(),
var format => instant.ToLocalString(format)
};
public Member? TryGetMember(Snowflake id) => _membersById.GetValueOrDefault(id);
public Member? TryGetMember(Snowflake id) => _members.GetValueOrDefault(id);
public Channel? TryGetChannel(Snowflake id) => _channelsById.GetValueOrDefault(id);
public Channel? TryGetChannel(Snowflake id) => _channels.GetValueOrDefault(id);
public Role? TryGetRole(Snowflake id) => _rolesById.GetValueOrDefault(id);
public Role? TryGetRole(Snowflake id) => _roles.GetValueOrDefault(id);
public IReadOnlyList<Role> GetUserRoles(Snowflake id) =>
TryGetMember(id)?.RoleIds
.Select(TryGetRole)
.WhereNotNull()
.OrderByDescending(r => r.Position)
.ToArray() ?? Array.Empty<Role>();
public IReadOnlyList<Role> GetUserRoles(Snowflake id) => TryGetMember(id)?
.RoleIds
.Select(TryGetRole)
.WhereNotNull()
.OrderByDescending(r => r.Position)
.ToArray() ?? Array.Empty<Role>();
public Color? TryGetUserColor(Snowflake id) =>
GetUserRoles(id).Where(r => r.Color is not null).Select(r => r.Color).FirstOrDefault();
public Color? TryGetUserColor(Snowflake id) => GetUserRoles(id)
.Where(r => r.Color is not null)
.Select(r => r.Color)
.FirstOrDefault();
public async ValueTask<string> ResolveAssetUrlAsync(string url, CancellationToken cancellationToken = default)
public async ValueTask<string> ResolveAssetUrlAsync(
string url,
CancellationToken cancellationToken = default
)
{
if (!Request.ShouldDownloadAssets)
return url;
@@ -114,8 +126,14 @@ internal class ExportContext
// Prefer relative paths so that the output files can be copied around without breaking references.
// If the asset directory is outside of the export directory, use an absolute path instead.
var optimalFilePath =
relativeFilePath.StartsWith(".." + Path.DirectorySeparatorChar, StringComparison.Ordinal) ||
relativeFilePath.StartsWith(".." + Path.AltDirectorySeparatorChar, StringComparison.Ordinal)
relativeFilePath.StartsWith(
".." + Path.DirectorySeparatorChar,
StringComparison.Ordinal
)
|| relativeFilePath.StartsWith(
".." + Path.AltDirectorySeparatorChar,
StringComparison.Ordinal
)
? filePath
: relativeFilePath;
@@ -138,4 +156,4 @@ internal class ExportContext
return url;
}
}
}
}
@@ -13,23 +13,25 @@ public enum ExportFormat
public static class ExportFormatExtensions
{
public static string GetFileExtension(this ExportFormat format) => format switch
{
ExportFormat.PlainText => "txt",
ExportFormat.HtmlDark => "html",
ExportFormat.HtmlLight => "html",
ExportFormat.Csv => "csv",
ExportFormat.Json => "json",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
public static string GetFileExtension(this ExportFormat format) =>
format switch
{
ExportFormat.PlainText => "txt",
ExportFormat.HtmlDark => "html",
ExportFormat.HtmlLight => "html",
ExportFormat.Csv => "csv",
ExportFormat.Json => "json",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
public static string GetDisplayName(this ExportFormat format) => format switch
{
ExportFormat.PlainText => "TXT",
ExportFormat.HtmlDark => "HTML (Dark)",
ExportFormat.HtmlLight => "HTML (Light)",
ExportFormat.Csv => "CSV",
ExportFormat.Json => "JSON",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
}
public static string GetDisplayName(this ExportFormat format) =>
format switch
{
ExportFormat.PlainText => "TXT",
ExportFormat.HtmlDark => "HTML (Dark)",
ExportFormat.HtmlLight => "HTML (Light)",
ExportFormat.Csv => "CSV",
ExportFormat.Json => "JSON",
_ => throw new ArgumentOutOfRangeException(nameof(format))
};
}
@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
@@ -38,7 +39,11 @@ public partial class ExportRequest
public bool ShouldReuseAssets { get; }
public string DateFormat { get; }
public string Locale { get; }
public CultureInfo CultureInfo { get; }
public bool IsUtcNormalizationEnabled { get; }
public ExportRequest(
Guild guild,
@@ -53,7 +58,9 @@ public partial class ExportRequest
bool shouldFormatMarkdown,
bool shouldDownloadAssets,
bool shouldReuseAssets,
string dateFormat)
string locale,
bool isUtcNormalizationEnabled
)
{
Guild = guild;
Channel = channel;
@@ -65,28 +72,18 @@ public partial class ExportRequest
ShouldFormatMarkdown = shouldFormatMarkdown;
ShouldDownloadAssets = shouldDownloadAssets;
ShouldReuseAssets = shouldReuseAssets;
DateFormat = dateFormat;
Locale = locale;
IsUtcNormalizationEnabled = isUtcNormalizationEnabled;
OutputFilePath = GetOutputBaseFilePath(
Guild,
Channel,
outputPath,
Format,
After,
Before
);
OutputFilePath = GetOutputBaseFilePath(Guild, Channel, outputPath, Format, After, Before);
OutputDirPath = Path.GetDirectoryName(OutputFilePath)!;
AssetsDirPath = !string.IsNullOrWhiteSpace(assetsDirPath)
? FormatPath(
assetsDirPath,
Guild,
Channel,
After,
Before
)
? FormatPath(assetsDirPath, Guild, Channel, After, Before)
: $"{OutputFilePath}_Files{Path.DirectorySeparatorChar}";
CultureInfo = CultureInfo.GetCultureInfo(Locale);
}
}
@@ -97,12 +94,26 @@ public partial class ExportRequest
Channel channel,
ExportFormat format,
Snowflake? after = null,
Snowflake? before = null)
Snowflake? before = null
)
{
var buffer = new StringBuilder();
// Guild and channel names
buffer.Append($"{guild.Name} - {channel.Category} - {channel.Name} [{channel.Id}]");
// Guild name
buffer.Append(guild.Name);
// Parent name
if (channel.Parent is not null)
buffer.Append(" - ").Append(channel.Parent.Name);
// Channel name and ID
buffer
.Append(" - ")
.Append(channel.Name)
.Append(' ')
.Append('[')
.Append(channel.Id)
.Append(']');
// Date range
if (after is not null || before is not null)
@@ -112,7 +123,9 @@ public partial class ExportRequest
// Both 'after' and 'before' are set
if (after is not null && before is not null)
{
buffer.Append($"{after.Value.ToDate():yyyy-MM-dd} to {before.Value.ToDate():yyyy-MM-dd}");
buffer.Append(
$"{after.Value.ToDate():yyyy-MM-dd} to {before.Value.ToDate():yyyy-MM-dd}"
);
}
// Only 'after' is set
else if (after is not null)
@@ -139,29 +152,46 @@ public partial class ExportRequest
Guild guild,
Channel channel,
Snowflake? after,
Snowflake? before)
{
return Regex.Replace(
Snowflake? before
) =>
Regex.Replace(
path,
"%.",
m => PathEx.EscapeFileName(m.Value switch
{
"%g" => guild.Id.ToString(),
"%G" => guild.Name,
"%t" => channel.Parent?.Id.ToString() ?? "",
"%T" => channel.Parent?.Name ?? "",
"%c" => channel.Id.ToString(),
"%C" => channel.Name,
"%p" => channel.Position?.ToString() ?? "0",
"%P" => channel.Parent?.Position?.ToString() ?? "0",
"%a" => after?.ToDate().ToString("yyyy-MM-dd") ?? "",
"%b" => before?.ToDate().ToString("yyyy-MM-dd") ?? "",
"%d" => DateTimeOffset.Now.ToString("yyyy-MM-dd"),
"%%" => "%",
_ => m.Value
})
m =>
PathEx.EscapeFileName(
m.Value switch
{
"%g" => guild.Id.ToString(),
"%G" => guild.Name,
"%t" => channel.Parent?.Id.ToString() ?? "",
"%T" => channel.Parent?.Name ?? "",
"%c" => channel.Id.ToString(),
"%C" => channel.Name,
"%p" => channel.Position?.ToString(CultureInfo.InvariantCulture) ?? "0",
"%P"
=> channel.Parent?.Position?.ToString(CultureInfo.InvariantCulture)
?? "0",
"%a"
=> after?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)
?? "",
"%b"
=> before?.ToDate().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)
?? "",
"%d"
=> DateTimeOffset.Now.ToString(
"yyyy-MM-dd",
CultureInfo.InvariantCulture
),
"%%" => "%",
_ => m.Value
}
)
);
}
private static string GetOutputBaseFilePath(
Guild guild,
@@ -169,12 +199,16 @@ public partial class ExportRequest
string outputPath,
ExportFormat format,
Snowflake? after = null,
Snowflake? before = null)
Snowflake? before = null
)
{
var actualOutputPath = FormatPath(outputPath, guild, channel, after, before);
// Output is a directory
if (Directory.Exists(actualOutputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(actualOutputPath)))
if (
Directory.Exists(actualOutputPath)
|| string.IsNullOrWhiteSpace(Path.GetExtension(actualOutputPath))
)
{
var fileName = GetDefaultOutputFileName(guild, channel, format, after, before);
return Path.Combine(actualOutputPath, fileName);
@@ -183,4 +217,4 @@ public partial class ExportRequest
// Output is a file
return actualOutputPath;
}
}
}
@@ -4,4 +4,4 @@ internal enum BinaryExpressionKind
{
Or,
And
}
}
@@ -9,17 +9,22 @@ internal class BinaryExpressionMessageFilter : MessageFilter
private readonly MessageFilter _second;
private readonly BinaryExpressionKind _kind;
public BinaryExpressionMessageFilter(MessageFilter first, MessageFilter second, BinaryExpressionKind kind)
public BinaryExpressionMessageFilter(
MessageFilter first,
MessageFilter second,
BinaryExpressionKind kind
)
{
_first = first;
_second = second;
_kind = kind;
}
public override bool IsMatch(Message message) => _kind switch
{
BinaryExpressionKind.Or => _first.IsMatch(message) || _second.IsMatch(message),
BinaryExpressionKind.And => _first.IsMatch(message) && _second.IsMatch(message),
_ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.")
};
}
public override bool IsMatch(Message message) =>
_kind switch
{
BinaryExpressionKind.Or => _first.IsMatch(message) || _second.IsMatch(message),
BinaryExpressionKind.And => _first.IsMatch(message) && _second.IsMatch(message),
_ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.")
};
}
@@ -17,25 +17,21 @@ internal class ContainsMessageFilter : MessageFilter
// parentheses are not considered word characters.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/909
private bool IsMatch(string? content) =>
!string.IsNullOrWhiteSpace(content) &&
Regex.IsMatch(
!string.IsNullOrWhiteSpace(content)
&& Regex.IsMatch(
content,
@"(?:\b|\s|^)" +
Regex.Escape(_text) +
@"(?:\b|\s|$)",
@"(?:\b|\s|^)" + Regex.Escape(_text) + @"(?:\b|\s|$)",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant
);
public override bool IsMatch(Message message) =>
IsMatch(message.Content) ||
message.Embeds.Any(e =>
IsMatch(e.Title) ||
IsMatch(e.Author?.Name) ||
IsMatch(e.Description) ||
IsMatch(e.Footer?.Text) ||
e.Fields.Any(f =>
IsMatch(f.Name) ||
IsMatch(f.Value)
)
IsMatch(message.Content)
|| message.Embeds.Any(
e =>
IsMatch(e.Title)
|| IsMatch(e.Author?.Name)
|| IsMatch(e.Description)
|| IsMatch(e.Footer?.Text)
|| e.Fields.Any(f => IsMatch(f.Name) || IsMatch(f.Value))
);
}
}
@@ -10,8 +10,8 @@ internal class FromMessageFilter : MessageFilter
public FromMessageFilter(string value) => _value = value;
public override bool IsMatch(Message message) =>
string.Equals(_value, message.Author.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.DisplayName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase);
}
string.Equals(_value, message.Author.Name, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, message.Author.DisplayName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase);
}
@@ -11,15 +11,20 @@ internal class HasMessageFilter : MessageFilter
public HasMessageFilter(MessageContentMatchKind kind) => _kind = kind;
public override bool IsMatch(Message message) => _kind switch
{
MessageContentMatchKind.Link => Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]"),
MessageContentMatchKind.Embed => message.Embeds.Any(),
MessageContentMatchKind.File => message.Attachments.Any(),
MessageContentMatchKind.Video => message.Attachments.Any(file => file.IsVideo),
MessageContentMatchKind.Image => message.Attachments.Any(file => file.IsImage),
MessageContentMatchKind.Sound => message.Attachments.Any(file => file.IsAudio),
MessageContentMatchKind.Pin => message.IsPinned,
_ => throw new InvalidOperationException($"Unknown message content match kind '{_kind}'.")
};
}
public override bool IsMatch(Message message) =>
_kind switch
{
MessageContentMatchKind.Link
=> Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]"),
MessageContentMatchKind.Embed => message.Embeds.Any(),
MessageContentMatchKind.File => message.Attachments.Any(),
MessageContentMatchKind.Video => message.Attachments.Any(file => file.IsVideo),
MessageContentMatchKind.Image => message.Attachments.Any(file => file.IsImage),
MessageContentMatchKind.Sound => message.Attachments.Any(file => file.IsAudio),
MessageContentMatchKind.Pin => message.IsPinned,
_
=> throw new InvalidOperationException(
$"Unknown message content match kind '{_kind}'."
)
};
}
@@ -10,10 +10,12 @@ internal class MentionsMessageFilter : MessageFilter
public MentionsMessageFilter(string value) => _value = value;
public override bool IsMatch(Message message) => message.MentionedUsers.Any(user =>
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.DisplayName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase)
);
}
public override bool IsMatch(Message message) =>
message.MentionedUsers.Any(
user =>
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, user.DisplayName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase)
|| string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase)
);
}
@@ -9,4 +9,4 @@ internal enum MessageContentMatchKind
Image,
Sound,
Pin
}
}
@@ -14,4 +14,4 @@ public partial class MessageFilter
public static MessageFilter Null { get; } = new NullMessageFilter();
public static MessageFilter Parse(string value) => FilterGrammar.Filter.Parse(value);
}
}
@@ -9,4 +9,4 @@ internal class NegatedMessageFilter : MessageFilter
public NegatedMessageFilter(MessageFilter filter) => _filter = filter;
public override bool IsMatch(Message message) => !_filter.IsMatch(message);
}
}

Some files were not shown because too many files have changed in this diff Show More