Compare commits

..

138 Commits

Author SHA1 Message Date
Oleksii Holub 2024d3658b Update version 2022-08-26 17:58:41 +03:00
Oleksii Holub 9453cac389 Fix regex in contains filter 2022-08-26 00:00:02 +03:00
Oleksii Holub 504c3c53cd Fail if an export command is called with multiple channels but a single-file output path
Closes #799
2022-08-25 22:11:26 +03:00
Oleksii Holub 4e358e8e6f Add support for animated guild icons
Closes #900
2022-08-25 21:38:59 +03:00
Oleksii Holub 5e852235ce Retry on all Discord server side errors
Closes #908
2022-08-25 21:32:47 +03:00
Oleksii Holub 612c6d37a4 Make contains filter match on spaces and input boundaries, as well as word boundaries
Closes #909
2022-08-25 21:26:56 +03:00
Oleksii Holub 92cf886eab Fix not being able to parse message filters that share common prefix with one of the named filters
Related to #909
2022-08-25 21:20:22 +03:00
Oleksii Holub f48af53d72 Simplify filter grammar 2022-08-25 21:07:02 +03:00
Oleksii Holub 99d721c63b Update nuget packages 2022-08-25 20:22:10 +03:00
Oleksii Holub 11414b5937 Refactor message filter grammar and add ~ as an alias for - in negated filters for better UX when using CLI 2022-08-25 20:09:06 +03:00
Oleksii Holub 6c039ebfa6 Filter out categories when displaying channels 2022-08-19 18:39:19 +03:00
TSRBerry 9cffbbc5ce Expand tilde to home directory on *nix systems in output path (#903) 2022-08-18 13:23:26 -07:00
Oleksii Holub 6f87960513 Make workflow stages less ambigious 2022-08-13 04:13:09 -07:00
Oleksii Holub e8f8c47dbd Make JS in HTML templates more consistent 2022-07-19 16:16:48 +03:00
Oleksii Holub 9aba36fce5 Update twemoji URLs 2022-07-19 15:55:58 +03:00
Oleksii Holub f29e1d2d9f Inform about a breaking change in Docker configuration
Related to #884
2022-07-10 02:42:43 -07:00
Oleksii Holub b000a72f81 Update download instructions 2022-07-06 14:08:07 +03:00
Oleksii Holub f5318e77b7 Update version 2022-07-02 13:20:03 +03:00
Oleksii Holub e171b43c22 Simplify CSS 2022-07-01 13:17:13 +03:00
Oleksii Holub 23e1850d15 Render consecutive twitter embeds as on
Closes #695
2022-06-30 20:56:44 +03:00
Oleksii Holub 057902f919 More test stuff 2022-06-30 19:07:11 +03:00
Oleksii Holub 565fb53d3b Cleanup tests 2022-06-30 18:13:14 +03:00
Oleksii Holub edba9f36a7 Hide message content in HTML when linking to an image
Closes #695
2022-06-29 21:43:43 +03:00
Oleksii Holub 356b1a6afd Don't use records for spec classes 2022-06-29 20:49:51 +03:00
Oleksii Holub 70edc8989c Add test for plain image embed 2022-06-29 20:38:48 +03:00
Oleksii Holub 2fd7e327a9 Add support for YouTube Shorts video embeds in HTML 2022-06-29 20:26:04 +03:00
Oleksii Holub 9182c3d934 Fix tests 2022-06-29 12:59:20 +03:00
Oleksii Holub 01389b7e28 Fix rendering of code blocks in HTML 2022-06-28 21:18:57 +03:00
Oleksii Holub bd0701cb82 Render quotes inline in referenced messages in HTML 2022-06-28 21:02:11 +03:00
Oleksii Holub 4283ef22b1 Add HTML minification
Closes #345
Closes #743
2022-06-28 20:20:33 +03:00
Oleksii Holub 94ef4b6981 Support listing and exporting voice channels
Closes #874
2022-06-28 18:27:01 +03:00
Oleksii Holub 7b72f473fd Pad outputs from guilds, channels, dm to have more aligned columns
Closes #877
2022-06-28 17:39:32 +03:00
Oleksii Holub 2463cb5087 Sort DMs by last message and avoid grouping them in GUI
Closes #271
2022-06-28 17:23:01 +03:00
Oleksii Holub 2c7986c4e6 Refactor IsTextChannel and IsVoiceChannel to extensions 2022-06-28 16:38:44 +03:00
Oleksii Holub 5755334492 Update war in Ukraine message in CLI 2022-06-24 12:26:35 +03:00
Oleksii Holub 26c523a3e9 Update usage instructions to direct towards the Network tab method
Closes #835
Closes #842
2022-06-24 12:25:50 +03:00
Oleksii Holub 371c7d2b75 Make TOS warning more prominent
Close #878
2022-06-24 12:14:29 +03:00
Oleksii Holub 62400f305a Make CLI attributes look tidier 2022-06-19 12:21:09 +03:00
Oleksii Holub 65a528e7fe Simplify dockerfile 2022-06-18 22:37:24 +03:00
Lucas LaBuff 47d8163a46 Add filename token for current date (#869) 2022-06-17 12:06:12 -07:00
Clint Herron 1d9224ebd8 Update guide to direct users to the Advanced section of Discord settings rather than Appearance (#867) 2022-06-16 02:01:15 -07:00
Ethan ddc7f921bd Fix Twemoji URLs with ZWJs being generated incorrectly (#865) 2022-06-11 15:46:39 -07:00
Oleksii Holub aca94ddccb Update test logger 2022-06-10 23:16:47 +03:00
Oleksii Holub 3661c42df4 Fix tests by fixing typo in last commit 2022-06-10 22:32:16 +03:00
Oleksii Holub 005c1786ec Fix tests 2022-06-10 22:23:20 +03:00
Oleksii Holub b6fbf2393e Update bootstrapper and add it to the CLI flavor too 2022-06-10 22:12:48 +03:00
Philipp C. Heckel 0e49936e38 Make messages linkable in HTML (#864) 2022-06-09 09:47:00 -07:00
Oleksii Holub 8d4e6b77c4 Fix docker workflow not tagging with version number 2022-06-01 20:06:59 +03:00
Oleksii Holub 6f47fb6a1c Update version 2022-06-01 19:55:49 +03:00
Oleksii Holub 6150720b1e Update readme 2022-05-31 12:29:31 +03:00
Oleksii Holub dcdeea19ab Update nuget packages 2022-05-29 22:43:42 +03:00
Oleksii Holub ca56d47110 Allow whitespace in sticker names 2022-05-20 20:45:42 +03:00
Oleksii Holub 17b548d679 Add test for last change 2022-05-18 20:19:46 +03:00
Andrew Kolos 20eeaa58e9 Add has:pin filter (#853)
Adds a new `has:*` filter. Specifically, `has:pin`.
The filter filters for messages that are pinned.
2022-05-18 10:11:50 -07:00
Oleksii Holub 6621c10ae0 Satisfy my OCD 2022-04-30 16:04:57 +03:00
Oleksii Holub 75214354ca Update CliFx 2022-04-22 19:48:11 +03:00
Oleksii Holub 84d99e8520 Reorganize assets 2022-04-22 17:19:51 +03:00
Oleksii Holub 4b80eabb73 Update NuGet.config 2022-04-22 17:03:03 +03:00
Oleksii Holub e6830bbd0e Plural artifacts 2022-04-22 02:10:00 +03:00
Oleksii Holub 97c751f171 Simplify workflows 2022-04-22 01:23:48 +03:00
Oleksii Holub dc814e08e7 Update readme 2022-04-22 01:14:35 +03:00
Oleksii Holub 54bf7c204a Consolidate workflows 2022-04-22 01:13:36 +03:00
Oleksii Holub 36c9983c9e Update test logger 2022-04-19 04:29:50 +03:00
Oleksii Holub 361e0489ed Update NuGet packages 2022-04-17 03:36:15 +03:00
Oleksii Holub 77436120a8 Update readme 2022-04-16 18:48:39 +03:00
Oleksii Holub 2d7ab5411f Revert "Use nightly.link for linking CI builds"
This reverts commit 1ce523b252.
2022-04-13 22:40:27 +03:00
Oleksii Holub 1ce523b252 Use nightly.link for linking CI builds 2022-04-13 22:38:41 +03:00
Oleksii Holub bb81cf06ae Rework retry policy
Fixes #832
2022-04-12 23:55:03 +03:00
Oleksii Holub f7f6ac9494 Update version 2022-04-10 21:43:52 +03:00
Oleksii Holub 481958f4d3 Update Readme.md 2022-04-09 21:58:41 +03:00
Oleksii Holub 5e3502cbc5 Update Readme.md 2022-04-09 21:57:34 +03:00
Oleksii Holub 3b03c30865 Update screenshots in readme 2022-04-09 21:04:03 +03:00
Oleksii Holub c9e16a7f74 Spaces in css rules 2022-04-09 20:37:12 +03:00
Oleksii Holub f9b6793d47 Update margins in HTML embeds 2022-04-09 20:30:17 +03:00
Oleksii Holub cfbd267d3a Fix embed styles in HTML 2022-04-09 20:25:13 +03:00
Oleksii Holub 2b6f01bf6f Fix font sizes in HTML template 2022-04-09 20:08:37 +03:00
Oleksii Holub a44d43eefc Use a lock when opening dialogs 2022-04-09 03:17:30 +03:00
Oleksii Holub 0e138bac7d Improve top-level spacing in html template 2022-04-09 03:05:54 +03:00
Oleksii Holub 71876febf8 Use attachment description as alt text in html
Closes #805
2022-04-09 02:52:57 +03:00
Oleksii Holub 489c786c1e Pass token without validation
Fixes #828
2022-04-09 02:46:40 +03:00
Oleksii Holub eb0c61e173 Try to fix remaining test 2022-04-09 02:35:07 +03:00
Oleksii Holub 9341c21963 Add clickable links to usage guide in GUI 2022-04-09 02:31:52 +03:00
Oleksii Holub 704c483074 Update nuget packages 2022-04-09 02:22:32 +03:00
Oleksii Holub 1f0e667085 Fix missing awaits in preamble template 2022-04-09 02:21:49 +03:00
Oleksii Holub 0b72b73d1d Fix more tests 2022-04-09 02:13:58 +03:00
Oleksii Holub 32dd1689d0 Fix tests 2022-04-09 02:07:51 +03:00
Oleksii Holub 2e4d19671a Append channel name for export errors in GUI 2022-04-09 02:01:19 +03:00
Oleksii Holub 5b4d88ca26 Update HTML layout and styles 2022-04-09 01:58:13 +03:00
Chris Helder 65cc3954b6 Use 512x512 user profile pictures (#826) 2022-04-08 00:39:07 +03:00
Oleksii Holub ffe5544f9c Dowload fonts in HTML export when using download media 2022-04-07 20:26:12 +03:00
Oleksii Holub bf1ad07a9c Use coverlet collector 2022-04-06 21:17:47 +00:00
Oleksii Holub 338566c1d6 More cleanup 2022-04-06 19:43:10 +03:00
Oleksii Holub 91ea7a443c More cleanup 2022-04-06 19:12:27 +03:00
Oleksii Holub aea96d5eba Cleanup 2022-04-06 19:00:19 +03:00
Oleksii Holub e29f08264c Refactor portions of RootViewModel into DashboardViewModel 2022-04-04 01:34:46 +03:00
Oleksii Holub ad84ecf6a4 Fix hyperlink style 2022-04-02 01:11:13 +03:00
Oleksii Holub ae57a0ff7c Update readme 2022-04-01 02:27:01 +03:00
Oleksii Holub 1daff4178d Cleanup GUI 2022-04-01 00:10:17 +03:00
Oleksii Holub 3a2b119618 Update Readme.md 2022-03-22 11:02:14 -07:00
Oleksii Holub 5f27426f44 Update Readme.md 2022-03-21 10:24:44 -07:00
Oleksii Holub a893ebda68 Update version 2022-03-07 03:43:47 +02:00
Oleksii Holub 9221966171 Fix binding issue by using an older .NET SDK to build 2022-03-07 03:33:54 +02:00
Oleksii Holub 67325b610a Update version 2022-03-07 03:02:28 +02:00
Oleksii Holub 3186ce90c8 Fix GUI not launching due to .NET bug 2022-03-07 02:46:56 +02:00
Oleksii Holub 168d7c8ec9 Update the war message in CLI 2022-03-07 02:39:05 +02:00
Oleksii Holub daa1b3f6b3 Improve the war message 2022-03-07 01:09:56 +02:00
Oleksii Holub e2780d6666 Update version 2022-03-06 19:41:11 +02:00
Oleksii Holub b996215e36 Add messages about war in Ukraine 2022-03-06 19:24:22 +02:00
Oleksii Holub 08d9af534b Add terms of use 2022-03-05 09:30:41 -08:00
Oleksii Holub bef110a193 Add fuck russia banner to readme 2022-02-23 18:25:23 +02:00
Oleksii Holub 12555e576f Update NuGet packages 2022-02-21 22:47:40 +02:00
Oleksii Holub afb4bc67ae Auto-detect dark mode preference in GUI 2022-02-20 22:29:23 +02:00
Oleksii Holub f80a5fe644 Fix multiline quote rendering in HTML
Fixes #756
2022-02-19 00:49:56 +02:00
Oleksii Holub b7dca87fb9 Add reaction filter
Closes #715
2022-02-19 00:28:38 +02:00
Oleksii Holub 8ecc2940b0 HTML-escape links inside HTML export instead of URL-escaping 2022-02-19 00:26:54 +02:00
Oleksii Holub 407c2bd2ae Don't crash on invalid mentions
Fixes #816
2022-02-19 00:26:53 +02:00
Oleksii Holub 5b563d4f00 Cleanup some code 2022-02-19 00:26:53 +02:00
Tomlacko ecb615ecf1 Fix for extremely long file extensions (#812) 2022-02-17 19:58:47 -08:00
Oleksii Holub 7c212ef893 Update NuGet packages 2022-02-16 14:34:01 +02:00
Oleksii Holub 36b4a45f3a Update Gress 2022-02-16 14:30:26 +02:00
Oleksii Holub 51cc132e5d Display 24-hour time pickers for appropriate cultures in GUI
Related to #808
2022-02-10 17:23:39 +02:00
Oleksii Holub cf83cbd89c Add support for stickers (#802) 2022-02-05 16:26:23 -08:00
Oleksii Holub d51d0d4872 Use snowflakes in more places 2022-02-06 01:15:56 +02:00
Oleksii Holub 2467caac9f Use api v9 2022-02-05 20:19:16 +02:00
Oleksii Holub bd98f4cb6a Refactor StringPart into StringSegment 2022-02-05 04:10:35 +02:00
Oleksii Holub fc7191d74c Fix changelog date for last entry 2022-01-27 18:44:40 +02:00
Oleksii Holub 6c484d1f88 Update version 2022-01-27 18:36:28 +02:00
Oleksii Holub c36b8df8e7 Update NuGet packages 2022-01-27 18:25:21 +02:00
Dhananjay-JSR 0408096746 Update token extraction guide (#791) 2022-01-25 15:11:22 -08:00
Tyrrrz 40d833cd57 Update my name to match correct spelling 2022-01-15 03:27:19 +02:00
Tyrrrz 0de1f47310 More parsing leniency
Closes #767
2022-01-09 22:24:02 +02:00
Tyrrrz e31ff55e33 Handle invalid dates more gracefully
Closes #766
2022-01-05 18:54:23 +02:00
Tyrrrz 81525a1076 Update license 2022-01-05 18:31:16 +02:00
Tyrrrz 65ab003ff5 Relax parsing rules for embed fields
Closes #765
2022-01-04 22:44:03 +02:00
Alexey Golub 2156c6cd0c Automatically detect token kind (#764) 2022-01-03 14:52:16 -08:00
Tyrrrz e97151cd19 Update NuGet packages 2022-01-03 00:36:10 +02:00
Tyrrrz 271a478a0e Minor GUI tweaks 2021-12-15 16:30:23 +02:00
Tyrrrz b40a88c4d3 Add border to settings view too 2021-12-15 16:13:48 +02:00
131 changed files with 3620 additions and 2299 deletions
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

-30
View File
@@ -1,30 +0,0 @@
name: Docker CD
on:
push:
tags:
- "*"
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2.3.3
- name: Get release version
id: get_version
uses: dawidd6/action-get-tag@v1
- name: Login
run: docker login -u tyrrrz -p ${{ secrets.DOCKER_TOKEN }}
- name: Build
run: docker build -t tyrrrz/discordchatexporter:${{ steps.get_version.outputs.tag }} -t tyrrrz/discordchatexporter:stable .
- name: Deploy (version)
run: docker push tyrrrz/discordchatexporter:${{ steps.get_version.outputs.tag }}
- name: Deploy (stable)
run: docker push tyrrrz/discordchatexporter:stable
-84
View File
@@ -1,84 +0,0 @@
name: CD
on:
push:
tags:
- "*"
jobs:
build:
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v2.3.3
- name: Get release version
id: get_version
uses: dawidd6/action-get-tag@v1
- name: Install .NET
uses: actions/setup-dotnet@v1.7.2
with:
dotnet-version: 6.0.x
- name: Build & publish (CLI)
run: dotnet publish DiscordChatExporter.Cli/ -o DiscordChatExporter.Cli/bin/Publish/ --configuration Release
- name: Build & publish (GUI)
run: dotnet publish DiscordChatExporter.Gui/ -o DiscordChatExporter.Gui/bin/Publish/ --configuration Release
- name: Pack (CLI)
run: Compress-Archive -Path DiscordChatExporter.Cli/bin/Publish/* -DestinationPath DiscordChatExporter.Cli/bin/Publish/Archive.zip -Force
shell: pwsh
- name: Pack (GUI)
run: Compress-Archive -Path DiscordChatExporter.Gui/bin/Publish/* -DestinationPath DiscordChatExporter.Gui/bin/Publish/Archive.zip -Force
shell: pwsh
- name: Create release
id: create_release
uses: actions/create-release@v1.0.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: ${{ github.ref }}
body: |
[Changelog](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/Changelog.md)
draft: false
prerelease: false
- name: Upload release asset (CLI)
uses: actions/upload-release-asset@v1.0.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: DiscordChatExporter.Cli/bin/Publish/Archive.zip
asset_name: DiscordChatExporter.CLI.zip
asset_content_type: application/zip
- name: Upload release asset (GUI)
uses: actions/upload-release-asset@v1.0.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: DiscordChatExporter.Gui/bin/Publish/Archive.zip
asset_name: DiscordChatExporter.zip
asset_content_type: application/zip
- name: Notify Discord
uses: satak/webrequest-action@v1.2.4
with:
url: ${{ secrets.DISCORD_WEBHOOK }}
method: POST
headers: |
{
"ContentType": "application/json; charset=UTF-8"
}
payload: |
{
"content": "**DiscordChatExporter** new version released!\nVersion: `${{ steps.get_version.outputs.tag }}`\nChangelog: <https://github.com/Tyrrrz/DiscordChatExporter/blob/${{ steps.get_version.outputs.tag }}/Changelog.md>"
}
-23
View File
@@ -1,23 +0,0 @@
name: Docker CI
on:
push:
branches:
- master
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2.3.3
- name: Login
run: docker login -u tyrrrz -p ${{ secrets.DOCKER_TOKEN }}
- name: Build
run: docker build -t tyrrrz/discordchatexporter:latest .
- name: Deploy (latest)
run: docker push tyrrrz/discordchatexporter:latest
-47
View File
@@ -1,47 +0,0 @@
name: CI
on: [push, pull_request]
jobs:
build:
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v2.3.3
- name: Install .NET
uses: actions/setup-dotnet@v1.7.2
with:
dotnet-version: 6.0.x
- name: Build & test
# Tests need access to secrets, so we can't run them against PRs because of limited trust
if: ${{ github.event_name != 'pull_request' }}
run: dotnet test --configuration Release --logger GitHubActions
env:
DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
DISCORD_TOKEN_BOT: true
- name: Upload coverage
uses: codecov/codecov-action@v1.0.5
with:
token: ${{ secrets.CODECOV_TOKEN }}
- name: Build & publish (CLI)
run: dotnet publish DiscordChatExporter.Cli/ -o DiscordChatExporter.Cli/bin/Publish/ --configuration Release
- name: Build & publish (GUI)
run: dotnet publish DiscordChatExporter.Gui/ -o DiscordChatExporter.Gui/bin/Publish/ --configuration Release
- name: Upload build artifacts (CLI)
uses: actions/upload-artifact@v1
with:
name: DiscordChatExporter.CLI
path: DiscordChatExporter.Cli/bin/Publish/
- name: Upload build artifact (GUI)
uses: actions/upload-artifact@v1
with:
name: DiscordChatExporter
path: DiscordChatExporter.Gui/bin/Publish/
+55
View File
@@ -0,0 +1,55 @@
name: docker
on: [push, pull_request]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Build container
run: docker build -f DiscordChatExporter.Cli.dockerfile .
deploy-latest:
if: ${{ github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master' }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Login to DockerHub
run: docker login -u tyrrrz -p ${{ secrets.DOCKER_TOKEN }}
- name: Build container
run: docker build -f DiscordChatExporter.Cli.dockerfile -t tyrrrz/discordchatexporter:latest .
- name: Push container
run: docker push tyrrrz/discordchatexporter:latest
deploy-stable:
if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Get release version
id: get_version
uses: dawidd6/action-get-tag@v1
- name: Login to DockerHub
run: docker login -u tyrrrz -p ${{ secrets.DOCKER_TOKEN }}
- name: Build container
run: docker build -f DiscordChatExporter.Cli.dockerfile -t tyrrrz/discordchatexporter:stable -t tyrrrz/discordchatexporter:${{ steps.get_version.outputs.tag }} .
- name: Push container (stable)
run: docker push tyrrrz/discordchatexporter:stable
- name: Push container (${{ steps.get_version.outputs.tag }})
run: docker push tyrrrz/discordchatexporter:${{ steps.get_version.outputs.tag }}
+154
View File
@@ -0,0 +1,154 @@
name: main
on: [push, pull_request]
jobs:
test:
runs-on: windows-latest
env:
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
DOTNET_NOLOGO: true
DOTNET_CLI_TELEMETRY_OPTOUT: true
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install .NET
uses: actions/setup-dotnet@v2
with:
# Fixed version, workaround for https://github.com/dotnet/core/issues/7176
dotnet-version: 6.0.100
- name: Run tests
# Tests need access to secrets, so we can't run them against PRs because of limited trust
if: ${{ github.event_name != 'pull_request' }}
run: dotnet test --configuration Release --logger GitHubActions --collect:"XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=opencover
env:
DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
DISCORD_TOKEN_BOT: true
- name: Upload coverage
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
package:
needs: test
runs-on: windows-latest
env:
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
DOTNET_NOLOGO: true
DOTNET_CLI_TELEMETRY_OPTOUT: true
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install .NET
uses: actions/setup-dotnet@v2
with:
# Fixed version, workaround for https://github.com/dotnet/core/issues/7176
dotnet-version: 6.0.100
- name: Publish (CLI)
run: dotnet publish DiscordChatExporter.Cli/ -o DiscordChatExporter.Cli/bin/Publish/ --configuration Release
- name: Publish (GUI)
run: dotnet publish DiscordChatExporter.Gui/ -o DiscordChatExporter.Gui/bin/Publish/ --configuration Release
- name: Upload artifacts (CLI)
uses: actions/upload-artifact@v3
with:
name: DiscordChatExporter.Cli
path: DiscordChatExporter.Cli/bin/Publish/
- name: Upload artifacts (GUI)
uses: actions/upload-artifact@v3
with:
name: DiscordChatExporter
path: DiscordChatExporter.Gui/bin/Publish/
deploy:
if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
needs: package
runs-on: ubuntu-latest
steps:
- name: Download artifacts (CLI)
uses: actions/download-artifact@v3
with:
name: DiscordChatExporter.Cli
path: DiscordChatExporter.Cli
- name: Download artifacts (GUI)
uses: actions/download-artifact@v3
with:
name: DiscordChatExporter
path: DiscordChatExporter.Gui
- name: Create package (CLI)
run: Compress-Archive -Path DiscordChatExporter.Cli/* -DestinationPath DiscordChatExporter.Cli.zip -Force
shell: pwsh
- name: Create package (GUI)
run: Compress-Archive -Path DiscordChatExporter.Gui/* -DestinationPath DiscordChatExporter.Gui.zip -Force
shell: pwsh
- name: Create release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: ${{ github.ref }}
body: |
[Changelog](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/Changelog.md)
draft: false
prerelease: false
- name: Upload release asset (CLI)
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_name: DiscordChatExporter.Cli.zip
asset_path: DiscordChatExporter.Cli.zip
asset_content_type: application/zip
- name: Upload release asset (GUI)
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_name: DiscordChatExporter.zip
asset_path: DiscordChatExporter.Gui.zip
asset_content_type: application/zip
notify:
needs: deploy
runs-on: ubuntu-latest
steps:
- name: Get release version
id: get_version
uses: dawidd6/action-get-tag@v1
- name: Notify Discord
uses: satak/webrequest-action@v1.2.4
with:
url: ${{ secrets.DISCORD_WEBHOOK }}
method: POST
headers: |
{
"ContentType": "application/json; charset=UTF-8"
}
payload: |
{
"content": "**DiscordChatExporter** new version released!\nVersion: `${{ steps.get_version.outputs.tag }}`\nChangelog: <https://github.com/Tyrrrz/DiscordChatExporter/blob/${{ steps.get_version.outputs.tag }}/Changelog.md>"
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 33 KiB

+76 -1
View File
@@ -1,6 +1,81 @@
### v2.35.1 (26-Aug-2022)
- Added `~` character as an alias for `-` when negating a message filter. This should make it easier to use negated filters from CLI where the dash character already has other meanings.
- Extended retry logic to encompass all server-side errors from Discord API, instead of only `500 Internal Server Error`.
- [CLI] Attempt to export multiple channels will now fail if provided with an output path that is neither a directory, nor a file name template. Previously, this resulted in all channels being exported to the same file, overwriting each other.
- Fixed an issue where a text message filter didn't work if it started with a character that's also the first character in one of the named filters (e.g. `from` or `mentions`).
- Fixed an issue where a text message filter didn't work correctly if it was enclosed in parentheses.
- Fixed an issue where categories were displayed alongside regular channels in the channel list, both in GUI and in CLI (when using the `guild` command).
- [CLI] Fixed an issue where an output path starting with a tilde on Linux was not expanded properly into a full path. (Thanks [@TSRBerry](https://github.com/TSRBerry))
### v2.35 (02-Jul-2022)
- Added support for exporting text chat in voice channels. These channels are now also displayed when browsing in the GUI and in the CLI when executing the `channels` command.
- [GUI] Direct message channels are now sorted by the timestamp of their last message, similarly to how they appear in the Discord client.
- [GUI] Removed artificial grouping of direct message channels into "Private" and "Group".
- [CLI] Added runtime bootstrapper capabilities to the CLI flavor of DiscordChatExporter on Windows. Now, if you attempt to run the application without the required prerequisites installed, you will be provided with an option to install them automatically.
- [CLI] Fixed outdated information in the usage guide. (Thanks [@Clint Herron](https://github.com/HanClinto))
- [CLI] Added whitespace padding to the output of `guilds`, `channels`, and `dm` to make the tables look more uniform.
- [HTML] Added hyperlinks to message timestamps, which allows you to quickly copy a link to a specific message in the export. (Thanks [@Philipp C. Heckel](https://github.com/binwiederhier))
- [HTML] Added minification. Chat exports in HTML format are now 30-35% smaller than before.
- [HTML] Added support for rendering embeds of Twitter posts that contain multiple images. Previously, this resulted in multiple separate embeds instead of one.
- [HTML] Added support for rendering embeds of YouTube Shorts videos.
- [HTML] Text content is now hidden if the message only contains a link to an image and nothing else. The link itself is resolved as an image embed.
- [HTML] Fixed an issue where some emoji that included a zero-width-joiner rune were not rendered correctly due to a wrong Twemoji URL being generated. (Thanks [@Ethan](https://github.com/ethanndickson))
- [HTML] Fixed an issue where replies to a message that contained a quote (i.e. `>`) were not rendered correctly.
- [HTML] Fixed an issue where code blocks with language highlighting did not have the correct background color.
- Added file name template token that resolves to the current date (`%d`). (Thanks [@Lucas LaBuff](https://github.com/96-LB))
- Updated the usage guide to replace the instructions for retrieving the user token with those that appear to work for more people.
- Updated the usage guide to feature the TOS warning more prominently.
- [Docker] Changed internal working directory from `/app/out` to `/out`. This was an unintended breaking change. Please update your volume binding configuration from `-v /path/on/machine:/app/out` to `-v /path/on/machine:/out`.
### v2.34.1 (01-Jun-2022)
- Improved retry policy to handle more transient errors. This should result in fewer errors exporting a channel due to timeout.
- Added `has:pin` message filter. You can use it to limit exported messages to those that have been pinned. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
- Fixed an issue which caused the export to fail when processing stickers that have no name.
### v2.34 (10-Apr-2022)
- [HTML] Reworked layout to be more consistent with Discord.
- [HTML] Changed user profile picture dimensions from `128x128` to `512x512`. (Thanks [@Chris Helder](https://github.com/TheDude53))
- [HTML] Improved image scaling for Chromium-based browsers. This should result in higher quality images across the entire export.
- [HTML] Hovering over messages will now show short timestamps on the left, indicating the time that the individual messages of the group were sent.
- [HTML] Using "download media" option will now also download scripts, styles, fonts, and other similar resources used by the export.
- [HTML] Image attachments will now use the provided description as alt text.
- [HTML] Fixed a few layout inconsistencies related to embeds.
- [GUI] Added a few clickable links to the usage guide to make the process of locating tokens easier.
### v2.33.2 (07-Mar-2022)
- Actually fixed it this time.
### v2.33.1 (07-Mar-2022)
- Fixed an issue where the application silently failed to run if the system didn't have .NET Runtime 6.0.2 installed. If you continue seeing this issue, please uninstall all existing .NET runtimes from your computer and then try running the application again.
### v2.33 (06-Mar-2022)
- Added messages informing about war in Ukraine and available ways to help.
- Added support for rendering stickers in HTML and JSON. Lottie-based stickers currently cannot be displayed in HTML exports (see [#803](https://github.com/Tyrrrz/DiscordChatExporter/issues/803)).
- Added a new `reaction:` message filter which can be used to check if someone reacted to a message with a specific emoji. You can either pass the emoji name (e.g. `reaction:LUL`) or its ID (e.g. `reaction:41771983429993937`).
- [GUI] Added auto-detection for dark mode. If your system is configured to prefer dark mode in applications, DiscordChatExporter will use it by default instead of light mode.
- Fixed an issue which caused the export process to crash when downloading media files with extremely long file extensions. (Thanks [@Tomlacko](https://github.com/Tomlacko))
- Fixed an issue which caused the export process to crash on invalid mentions.
- [GUI] Fixed an issue where the time pickers used to specify export ranges always displayed time in 12-hour format, instead of respecting the system locale.
### v2.32 (27-Jan-2022)
- Token kind (user or bot) is now detected automatically. Removed the button to switch token kind in GUI. Option `-b|--bot` in CLI is now deprecated and does nothing.
- Updated user token extraction guide to reflect the fact that devtools are no longer accessible in the desktop version of Discord client. The recommended workaround is to open Discord in browser. (Thanks [@Dhananjay-JSR](https://github.com/Dhananjay-JSR))
- Fixed an issue which caused an error when parsing invalid date timestamps. Such timestamps are now rendered as "Invalid date", similarly to how the Discord client does it.
- Fixed an issue which caused an error when parsing certain embed fields.
- Fixed an issue which caused an error when parsing a mention to a user that has no username.
- [GUI] Minor visual fixes.
### v2.31.1 (15-Dec-2021)
- Fixed an issue which caused an exception when parsing certain custom emoji reactions.
- Fixed an issue which caused an error when parsing certain custom emoji reactions.
- [GUI] Improved user interface.
### v2.31 (06-Dec-2021)
+2 -2
View File
@@ -2,9 +2,9 @@
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<Version>2.31.1</Version>
<Version>2.35.1</Version>
<Company>Tyrrrz</Company>
<Copyright>Copyright (c) Alexey Golub</Copyright>
<Copyright>Copyright (c) Oleksii Holub</Copyright>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<WarningsAsErrors>nullable</WarningsAsErrors>
@@ -1,28 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
<CollectCoverage>true</CollectCoverage>
<CoverletOutputFormat>opencover</CoverletOutputFormat>
<NoWarn>$(NoWarn);xUnit1013</NoWarn>
</PropertyGroup>
<ItemGroup>
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
<None Include="*.secret" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AngleSharp" Version="0.16.1" />
<PackageReference Include="FluentAssertions" Version="6.2.0" />
<PackageReference Include="GitHubActionsTestLogger" Version="1.2.0" />
<PackageReference Include="AngleSharp" Version="0.17.1" />
<PackageReference Include="FluentAssertions" Version="6.7.0" />
<PackageReference Include="GitHubActionsTestLogger" Version="2.0.1" PrivateAssets="all" />
<PackageReference Include="JsonExtensions" Version="1.2.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
<PackageReference Include="coverlet.msbuild" Version="3.1.0" PrivateAssets="all" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.0" />
<PackageReference Include="System.Reactive" Version="5.0.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" PrivateAssets="all" />
<PackageReference Include="xunit" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5" PrivateAssets="all" />
<PackageReference Include="coverlet.collector" Version="3.1.2" PrivateAssets="all" />
</ItemGroup>
<ItemGroup>
@@ -36,8 +36,7 @@ public class ExportWrapperFixture : IDisposable
{
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { channelId },
ExportFormat = format,
OutputPath = filePath
@@ -47,58 +46,46 @@ public class ExportWrapperFixture : IDisposable
return await File.ReadAllTextAsync(filePath);
}
public async ValueTask<IHtmlDocument> ExportAsHtmlAsync(Snowflake channelId)
{
var data = await ExportAsync(channelId, ExportFormat.HtmlDark);
return Html.Parse(data);
}
public async ValueTask<IHtmlDocument> ExportAsHtmlAsync(Snowflake channelId) => Html.Parse(
await ExportAsync(channelId, ExportFormat.HtmlDark)
);
public async ValueTask<JsonElement> ExportAsJsonAsync(Snowflake channelId)
{
var data = await ExportAsync(channelId, ExportFormat.Json);
return Json.Parse(data);
}
public async ValueTask<JsonElement> ExportAsJsonAsync(Snowflake channelId) => Json.Parse(
await ExportAsync(channelId, ExportFormat.Json)
);
public async ValueTask<string> ExportAsPlainTextAsync(Snowflake channelId)
{
var data = await ExportAsync(channelId, ExportFormat.PlainText);
return data;
}
public async ValueTask<string> ExportAsPlainTextAsync(Snowflake channelId) =>
await ExportAsync(channelId, ExportFormat.PlainText);
public async ValueTask<string> ExportAsCsvAsync(Snowflake channelId)
{
var data = await ExportAsync(channelId, ExportFormat.Csv);
return data;
}
public async ValueTask<string> ExportAsCsvAsync(Snowflake channelId) =>
await ExportAsync(channelId, ExportFormat.Csv);
public async ValueTask<IReadOnlyList<IElement>> GetMessagesAsHtmlAsync(Snowflake channelId)
{
var document = await ExportAsHtmlAsync(channelId);
return document.QuerySelectorAll("[data-message-id]").ToArray();
}
public async ValueTask<IReadOnlyList<IElement>> GetMessagesAsHtmlAsync(Snowflake channelId) =>
(await ExportAsHtmlAsync(channelId))
.QuerySelectorAll("[data-message-id]")
.ToArray();
public async ValueTask<IReadOnlyList<JsonElement>> GetMessagesAsJsonAsync(Snowflake channelId)
{
var document = await ExportAsJsonAsync(channelId);
return document.GetProperty("messages").EnumerateArray().ToArray();
}
public async ValueTask<IReadOnlyList<JsonElement>> GetMessagesAsJsonAsync(Snowflake channelId) =>
(await ExportAsJsonAsync(channelId))
.GetProperty("messages")
.EnumerateArray()
.ToArray();
public async ValueTask<IElement> GetMessageAsHtmlAsync(Snowflake channelId, Snowflake messageId)
{
var messages = await GetMessagesAsHtmlAsync(channelId);
var message = messages.SingleOrDefault(e =>
string.Equals(
e.GetAttribute("data-message-id"),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
);
var message = (await GetMessagesAsHtmlAsync(channelId))
.SingleOrDefault(e =>
string.Equals(
e.GetAttribute("data-message-id"),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
);
if (message is null)
{
throw new InvalidOperationException(
$"Message '{messageId}' does not exist in export of channel '{channelId}'."
$"Message '{messageId}' does not exist in the export of channel '{channelId}'."
);
}
@@ -107,20 +94,19 @@ public class ExportWrapperFixture : IDisposable
public async ValueTask<JsonElement> GetMessageAsJsonAsync(Snowflake channelId, Snowflake messageId)
{
var messages = await GetMessagesAsJsonAsync(channelId);
var message = messages.FirstOrDefault(j =>
string.Equals(
j.GetProperty("id").GetString(),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
);
var message = (await GetMessagesAsJsonAsync(channelId))
.SingleOrDefault(j =>
string.Equals(
j.GetProperty("id").GetString(),
messageId.ToString(),
StringComparison.OrdinalIgnoreCase
)
);
if (message.ValueKind == JsonValueKind.Undefined)
{
throw new InvalidOperationException(
$"Message '{messageId}' does not exist in export of channel '{channelId}'."
$"Message '{messageId}' does not exist in the export of channel '{channelId}'."
);
}
@@ -22,24 +22,5 @@ internal static class Secrets
throw new InvalidOperationException("Discord token not provided for tests.");
});
private static readonly Lazy<bool> IsDiscordTokenBotLazy = new(() =>
{
var fromEnvironment = Environment.GetEnvironmentVariable("DISCORD_TOKEN_BOT");
if (!string.IsNullOrWhiteSpace(fromEnvironment))
return string.Equals(fromEnvironment, "true", StringComparison.OrdinalIgnoreCase);
var secretFilePath = Path.Combine(
Path.GetDirectoryName(typeof(Secrets).Assembly.Location) ?? Directory.GetCurrentDirectory(),
"DiscordTokenBot.secret"
);
if (File.Exists(secretFilePath))
return true;
return false;
});
public static string DiscordToken => DiscordTokenLazy.Value;
public static bool IsDiscordTokenBot => IsDiscordTokenBotLazy.Value;
}
+2 -3
View File
@@ -3,12 +3,11 @@
This test suite runs against a real Discord server, specifically created to exercise different behaviors required by the test scenarios.
In order to run these tests locally, you need to join the test server and configure your authentication token.
1. Join the server: https://discord.gg/eRV8Vap5bm
1. [Join the test server](https://discord.gg/eRV8Vap5bm)
2. Locate your Discord authentication token
3. Specify your token using a file or an environment variable:
- **Using a file**: put your token in a new `DiscordToken.secret` file created in this directory
- **Using an environment variable**: set `DISCORD_TOKEN` variable to your token
4. Run the tests: `dotnet test`
> If you want to have a new test case or a scenario added, please let me know in your pull request.
Currently, it's not possible to add them by yourself.
> If you want to add a new test case, please let me know and I will give you the required permissions
@@ -6,13 +6,20 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.CsvWriting;
public record ContentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class ContentSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public ContentSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Messages_are_exported_correctly()
{
// Act
var document = await ExportWrapper.ExportAsCsvAsync(ChannelIds.DateRangeTestCases);
var document = await _exportWrapper.ExportAsCsvAsync(ChannelIds.DateRangeTestCases);
// Assert
document.Should().ContainAll(
@@ -15,36 +15,40 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs;
public record DateRangeSpecs(TempOutputFixture TempOutput) : IClassFixture<TempOutputFixture>
public class DateRangeSpecs : IClassFixture<TempOutputFixture>
{
private readonly TempOutputFixture _tempOutput;
public DateRangeSpecs(TempOutputFixture tempOutput)
{
_tempOutput = tempOutput;
}
[Fact]
public async Task Messages_filtered_after_specific_date_only_include_messages_sent_after_that_date()
{
// Arrange
var after = new DateTimeOffset(2021, 07, 24, 0, 0, 0, TimeSpan.Zero);
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.DateRangeTestCases },
ExportFormat = ExportFormat.Json,
OutputPath = filePath,
After = Snowflake.FromDate(after)
}.ExecuteAsync(new FakeConsole());
var data = await File.ReadAllTextAsync(filePath);
var document = Json.Parse(data);
var timestamps = document
// Assert
var timestamps = Json
.Parse(await File.ReadAllTextAsync(filePath))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("timestamp").GetDateTimeOffset())
.ToArray();
// Assert
timestamps.All(t => t > after).Should().BeTrue();
timestamps.Should().BeEquivalentTo(new[]
@@ -69,29 +73,26 @@ public record DateRangeSpecs(TempOutputFixture TempOutput) : IClassFixture<TempO
{
// Arrange
var before = new DateTimeOffset(2021, 07, 24, 0, 0, 0, TimeSpan.Zero);
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.DateRangeTestCases },
ExportFormat = ExportFormat.Json,
OutputPath = filePath,
Before = Snowflake.FromDate(before)
}.ExecuteAsync(new FakeConsole());
var data = await File.ReadAllTextAsync(filePath);
var document = Json.Parse(data);
var timestamps = document
// Assert
var timestamps = Json
.Parse(await File.ReadAllTextAsync(filePath))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("timestamp").GetDateTimeOffset())
.ToArray();
// Assert
timestamps.All(t => t < before).Should().BeTrue();
timestamps.Should().BeEquivalentTo(new[]
@@ -115,13 +116,12 @@ public record DateRangeSpecs(TempOutputFixture TempOutput) : IClassFixture<TempO
// Arrange
var after = new DateTimeOffset(2021, 07, 24, 0, 0, 0, TimeSpan.Zero);
var before = new DateTimeOffset(2021, 08, 01, 0, 0, 0, TimeSpan.Zero);
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.DateRangeTestCases },
ExportFormat = ExportFormat.Json,
OutputPath = filePath,
@@ -129,16 +129,14 @@ public record DateRangeSpecs(TempOutputFixture TempOutput) : IClassFixture<TempO
After = Snowflake.FromDate(after)
}.ExecuteAsync(new FakeConsole());
var data = await File.ReadAllTextAsync(filePath);
var document = Json.Parse(data);
var timestamps = document
// Assert
var timestamps = Json
.Parse(await File.ReadAllTextAsync(filePath))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("timestamp").GetDateTimeOffset())
.ToArray();
// Assert
timestamps.All(t => t < before && t > after).Should().BeTrue();
timestamps.Should().BeEquivalentTo(new[]
@@ -14,30 +14,34 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs;
public record FilterSpecs(TempOutputFixture TempOutput) : IClassFixture<TempOutputFixture>
public class FilterSpecs : IClassFixture<TempOutputFixture>
{
private readonly TempOutputFixture _tempOutput;
public FilterSpecs(TempOutputFixture tempOutput)
{
_tempOutput = tempOutput;
}
[Fact]
public async Task Messages_filtered_by_text_only_include_messages_that_contain_that_text()
{
// Arrange
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.FilterTestCases },
ExportFormat = ExportFormat.Json,
OutputPath = filePath,
MessageFilter = MessageFilter.Parse("some text")
}.ExecuteAsync(new FakeConsole());
var data = await File.ReadAllTextAsync(filePath);
var document = Json.Parse(data);
// Assert
document
Json
.Parse(await File.ReadAllTextAsync(filePath))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
@@ -49,24 +53,21 @@ public record FilterSpecs(TempOutputFixture TempOutput) : IClassFixture<TempOutp
public async Task Messages_filtered_by_author_only_include_messages_sent_by_that_author()
{
// Arrange
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.FilterTestCases },
ExportFormat = ExportFormat.Json,
OutputPath = filePath,
MessageFilter = MessageFilter.Parse("from:Tyrrrz")
}.ExecuteAsync(new FakeConsole());
var data = await File.ReadAllTextAsync(filePath);
var document = Json.Parse(data);
// Assert
document
Json
.Parse(await File.ReadAllTextAsync(filePath))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("author").GetProperty("name").GetString())
@@ -78,24 +79,21 @@ public record FilterSpecs(TempOutputFixture TempOutput) : IClassFixture<TempOutp
public async Task Messages_filtered_by_content_only_include_messages_that_have_that_content()
{
// Arrange
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.FilterTestCases },
ExportFormat = ExportFormat.Json,
OutputPath = filePath,
MessageFilter = MessageFilter.Parse("has:image")
}.ExecuteAsync(new FakeConsole());
var data = await File.ReadAllTextAsync(filePath);
var document = Json.Parse(data);
// Assert
document
Json
.Parse(await File.ReadAllTextAsync(filePath))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
@@ -104,27 +102,50 @@ public record FilterSpecs(TempOutputFixture TempOutput) : IClassFixture<TempOutp
}
[Fact]
public async Task Messages_filtered_by_mention_only_include_messages_that_have_that_mention()
public async Task Messages_filtered_by_pin_only_include_messages_that_have_been_pinned()
{
// Arrange
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.FilterTestCases },
ExportFormat = ExportFormat.Json,
OutputPath = filePath,
MessageFilter = MessageFilter.Parse("has:pin")
}.ExecuteAsync(new FakeConsole());
// Assert
Json
.Parse(await File.ReadAllTextAsync(filePath))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
.Should()
.ContainSingle("This is pinned");
}
[Fact]
public async Task Messages_filtered_by_mention_only_include_messages_that_have_that_mention()
{
// Arrange
var filePath = _tempOutput.GetTempFilePath();
// Act
await new ExportChannelsCommand
{
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.FilterTestCases },
ExportFormat = ExportFormat.Json,
OutputPath = filePath,
MessageFilter = MessageFilter.Parse("mentions:Tyrrrz")
}.ExecuteAsync(new FakeConsole());
var data = await File.ReadAllTextAsync(filePath);
var document = Json.Parse(data);
// Assert
document
Json
.Parse(await File.ReadAllTextAsync(filePath))
.GetProperty("messages")
.EnumerateArray()
.Select(j => j.GetProperty("content").GetString())
@@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.Linq;
using System.Threading.Tasks;
using AngleSharp.Dom;
using DiscordChatExporter.Cli.Tests.Fixtures;
using DiscordChatExporter.Cli.Tests.TestData;
@@ -8,19 +9,24 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.HtmlWriting;
public record AttachmentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class AttachmentSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public AttachmentSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Message_with_a_generic_attachment_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.AttachmentTestCases,
Snowflake.Parse("885587844989612074")
);
var fileUrl = message.QuerySelector("a")?.GetAttribute("href");
// Assert
message.Text().Should().ContainAll(
"Generic file attachment",
@@ -28,45 +34,52 @@ public record AttachmentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixtur
"11 bytes"
);
fileUrl.Should().StartWithEquivalentOf(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
);
message
.QuerySelectorAll("a")
.Select(e => e.GetAttribute("href"))
.Should()
.Contain(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
);
}
[Fact]
public async Task Message_with_an_image_attachment_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.AttachmentTestCases,
Snowflake.Parse("885654862656843786")
);
var imageUrl = message.QuerySelector("img")?.GetAttribute("src");
// Assert
message.Text().Should().Contain("Image attachment");
imageUrl.Should().StartWithEquivalentOf(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
);
message
.QuerySelectorAll("img")
.Select(e => e.GetAttribute("src"))
.Should()
.Contain(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
);
}
[Fact]
public async Task Message_with_a_video_attachment_is_rendered_correctly()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/333
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.AttachmentTestCases,
Snowflake.Parse("885655761919836171")
);
var videoUrl = message.QuerySelector("video source")?.GetAttribute("src");
// Assert
message.Text().Should().Contain("Video attachment");
videoUrl.Should().StartWithEquivalentOf(
var videoUrl = message.QuerySelector("video source")?.GetAttribute("src");
videoUrl.Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
}
@@ -74,18 +87,19 @@ public record AttachmentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixtur
[Fact]
public async Task Message_with_an_audio_attachment_is_rendered_correctly()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/333
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.AttachmentTestCases,
Snowflake.Parse("885656175620808734")
);
var audioUrl = message.QuerySelector("audio source")?.GetAttribute("src");
// Assert
message.Text().Should().Contain("Audio attachment");
audioUrl.Should().StartWithEquivalentOf(
var audioUrl = message.QuerySelector("audio source")?.GetAttribute("src");
audioUrl.Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
}
@@ -8,13 +8,20 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.HtmlWriting;
public record ContentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class ContentSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public ContentSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Messages_are_exported_correctly()
{
// Act
var messages = await ExportWrapper.GetMessagesAsHtmlAsync(ChannelIds.DateRangeTestCases);
var messages = await _exportWrapper.GetMessagesAsHtmlAsync(ChannelIds.DateRangeTestCases);
// Assert
messages.Select(e => e.GetAttribute("data-message-id")).Should().Equal(
@@ -28,7 +35,7 @@ public record ContentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<E
"885169254029213696"
);
messages.Select(e => e.QuerySelector(".chatlog__content")?.Text().Trim()).Should().Equal(
messages.SelectMany(e => e.Text()).Should().ContainInOrder(
"Hello world",
"Goodbye world",
"Foo bar",
@@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.Linq;
using System.Threading.Tasks;
using AngleSharp.Dom;
using DiscordChatExporter.Cli.Tests.Fixtures;
using DiscordChatExporter.Cli.Tests.TestData;
@@ -8,13 +9,20 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.HtmlWriting;
public record EmbedSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class EmbedSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public EmbedSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Message_with_an_embed_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.EmbedTestCases,
Snowflake.Parse("866769910729146400")
);
@@ -32,32 +40,95 @@ public record EmbedSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<Exp
}
[Fact]
public async Task Message_with_a_Spotify_track_is_rendered_using_an_iframe()
public async Task Message_containing_an_image_link_is_rendered_with_an_image_embed()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/537
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.EmbedTestCases,
Snowflake.Parse("991768701126852638")
);
// Assert
message
.QuerySelectorAll("img")
.Select(e => e.GetAttribute("src"))
.Should()
.Contain("https://i.redd.it/f8w05ja8s4e61.png");
}
[Fact]
public async Task Message_containing_an_image_link_and_nothing_else_is_rendered_without_text_content()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/682
// Act
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.EmbedTestCases,
Snowflake.Parse("991768701126852638")
);
// Assert
var content = message.QuerySelector(".chatlog__content")?.Text();
content.Should().BeNullOrEmpty();
}
[Fact]
public async Task Message_containing_a_Spotify_track_link_is_rendered_with_a_track_embed()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/657
// Act
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.EmbedTestCases,
Snowflake.Parse("867886632203976775")
);
var iframeSrc = message.QuerySelector("iframe")?.GetAttribute("src");
// Assert
iframeSrc.Should().StartWithEquivalentOf("https://open.spotify.com/embed/track/1LHZMWefF9502NPfArRfvP");
var iframeUrl = message.QuerySelector("iframe")?.GetAttribute("src");
iframeUrl.Should().Be("https://open.spotify.com/embed/track/1LHZMWefF9502NPfArRfvP");
}
[Fact]
public async Task Message_with_a_YouTube_video_is_rendered_using_an_iframe()
public async Task Message_containing_a_YouTube_video_link_is_rendered_with_a_video_embed()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/570
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.EmbedTestCases,
Snowflake.Parse("866472508588294165")
);
var iframeSrc = message.QuerySelector("iframe")?.GetAttribute("src");
// Assert
var iframeUrl = message.QuerySelector("iframe")?.GetAttribute("src");
iframeUrl.Should().Be("https://www.youtube.com/embed/qOWW4OlgbvE");
}
[Fact]
public async Task Message_containing_a_Twitter_post_link_with_multiple_images_is_rendered_as_a_single_embed()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/695
// Act
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.EmbedTestCases,
Snowflake.Parse("991757444017557665")
);
// Assert
iframeSrc.Should().StartWithEquivalentOf("https://www.youtube.com/embed/qOWW4OlgbvE");
message
.QuerySelectorAll("img")
.Select(e => e.GetAttribute("src"))
.Should()
.ContainInOrder(
"https://images-ext-1.discordapp.net/external/-n--xW3EHH_3jlrheVkMXHCM7T86b5Ty4-MzXCT4m1Q/https/pbs.twimg.com/media/FVYIzYPWAAAMBqZ.png",
"https://images-ext-2.discordapp.net/external/z5nEmGeEldV-kswydGLhqUsFHbb5AWHtdvc9XT6N5rE/https/pbs.twimg.com/media/FVYJBWJWAAMNAx2.png",
"https://images-ext-2.discordapp.net/external/gnip03SawMB6uZLagN5sRDpA_1Ap1CcEhMbJfK1z6WQ/https/pbs.twimg.com/media/FVYJHiRX0AANZcz.png",
"https://images-ext-2.discordapp.net/external/jl1v6cCbLaGmiwmKU-ZkXnF4cFsJ39f9A3-oEdqPdZs/https/pbs.twimg.com/media/FVYJNZNXwAAPnVG.png"
);
message.QuerySelectorAll(".chatlog__embed").Should().ContainSingle();
}
}
@@ -0,0 +1,79 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using AngleSharp.Dom;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands;
using DiscordChatExporter.Cli.Tests.Fixtures;
using DiscordChatExporter.Cli.Tests.Infra;
using DiscordChatExporter.Cli.Tests.TestData;
using DiscordChatExporter.Cli.Tests.Utils;
using DiscordChatExporter.Core.Exporting;
using FluentAssertions;
using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.HtmlWriting;
public class GroupingSpecs : IClassFixture<TempOutputFixture>
{
private readonly TempOutputFixture _tempOutput;
public GroupingSpecs(TempOutputFixture tempOutput)
{
_tempOutput = tempOutput;
}
[Fact]
public async Task Messages_are_grouped_correctly()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/152
// Arrange
var filePath = _tempOutput.GetTempFilePath();
// Act
await new ExportChannelsCommand
{
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.GroupingTestCases },
ExportFormat = ExportFormat.HtmlDark,
OutputPath = filePath
}.ExecuteAsync(new FakeConsole());
// Assert
var messageGroups = Html
.Parse(await File.ReadAllTextAsync(filePath))
.QuerySelectorAll(".chatlog__message-group");
messageGroups.Should().HaveCount(2);
messageGroups[0]
.QuerySelectorAll(".chatlog__content")
.Select(e => e.Text())
.Should()
.ContainInOrder(
"First",
"Second",
"Third",
"Fourth",
"Fifth",
"Sixth",
"Seventh",
"Eighth",
"Ninth",
"Tenth"
);
messageGroups[1]
.QuerySelectorAll(".chatlog__content")
.Select(e => e.Text())
.Should()
.ContainInOrder(
"Eleventh",
"Twelveth",
"Thirteenth",
"Fourteenth",
"Fifteenth"
);
}
}
@@ -8,19 +8,26 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.HtmlWriting;
public record MentionSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class MentionSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public MentionSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task User_mention_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MentionTestCases,
Snowflake.Parse("866458840245076028")
);
// Assert
message.Text().Trim().Should().Be("User mention: @Tyrrrz");
message.Text().Should().Contain("User mention: @Tyrrrz");
message.InnerHtml.Should().Contain("Tyrrrz#5447");
}
@@ -28,38 +35,38 @@ public record MentionSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<E
public async Task Text_channel_mention_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MentionTestCases,
Snowflake.Parse("866459040480624680")
);
// Assert
message.Text().Trim().Should().Be("Text channel mention: #mention-tests");
message.Text().Should().Contain("Text channel mention: #mention-tests");
}
[Fact]
public async Task Voice_channel_mention_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MentionTestCases,
Snowflake.Parse("866459175462633503")
);
// Assert
message.Text().Trim().Should().Be("Voice channel mention: 🔊chaos-vc");
message.Text().Should().Contain("Voice channel mention: 🔊chaos-vc");
}
[Fact]
public async Task Role_mention_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.MentionTestCases,
Snowflake.Parse("866459254693429258")
);
// Assert
message.Text().Trim().Should().Be("Role mention: @Role 1");
message.Text().Should().Contain("Role mention: @Role 1");
}
}
@@ -8,49 +8,60 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.HtmlWriting;
public record ReplySpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class ReplySpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public ReplySpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Reply_to_a_normal_message_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.ReplyTestCases,
Snowflake.Parse("866460738239725598")
);
// Assert
message.Text().Trim().Should().Be("reply to original");
message.QuerySelector(".chatlog__reference-link")?.Text().Trim().Should().Be("original");
message.Text().Should().Contain("reply to original");
message.QuerySelector(".chatlog__reference-link")?.Text().Should().Contain("original");
}
[Fact]
public async Task Reply_to_a_deleted_message_is_rendered_correctly()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/645
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.ReplyTestCases,
Snowflake.Parse("866460975388819486")
);
// Assert
message.Text().Trim().Should().Be("reply to deleted");
message.QuerySelector(".chatlog__reference-link")?.Text().Trim().Should()
.Be("Original message was deleted or could not be loaded.");
message.Text().Should().Contain("reply to deleted");
message.QuerySelector(".chatlog__reference-link")?.Text().Should().Contain(
"Original message was deleted or could not be loaded."
);
}
[Fact]
public async Task Reply_to_a_empty_message_with_attachment_is_rendered_correctly()
public async Task Reply_to_an_empty_message_with_attachment_is_rendered_correctly()
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/634
// Act
var message = await ExportWrapper.GetMessageAsHtmlAsync(
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.ReplyTestCases,
Snowflake.Parse("866462470335627294")
);
// Assert
message.Text().Trim().Should().Be("reply to attachment");
message.QuerySelector(".chatlog__reference-link")?.Text().Trim().Should()
.Be("Click to see attachment 🖼️");
message.Text().Should().Contain("reply to attachment");
message.QuerySelector(".chatlog__reference-link")?.Text().Should().Contain("Click to see attachment");
}
}
@@ -0,0 +1,46 @@
using System.Threading.Tasks;
using DiscordChatExporter.Cli.Tests.Fixtures;
using DiscordChatExporter.Cli.Tests.TestData;
using DiscordChatExporter.Core.Discord;
using FluentAssertions;
using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.HtmlWriting;
public class StickerSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public StickerSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Message_with_a_PNG_based_sticker_is_rendered_correctly()
{
// Act
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.StickerTestCases,
Snowflake.Parse("939670623158943754")
);
// Assert
var stickerUrl = message.QuerySelector("[title='rock'] img")?.GetAttribute("src");
stickerUrl.Should().Be("https://discord.com/stickers/904215665597120572.png");
}
[Fact]
public async Task Message_with_a_Lottie_based_sticker_is_rendered_correctly()
{
// Act
var message = await _exportWrapper.GetMessageAsHtmlAsync(
ChannelIds.StickerTestCases,
Snowflake.Parse("939670526517997590")
);
// Assert
var stickerUrl = message.QuerySelector("[title='Yikes'] [data-source]")?.GetAttribute("data-source");
stickerUrl.Should().Be("https://discord.com/stickers/816087132447178774.json");
}
}
@@ -8,24 +8,30 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.JsonWriting;
public record AttachmentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class AttachmentSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public AttachmentSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Message_with_a_generic_attachment_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.AttachmentTestCases,
Snowflake.Parse("885587844989612074")
);
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
// Assert
message.GetProperty("content").GetString().Should().Be("Generic file attachment");
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments.Single().GetProperty("url").GetString().Should().StartWithEquivalentOf(
attachments.Single().GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885587844964417596/Test.txt"
);
attachments.Single().GetProperty("fileName").GetString().Should().Be("Test.txt");
@@ -36,18 +42,17 @@ public record AttachmentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixtur
public async Task Message_with_an_image_attachment_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.AttachmentTestCases,
Snowflake.Parse("885654862656843786")
);
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
// Assert
message.GetProperty("content").GetString().Should().Be("Image attachment");
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments.Single().GetProperty("url").GetString().Should().StartWithEquivalentOf(
attachments.Single().GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885654862430359613/bird-thumbnail.png"
);
attachments.Single().GetProperty("fileName").GetString().Should().Be("bird-thumbnail.png");
@@ -58,18 +63,17 @@ public record AttachmentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixtur
public async Task Message_with_a_video_attachment_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.AttachmentTestCases,
Snowflake.Parse("885655761919836171")
);
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
// Assert
message.GetProperty("content").GetString().Should().Be("Video attachment");
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments.Single().GetProperty("url").GetString().Should().StartWithEquivalentOf(
attachments.Single().GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885655761512968233/file_example_MP4_640_3MG.mp4"
);
attachments.Single().GetProperty("fileName").GetString().Should().Be("file_example_MP4_640_3MG.mp4");
@@ -80,18 +84,17 @@ public record AttachmentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixtur
public async Task Message_with_an_audio_attachment_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.AttachmentTestCases,
Snowflake.Parse("885656175620808734")
);
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
// Assert
message.GetProperty("content").GetString().Should().Be("Audio attachment");
var attachments = message.GetProperty("attachments").EnumerateArray().ToArray();
attachments.Should().HaveCount(1);
attachments.Single().GetProperty("url").GetString().Should().StartWithEquivalentOf(
attachments.Single().GetProperty("url").GetString().Should().Be(
"https://cdn.discordapp.com/attachments/885587741654536192/885656175348187146/file_example_MP3_1MG.mp3"
);
attachments.Single().GetProperty("fileName").GetString().Should().Be("file_example_MP3_1MG.mp3");
@@ -7,13 +7,20 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.JsonWriting;
public record ContentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class ContentSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public ContentSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Messages_are_exported_correctly()
{
// Act
var messages = await ExportWrapper.GetMessagesAsJsonAsync(ChannelIds.DateRangeTestCases);
var messages = await _exportWrapper.GetMessagesAsJsonAsync(ChannelIds.DateRangeTestCases);
// Assert
messages.Select(j => j.GetProperty("id").GetString()).Should().Equal(
@@ -8,41 +8,47 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.JsonWriting;
public record EmbedSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class EmbedSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public EmbedSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Message_with_an_embed_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.EmbedTestCases,
Snowflake.Parse("866769910729146400")
);
var embed = message
.GetProperty("embeds")
.EnumerateArray()
.Single();
var embedAuthor = embed.GetProperty("author");
var embedThumbnail = embed.GetProperty("thumbnail");
var embedFooter = embed.GetProperty("footer");
var embedFields = embed.GetProperty("fields").EnumerateArray().ToArray();
// Assert
var embed = message.GetProperty("embeds").EnumerateArray().Single();
embed.GetProperty("title").GetString().Should().Be("Embed title");
embed.GetProperty("url").GetString().Should().Be("https://example.com");
embed.GetProperty("timestamp").GetString().Should().Be("2021-07-14T21:00:00+00:00");
embed.GetProperty("description").GetString().Should().Be("**Embed** _description_");
embed.GetProperty("color").GetString().Should().Be("#58B9FF");
var embedAuthor = embed.GetProperty("author");
embedAuthor.GetProperty("name").GetString().Should().Be("Embed author");
embedAuthor.GetProperty("url").GetString().Should().Be("https://example.com/author");
embedAuthor.GetProperty("iconUrl").GetString().Should().NotBeNullOrWhiteSpace();
var embedThumbnail = embed.GetProperty("thumbnail");
embedThumbnail.GetProperty("url").GetString().Should().NotBeNullOrWhiteSpace();
embedThumbnail.GetProperty("width").GetInt32().Should().Be(120);
embedThumbnail.GetProperty("height").GetInt32().Should().Be(120);
var embedFooter = embed.GetProperty("footer");
embedFooter.GetProperty("text").GetString().Should().Be("Embed footer");
embedFooter.GetProperty("iconUrl").GetString().Should().NotBeNullOrWhiteSpace();
var embedFields = embed.GetProperty("fields").EnumerateArray().ToArray();
embedFields.Should().HaveCount(3);
embedFields[0].GetProperty("name").GetString().Should().Be("Field 1");
embedFields[0].GetProperty("value").GetString().Should().Be("Value 1");
@@ -8,13 +8,20 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.JsonWriting;
public record MentionSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class MentionSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public MentionSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task User_mention_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.MentionTestCases,
Snowflake.Parse("866458840245076028")
);
@@ -26,14 +33,15 @@ public record MentionSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<E
.GetProperty("mentions")
.EnumerateArray()
.Select(j => j.GetProperty("id").GetString())
.Should().Contain("128178626683338752");
.Should()
.Contain("128178626683338752");
}
[Fact]
public async Task Text_channel_mention_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.MentionTestCases,
Snowflake.Parse("866459040480624680")
);
@@ -46,7 +54,7 @@ public record MentionSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<E
public async Task Voice_channel_mention_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.MentionTestCases,
Snowflake.Parse("866459175462633503")
);
@@ -59,7 +67,7 @@ public record MentionSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<E
public async Task Role_mention_is_rendered_correctly()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.MentionTestCases,
Snowflake.Parse("866459254693429258")
);
@@ -0,0 +1,61 @@
using System.Linq;
using System.Threading.Tasks;
using DiscordChatExporter.Cli.Tests.Fixtures;
using DiscordChatExporter.Cli.Tests.TestData;
using DiscordChatExporter.Core.Discord;
using FluentAssertions;
using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.JsonWriting;
public class StickerSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public StickerSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Message_with_a_PNG_based_sticker_is_rendered_correctly()
{
// Act
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.StickerTestCases,
Snowflake.Parse("939670623158943754")
);
// Assert
var sticker = message
.GetProperty("stickers")
.EnumerateArray()
.Single();
sticker.GetProperty("id").GetString().Should().Be("904215665597120572");
sticker.GetProperty("name").GetString().Should().Be("rock");
sticker.GetProperty("format").GetString().Should().Be("PngAnimated");
sticker.GetProperty("sourceUrl").GetString().Should().Be("https://discord.com/stickers/904215665597120572.png");
}
[Fact]
public async Task Message_with_a_Lottie_based_sticker_is_rendered_correctly()
{
// Act
var message = await _exportWrapper.GetMessageAsJsonAsync(
ChannelIds.StickerTestCases,
Snowflake.Parse("939670526517997590")
);
// Assert
var sticker = message
.GetProperty("stickers")
.EnumerateArray()
.Single();
sticker.GetProperty("id").GetString().Should().Be("816087132447178774");
sticker.GetProperty("name").GetString().Should().Be("Yikes");
sticker.GetProperty("format").GetString().Should().Be("Lottie");
sticker.GetProperty("sourceUrl").GetString().Should().Be("https://discord.com/stickers/816087132447178774.json");
}
}
@@ -12,21 +12,27 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs;
public record PartitioningSpecs(TempOutputFixture TempOutput) : IClassFixture<TempOutputFixture>
public class PartitioningSpecs : IClassFixture<TempOutputFixture>
{
private readonly TempOutputFixture _tempOutput;
public PartitioningSpecs(TempOutputFixture tempOutput)
{
_tempOutput = tempOutput;
}
[Fact]
public async Task Messages_partitioned_by_count_are_split_into_multiple_files_correctly()
public async Task Messages_partitioned_by_count_are_split_into_a_corresponding_number_of_files()
{
// Arrange
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(filePath);
var dirPath = Path.GetDirectoryName(filePath) ?? Directory.GetCurrentDirectory();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.DateRangeTestCases },
ExportFormat = ExportFormat.HtmlDark,
OutputPath = filePath,
@@ -40,27 +46,26 @@ public record PartitioningSpecs(TempOutputFixture TempOutput) : IClassFixture<Te
}
[Fact]
public async Task Messages_partitioned_by_file_size_are_split_into_multiple_files_correctly()
public async Task Messages_partitioned_by_file_size_are_split_into_a_corresponding_number_of_files()
{
// Arrange
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(filePath);
var dirPath = Path.GetDirectoryName(filePath) ?? Directory.GetCurrentDirectory();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.DateRangeTestCases },
ExportFormat = ExportFormat.HtmlDark,
OutputPath = filePath,
PartitionLimit = PartitionLimit.Parse("20kb")
PartitionLimit = PartitionLimit.Parse("1kb")
}.ExecuteAsync(new FakeConsole());
// Assert
Directory.EnumerateFiles(dirPath, fileNameWithoutExt + "*")
.Should()
.HaveCount(2);
.HaveCount(8);
}
}
@@ -6,13 +6,20 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs.PlainTextWriting;
public record ContentSpecs(ExportWrapperFixture ExportWrapper) : IClassFixture<ExportWrapperFixture>
public class ContentSpecs : IClassFixture<ExportWrapperFixture>
{
private readonly ExportWrapperFixture _exportWrapper;
public ContentSpecs(ExportWrapperFixture exportWrapper)
{
_exportWrapper = exportWrapper;
}
[Fact]
public async Task Messages_are_exported_correctly()
{
// Act
var document = await ExportWrapper.ExportAsPlainTextAsync(ChannelIds.DateRangeTestCases);
var document = await _exportWrapper.ExportAsPlainTextAsync(ChannelIds.DateRangeTestCases);
// Assert
document.Should().ContainAll(
@@ -13,31 +13,35 @@ using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs;
public record SelfContainedSpecs(TempOutputFixture TempOutput) : IClassFixture<TempOutputFixture>
public class SelfContainedSpecs : IClassFixture<TempOutputFixture>
{
private readonly TempOutputFixture _tempOutput;
public SelfContainedSpecs(TempOutputFixture tempOutput)
{
_tempOutput = tempOutput;
}
[Fact]
public async Task Messages_in_self_contained_export_only_reference_local_file_resources()
{
// Arrange
var filePath = TempOutput.GetTempFilePath();
var filePath = _tempOutput.GetTempFilePath();
var dirPath = Path.GetDirectoryName(filePath) ?? Directory.GetCurrentDirectory();
// Act
await new ExportChannelsCommand
{
TokenValue = Secrets.DiscordToken,
IsBotToken = Secrets.IsDiscordTokenBot,
Token = Secrets.DiscordToken,
ChannelIds = new[] { ChannelIds.SelfContainedTestCases },
ExportFormat = ExportFormat.HtmlDark,
OutputPath = filePath,
ShouldDownloadMedia = true
}.ExecuteAsync(new FakeConsole());
var data = await File.ReadAllTextAsync(filePath);
var document = Html.Parse(data);
// Assert
document
Html
.Parse(await File.ReadAllTextAsync(filePath))
.QuerySelectorAll("body [src]")
.Select(e => e.GetAttribute("src")!)
.Select(f => Path.GetFullPath(f, dirPath))
@@ -10,6 +10,8 @@ public static class ChannelIds
public static Snowflake EmbedTestCases { get; } = Snowflake.Parse("866472452459462687");
public static Snowflake GroupingTestCases { get; } = Snowflake.Parse("992092091545034842");
public static Snowflake FilterTestCases { get; } = Snowflake.Parse("866744075033641020");
public static Snowflake MentionTestCases { get; } = Snowflake.Parse("866458801389174794");
@@ -17,4 +19,6 @@ public static class ChannelIds
public static Snowflake ReplyTestCases { get; } = Snowflake.Parse("866459871934677052");
public static Snowflake SelfContainedTestCases { get; } = Snowflake.Parse("887441432678379560");
public static Snowflake StickerTestCases { get; } = Snowflake.Parse("939668868253769729");
}
+18
View File
@@ -0,0 +1,18 @@
# Build
FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build
COPY favicon.ico ./
COPY NuGet.config ./
COPY Directory.Build.props ./
COPY DiscordChatExporter.Core ./DiscordChatExporter.Core
COPY DiscordChatExporter.Cli ./DiscordChatExporter.Cli
RUN dotnet publish DiscordChatExporter.Cli -c Release -o ./publish
# Run
FROM mcr.microsoft.com/dotnet/runtime:6.0 AS run
COPY --from=build ./publish ./
WORKDIR ./out
ENTRYPOINT ["dotnet", "/DiscordChatExporter.Cli.dll"]
@@ -14,39 +14,82 @@ using DiscordChatExporter.Core.Exceptions;
using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Exporting.Filtering;
using DiscordChatExporter.Core.Exporting.Partitioning;
using DiscordChatExporter.Core.Utils;
using Gress;
namespace DiscordChatExporter.Cli.Commands.Base;
public abstract class ExportCommandBase : TokenCommandBase
{
[CommandOption("output", 'o', Description = "Output file or directory path.")]
public string OutputPath { get; init; } = Directory.GetCurrentDirectory();
private readonly string _outputPath = Directory.GetCurrentDirectory();
[CommandOption("format", 'f', Description = "Export format.")]
[CommandOption(
"output",
'o',
Description = "Output file or directory path."
)]
public string OutputPath
{
get => _outputPath;
// Handle ~/ in paths on *nix systems
// https://github.com/Tyrrrz/DiscordChatExporter/pull/903
init => _outputPath = Path.GetFullPath(value);
}
[CommandOption(
"format",
'f',
Description = "Export format."
)]
public ExportFormat ExportFormat { get; init; } = ExportFormat.HtmlDark;
[CommandOption("after", Description = "Only include messages sent after this date or message ID.")]
[CommandOption(
"after",
Description = "Only include messages sent after this date or message ID."
)]
public Snowflake? After { get; init; }
[CommandOption("before", Description = "Only include messages sent before this date or message ID.")]
[CommandOption(
"before",
Description = "Only include messages sent before this date or message ID."
)]
public Snowflake? Before { get; init; }
[CommandOption("partition", 'p', Description = "Split output into partitions, each limited to this number of messages (e.g. '100') or file size (e.g. '10mb').")]
[CommandOption(
"partition",
'p',
Description = "Split output into partitions, each limited to this number of messages (e.g. '100') or file size (e.g. '10mb')."
)]
public PartitionLimit PartitionLimit { get; init; } = PartitionLimit.Null;
[CommandOption("filter", Description = "Only include messages that satisfy this filter (e.g. 'from:foo#1234' or 'has:image').")]
[CommandOption(
"filter",
Description = "Only include messages that satisfy this filter (e.g. 'from:foo#1234' or 'has:image')."
)]
public MessageFilter MessageFilter { get; init; } = MessageFilter.Null;
[CommandOption("parallel", Description = "Limits how many channels can be exported in parallel.")]
[CommandOption(
"parallel",
Description = "Limits how many channels can be exported in parallel."
)]
public int ParallelLimit { get; init; } = 1;
[CommandOption("media", Description = "Download referenced media content.")]
[CommandOption(
"media",
Description = "Download referenced media content."
)]
public bool ShouldDownloadMedia { get; init; }
[CommandOption("reuse-media", Description = "Reuse already existing media content to skip redundant downloads.")]
[CommandOption(
"reuse-media",
Description = "Reuse already existing media content to skip redundant downloads."
)]
public bool ShouldReuseMedia { get; init; }
[CommandOption("dateformat", Description = "Format used when writing dates.")]
[CommandOption(
"dateformat",
Description = "Format used when writing dates."
)]
public string DateFormat { get; init; } = "dd-MMM-yy hh:mm tt";
private ChannelExporter? _channelExporter;
@@ -56,9 +99,20 @@ public abstract class ExportCommandBase : TokenCommandBase
{
var cancellationToken = console.RegisterCancellationHandler();
// https://github.com/Tyrrrz/DiscordChatExporter/issues/425
if (ShouldReuseMedia && !ShouldDownloadMedia)
{
throw new CommandException("Option --reuse-media cannot be used without --media.");
throw new CommandException(
"Option --reuse-media cannot be used without --media."
);
}
// https://github.com/Tyrrrz/DiscordChatExporter/issues/799
if (channels.Count > 1 && !PathEx.IsDirectoryPath(OutputPath) && !OutputPath.Contains('%'))
{
throw new CommandException(
"Attempted to export multiple channels, but the output path is neither a directory nor a template."
);
}
var errors = new ConcurrentDictionary<Channel, string>();
@@ -78,8 +132,7 @@ public abstract class ExportCommandBase : TokenCommandBase
{
try
{
await progressContext.StartTaskAsync(
$"{channel.Category.Name} / {channel.Name}",
await progressContext.StartTaskAsync($"{channel.Category.Name} / {channel.Name}",
async progress =>
{
var guild = await Discord.GetGuildAsync(channel.GuildId, innerCancellationToken);
@@ -98,7 +151,11 @@ public abstract class ExportCommandBase : TokenCommandBase
DateFormat
);
await Exporter.ExportChannelAsync(request, progress, innerCancellationToken);
await Exporter.ExportChannelAsync(
request,
progress.ToPercentageBased(),
innerCancellationToken
);
}
);
}
@@ -162,4 +219,16 @@ public abstract class ExportCommandBase : TokenCommandBase
await ExecuteAsync(console, channels);
}
public override ValueTask ExecuteAsync(IConsole console)
{
// War in Ukraine message
console.Output.WriteLine("========================================================================");
console.Output.WriteLine("|| Ukraine is at war! Support my country in its fight for freedom~ ||");
console.Output.WriteLine("|| Learn more on my website: https://tyrrrz.me ||");
console.Output.WriteLine("========================================================================");
console.Output.WriteLine("");
return default;
}
}
@@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Infrastructure;
@@ -8,22 +9,26 @@ namespace DiscordChatExporter.Cli.Commands.Base;
public abstract class TokenCommandBase : ICommand
{
[CommandOption("token", 't', IsRequired = true, EnvironmentVariable = "DISCORD_TOKEN", Description = "Authentication token.")]
public string TokenValue { get; init; } = "";
[CommandOption(
"token",
't',
IsRequired = true,
EnvironmentVariable = "DISCORD_TOKEN",
Description = "Authentication token."
)]
public string Token { get; init; } = "";
[CommandOption("bot", 'b', EnvironmentVariable = "DISCORD_TOKEN_BOT", Description = "Authenticate as a bot.")]
[Obsolete("This option doesn't do anything. Kept for backwards compatibility.")]
[CommandOption(
"bot",
'b',
EnvironmentVariable = "DISCORD_TOKEN_BOT",
Description = "This option doesn't do anything. Kept for backwards compatibility."
)]
public bool IsBotToken { get; init; }
private AuthToken? _authToken;
private AuthToken AuthToken => _authToken ??= new AuthToken(
IsBotToken
? AuthTokenKind.Bot
: AuthTokenKind.User,
TokenValue
);
private DiscordClient? _discordClient;
protected DiscordClient Discord => _discordClient ??= new DiscordClient(AuthToken);
protected DiscordClient Discord => _discordClient ??= new DiscordClient(Token);
public abstract ValueTask ExecuteAsync(IConsole console);
}
@@ -10,11 +10,16 @@ namespace DiscordChatExporter.Cli.Commands;
[Command("exportall", Description = "Export all accessible channels.")]
public class ExportAllCommand : ExportCommandBase
{
[CommandOption("include-dm", Description = "Include direct message channels.")]
[CommandOption(
"include-dm",
Description = "Include direct message channels."
)]
public bool IncludeDirectMessages { get; init; } = true;
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
var cancellationToken = console.RegisterCancellationHandler();
var channels = new List<Channel>();
@@ -26,13 +31,7 @@ public class ExportAllCommand : ExportCommandBase
continue;
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
{
// Skip non-text channels
if (!channel.IsTextChannel)
continue;
channels.Add(channel);
}
}
await base.ExecuteAsync(console, channels);
@@ -12,9 +12,17 @@ namespace DiscordChatExporter.Cli.Commands;
public class ExportChannelsCommand : ExportCommandBase
{
// TODO: change this to plural (breaking change)
[CommandOption("channel", 'c', IsRequired = true, Description = "Channel ID(s).")]
[CommandOption(
"channel",
'c',
IsRequired = true,
Description = "Channel ID(s)."
)]
public IReadOnlyList<Snowflake> ChannelIds { get; init; } = Array.Empty<Snowflake>();
public override async ValueTask ExecuteAsync(IConsole console) =>
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
await base.ExecuteAsync(console, ChannelIds);
}
}
@@ -13,12 +13,16 @@ public class ExportDirectMessagesCommand : ExportCommandBase
{
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
var cancellationToken = console.RegisterCancellationHandler();
await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
var textChannels = channels.Where(c => c.IsTextChannel).ToArray();
await base.ExecuteAsync(console, textChannels);
var channels = (await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken))
.Where(c => c.Kind != ChannelKind.GuildCategory)
.ToArray();
await base.ExecuteAsync(console, channels);
}
}
@@ -4,6 +4,7 @@ using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands;
@@ -11,17 +12,26 @@ namespace DiscordChatExporter.Cli.Commands;
[Command("exportguild", Description = "Export all channels within specified guild.")]
public class ExportGuildCommand : ExportCommandBase
{
[CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")]
[CommandOption(
"guild",
'g',
IsRequired = true,
Description = "Guild ID."
)]
public Snowflake GuildId { get; init; }
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
var cancellationToken = console.RegisterCancellationHandler();
await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(GuildId, cancellationToken);
var textChannels = channels.Where(c => c.IsTextChannel).ToArray();
await base.ExecuteAsync(console, textChannels);
var channels = (await Discord.GetGuildChannelsAsync(GuildId, cancellationToken))
.Where(c => c.Kind != ChannelKind.GuildCategory)
.ToArray();
await base.ExecuteAsync(console, channels);
}
}
@@ -5,6 +5,7 @@ using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands;
@@ -12,25 +13,30 @@ namespace DiscordChatExporter.Cli.Commands;
[Command("channels", Description = "Get the list of channels in a guild.")]
public class GetChannelsCommand : TokenCommandBase
{
[CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")]
[CommandOption(
"guild",
'g',
IsRequired = true,
Description = "Guild ID."
)]
public Snowflake GuildId { get; init; }
public override async ValueTask ExecuteAsync(IConsole console)
{
var cancellationToken = console.RegisterCancellationHandler();
var channels = await Discord.GetGuildChannelsAsync(GuildId, cancellationToken);
var textChannels = channels
.Where(c => c.IsTextChannel)
var channels = (await Discord.GetGuildChannelsAsync(GuildId, cancellationToken))
.Where(c => c.Kind != ChannelKind.GuildCategory)
.OrderBy(c => c.Category.Position)
.ThenBy(c => c.Name)
.ToArray();
foreach (var channel in textChannels)
foreach (var channel in channels)
{
// Channel ID
await console.Output.WriteAsync(channel.Id.ToString());
await console.Output.WriteAsync(
channel.Id.ToString().PadRight(18, ' ')
);
// Separator
using (console.WithForegroundColor(ConsoleColor.DarkGray))
@@ -16,18 +16,18 @@ public class GetDirectMessageChannelsCommand : TokenCommandBase
{
var cancellationToken = console.RegisterCancellationHandler();
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
var textChannels = channels
.Where(c => c.IsTextChannel)
.OrderBy(c => c.Category.Position)
var channels = (await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken))
.Where(c => c.Kind != ChannelKind.GuildCategory)
.OrderByDescending(c => c.LastMessageId)
.ThenBy(c => c.Name)
.ToArray();
foreach (var channel in textChannels)
foreach (var channel in channels)
{
// Channel ID
await console.Output.WriteAsync(channel.Id.ToString());
await console.Output.WriteAsync(
channel.Id.ToString().PadRight(18, ' ')
);
// Separator
using (console.WithForegroundColor(ConsoleColor.DarkGray))
@@ -4,6 +4,7 @@ using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands;
@@ -15,12 +16,17 @@ public class GetGuildsCommand : TokenCommandBase
{
var cancellationToken = console.RegisterCancellationHandler();
var guilds = await Discord.GetUserGuildsAsync(cancellationToken);
var guilds = (await Discord.GetUserGuildsAsync(cancellationToken))
// Show direct messages first
.OrderByDescending(g => g.Id == Guild.DirectMessages.Id)
.ThenBy(g => g.Name);
foreach (var guild in guilds.OrderBy(g => g.Name))
foreach (var guild in guilds)
{
// Guild ID
await console.Output.WriteAsync(guild.Id.ToString());
await console.Output.WriteAsync(
guild.Id.ToString().PadRight(18, ' ')
);
// Separator
using (console.WithForegroundColor(ConsoleColor.DarkGray))
@@ -15,15 +15,15 @@ public class GuideCommand : ICommand
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("To get user token:");
console.Output.WriteLine(" 1. Open Discord");
console.Output.WriteLine(" * Automating user accounts is technically against TOS — USE AT YOUR OWN RISK!");
console.Output.WriteLine(" 1. Open Discord in your web browser and login");
console.Output.WriteLine(" 2. Press Ctrl+Shift+I to show developer tools");
console.Output.WriteLine(" 3. Press Ctrl+Shift+M to toggle device toolbar");
console.Output.WriteLine(" 4. Navigate to the Application tab");
console.Output.WriteLine(" 5. On the left, expand Local Storage and select https://discord.com");
console.Output.WriteLine(" 6. Type \"token\" into the Filter box");
console.Output.WriteLine(" 7. If the token key does not appear, press Ctrl+R to reload");
console.Output.WriteLine(" 8. Copy the value of the token key");
console.Output.WriteLine(" * Automating user accounts is technically against TOS, use at your own risk.");
console.Output.WriteLine(" 3. Navigate to the Network tab");
console.Output.WriteLine(" 4. Press Ctrl+R to reload");
console.Output.WriteLine(" 5. Search for a request containing \"messages?limit=50\" or similar");
console.Output.WriteLine(" 6. Select the Headers tab on the right");
console.Output.WriteLine(" 7. Scroll down to the Request Headers section");
console.Output.WriteLine(" 8. Copy the value of the \"authorization\" header");
console.Output.WriteLine();
// Bot token
@@ -42,7 +42,7 @@ public class GuideCommand : ICommand
console.Output.WriteLine(" 1. Open Discord");
console.Output.WriteLine(" 2. Open Settings");
console.Output.WriteLine(" 3. Go to Appearance section");
console.Output.WriteLine(" 3. Go to Advanced section");
console.Output.WriteLine(" 4. Enable Developer Mode");
console.Output.WriteLine(" 5. Right click on the desired guild or channel and click Copy ID");
console.Output.WriteLine();
@@ -61,7 +61,7 @@ public class GuideCommand : ICommand
// Wiki link
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("For more information, check out the wiki:");
console.Output.WriteLine("If you have questions or issues, please refer to the wiki:");
using (console.WithForegroundColor(ConsoleColor.DarkCyan))
console.Output.WriteLine("https://github.com/Tyrrrz/DiscordChatExporter/wiki");
@@ -6,9 +6,10 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CliFx" Version="2.0.6" />
<PackageReference Include="Spectre.Console" Version="0.43.0" />
<PackageReference Include="Gress" Version="1.2.0" />
<PackageReference Include="CliFx" Version="2.3.0" />
<PackageReference Include="Spectre.Console" Version="0.44.0" />
<PackageReference Include="Gress" Version="2.0.1" />
<PackageReference Include="DotnetRuntimeBootstrapper" Version="2.3.1" PrivateAssets="all" />
</ItemGroup>
<ItemGroup>
@@ -1,12 +0,0 @@
using System.Net.Http.Headers;
namespace DiscordChatExporter.Core.Discord;
public record AuthToken(AuthTokenKind Kind, string Value)
{
public AuthenticationHeaderValue GetAuthenticationHeader() => Kind switch
{
AuthTokenKind.Bot => new AuthenticationHeaderValue("Bot", Value),
_ => new AuthenticationHeaderValue(Value)
};
}
@@ -13,6 +13,7 @@ public partial record Attachment(
Snowflake Id,
string Url,
string FileName,
string? Description,
int? Width,
int? Height,
FileSize FileSize) : IHasId
@@ -34,11 +35,12 @@ public partial record Attachment
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var url = json.GetProperty("url").GetNonWhiteSpaceString();
var fileName = json.GetProperty("filename").GetNonNullString();
var description = json.GetPropertyOrNull("description")?.GetNonWhiteSpaceStringOrNull();
var width = json.GetPropertyOrNull("width")?.GetInt32OrNull();
var height = json.GetPropertyOrNull("height")?.GetInt32OrNull();
var fileName = json.GetProperty("filename").GetNonWhiteSpaceString();
var fileSize = json.GetProperty("size").GetInt64().Pipe(FileSize.FromBytes);
return new Attachment(id, url, fileName, width, height, fileSize);
return new Attachment(id, url, fileName, description, width, height, fileSize);
}
}
@@ -14,16 +14,10 @@ public partial record Channel(
ChannelCategory Category,
string Name,
int? Position,
string? Topic) : IHasId
string? Topic,
Snowflake? LastMessageId) : IHasId
{
public bool IsTextChannel => Kind is
ChannelKind.GuildTextChat or
ChannelKind.DirectTextChat or
ChannelKind.DirectGroupTextChat or
ChannelKind.GuildNews or
ChannelKind.GuildStore;
public bool IsVoiceChannel => !IsTextChannel;
public bool SupportsVoice => Kind is ChannelKind.GuildVoiceChat or ChannelKind.GuildStageVoice;
}
public partial record Channel
@@ -36,7 +30,6 @@ public partial record Channel
ChannelKind.DirectTextChat => "Private",
ChannelKind.DirectGroupTextChat => "Group",
ChannelKind.GuildNews => "News",
ChannelKind.GuildStore => "Store",
_ => "Default"
},
null
@@ -45,9 +38,8 @@ public partial record Channel
public static Channel Parse(JsonElement json, ChannelCategory? category = null, int? positionHint = null)
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var guildId = json.GetPropertyOrNull("guild_id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse);
var topic = json.GetPropertyOrNull("topic")?.GetStringOrNull();
var kind = (ChannelKind)json.GetProperty("type").GetInt32();
var guildId = json.GetPropertyOrNull("guild_id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse);
var name =
// Guild channel
@@ -63,7 +55,16 @@ public partial record Channel
// Fallback
id.ToString();
var position = positionHint ?? json.GetPropertyOrNull("position")?.GetInt32OrNull();
var position =
positionHint ??
json.GetPropertyOrNull("position")?.GetInt32OrNull();
var topic = json.GetPropertyOrNull("topic")?.GetStringOrNull();
var lastMessageId = json
.GetPropertyOrNull("last_message_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
return new Channel(
id,
@@ -72,7 +73,8 @@ public partial record Channel
category ?? GetFallbackCategory(kind),
name,
position,
topic
topic,
lastMessageId
);
}
}
@@ -12,12 +12,15 @@ public record ChannelCategory(Snowflake Id, string Name, int? Position) : IHasId
public static ChannelCategory Parse(JsonElement json, int? positionHint = null)
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var position = positionHint ?? json.GetPropertyOrNull("position")?.GetInt32OrNull();
var name =
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ??
id.ToString();
var position =
positionHint ??
json.GetPropertyOrNull("position")?.GetInt32OrNull();
return new ChannelCategory(id, name, position);
}
}
@@ -5,10 +5,15 @@
public enum ChannelKind
{
GuildTextChat = 0,
DirectTextChat,
GuildVoiceChat,
DirectGroupTextChat,
GuildCategory,
GuildNews,
GuildStore
DirectTextChat = 1,
GuildVoiceChat = 2,
DirectGroupTextChat = 3,
GuildCategory = 4,
GuildNews = 5,
GuildNewsThread = 10,
GuildPublicThread = 11,
GuildPrivateThread = 12,
GuildStageVoice = 13,
GuildDirectory = 14,
GuildForum = 15
}
@@ -18,7 +18,7 @@ public partial record Embed(
string? Description,
IReadOnlyList<EmbedField> Fields,
EmbedImage? Thumbnail,
EmbedImage? Image,
IReadOnlyList<EmbedImage> Images,
EmbedFooter? Footer)
{
public PlainImageEmbedProjection? TryGetPlainImage() =>
@@ -39,17 +39,28 @@ public partial record Embed
var url = json.GetPropertyOrNull("url")?.GetNonWhiteSpaceStringOrNull();
var timestamp = json.GetPropertyOrNull("timestamp")?.GetDateTimeOffset();
var color = json.GetPropertyOrNull("color")?.GetInt32OrNull()?.Pipe(System.Drawing.Color.FromArgb).ResetAlpha();
var description = json.GetPropertyOrNull("description")?.GetStringOrNull();
var author = json.GetPropertyOrNull("author")?.Pipe(EmbedAuthor.Parse);
var thumbnail = json.GetPropertyOrNull("thumbnail")?.Pipe(EmbedImage.Parse);
var image = json.GetPropertyOrNull("image")?.Pipe(EmbedImage.Parse);
var footer = json.GetPropertyOrNull("footer")?.Pipe(EmbedFooter.Parse);
var description = json.GetPropertyOrNull("description")?.GetStringOrNull();
var fields =
json.GetPropertyOrNull("fields")?.EnumerateArrayOrNull()?.Select(EmbedField.Parse).ToArray() ??
Array.Empty<EmbedField>();
var thumbnail = json.GetPropertyOrNull("thumbnail")?.Pipe(EmbedImage.Parse);
// Under the Discord API model, embeds can only have at most one image.
// Because of that, embeds that are rendered with multiple images on the client
// (e.g. tweet embeds), are exposed from the API as multiple separate embeds.
// Our embed model is consistent with the user-facing side of Discord, so images
// are stored as an array. The API will only ever return one image, but we deal
// with this by merging related embeds at the end of the message parsing process.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/695
var images =
json.GetPropertyOrNull("image")?.Pipe(EmbedImage.Parse).Enumerate().ToArray() ??
Array.Empty<EmbedImage>();
var footer = json.GetPropertyOrNull("footer")?.Pipe(EmbedFooter.Parse);
return new Embed(
title,
url,
@@ -59,7 +70,7 @@ public partial record Embed
description,
fields,
thumbnail,
image,
images,
footer
);
}
@@ -11,8 +11,8 @@ public record EmbedField(
{
public static EmbedField Parse(JsonElement json)
{
var name = json.GetProperty("name").GetNonWhiteSpaceString();
var value = json.GetProperty("value").GetNonWhiteSpaceString();
var name = json.GetProperty("name").GetNonNullString();
var value = json.GetProperty("value").GetNonNullString();
var isInline = json.GetPropertyOrNull("inline")?.GetBooleanOrNull() ?? false;
return new EmbedField(name, value, isInline);
@@ -10,7 +10,7 @@ public partial record YouTubeVideoEmbedProjection(string VideoId)
public partial record YouTubeVideoEmbedProjection
{
// Adapted from YoutubeExplode
// https://github.com/Tyrrrz/YoutubeExplode/blob/5be164be20019783913f76fcc98f18c65aebe9f0/YoutubeExplode/Videos/VideoId.cs#L34-L64
// https://github.com/Tyrrrz/YoutubeExplode/blob/bc700b631bd105d0be208a88116347034bdca88b/YoutubeExplode/Videos/VideoId.cs#L40-L62
private static string? TryParseVideoId(string embedUrl)
{
// Regular URL
@@ -31,6 +31,12 @@ public partial record YouTubeVideoEmbedProjection
if (!string.IsNullOrWhiteSpace(embedMatch))
return embedMatch;
// Shorts URL
// https://www.youtube.com/shorts/sKL1vjP0tIo
var shortsMatch = Regex.Match(embedUrl, @"youtube\..+?/shorts/(.*?)(?:\?|&|/|$)").Groups[1].Value;
if (!string.IsNullOrWhiteSpace(shortsMatch))
return shortsMatch;
return null;
}
+25 -18
View File
@@ -10,40 +10,48 @@ namespace DiscordChatExporter.Core.Discord.Data;
// https://discord.com/developers/docs/resources/emoji#emoji-object
public partial record Emoji(
// Only present on custom emoji
string? Id,
// Name of custom emoji (e.g. LUL) or actual representation of standard emoji (e.g. 🙂)
Snowflake? Id,
// Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂)
string Name,
bool IsAnimated,
string ImageUrl)
{
// Name of custom emoji (e.g. LUL) or name of standard emoji (e.g. slight_smile)
public string Code => !string.IsNullOrWhiteSpace(Id)
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => Id is not null
? Name
: EmojiIndex.TryGetCode(Name) ?? Name;
}
public partial record Emoji
{
private static string GetTwemojiName(string name) => string.Join("-",
name
.GetRunes()
// Variant selector rune is skipped in Twemoji names
.Where(r => r.Value != 0xfe0f)
.Select(r => r.Value.ToString("x"))
);
private static string GetTwemojiId(string name)
{
var runes = name.GetRunes().ToArray();
private static string GetImageUrl(string id, bool isAnimated) => isAnimated
// Variant selector rune is skipped in Twemoji names, except when the emoji also contains a zero-width joiner.
// VS = 0xfe0f; ZWJ = 0x200d.
var filteredRunes = runes.Any(r => r.Value == 0x200d)
? runes
: runes.Where(r => r.Value != 0xfe0f);
return string.Join(
"-",
filteredRunes.Select(r => r.Value.ToString("x"))
);
}
private static string GetImageUrl(Snowflake id, bool isAnimated) => isAnimated
? $"https://cdn.discordapp.com/emojis/{id}.gif"
: $"https://cdn.discordapp.com/emojis/{id}.png";
private static string GetImageUrl(string name) =>
$"https://twemoji.maxcdn.com/2/svg/{GetTwemojiName(name)}.svg";
$"https://twemoji.maxcdn.com/v/latest/svg/{GetTwemojiId(name)}.svg";
public static string GetImageUrl(string? id, string? name, bool isAnimated)
public static string GetImageUrl(Snowflake? id, string? name, bool isAnimated)
{
// Custom emoji
if (!string.IsNullOrWhiteSpace(id))
return GetImageUrl(id, isAnimated);
if (id is not null)
return GetImageUrl(id.Value, isAnimated);
// Standard emoji
if (!string.IsNullOrWhiteSpace(name))
@@ -55,10 +63,9 @@ public partial record Emoji
public static Emoji Parse(JsonElement json)
{
var id = json.GetPropertyOrNull("id")?.GetNonWhiteSpaceStringOrNull();
var id = json.GetPropertyOrNull("id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse);
var name = json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull();
var isAnimated = json.GetPropertyOrNull("animated")?.GetBooleanOrNull() ?? false;
var imageUrl = GetImageUrl(id, name, isAnimated);
return new Emoji(
+12 -5
View File
@@ -1,4 +1,5 @@
using System.Text.Json;
using System;
using System.Text.Json;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
@@ -17,15 +18,21 @@ public record Guild(Snowflake Id, string Name, string IconUrl) : IHasId
private static string GetDefaultIconUrl() =>
"https://cdn.discordapp.com/embed/avatars/0.png";
private static string GetIconUrl(Snowflake id, string iconHash) =>
$"https://cdn.discordapp.com/icons/{id}/{iconHash}.png";
private static string GetIconUrl(Snowflake id, string iconHash)
{
var extension = iconHash.StartsWith("a_", StringComparison.Ordinal)
? "gif"
: "png";
return $"https://cdn.discordapp.com/icons/{id}/{iconHash}.{extension}";
}
public static Guild Parse(JsonElement json)
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var name = json.GetProperty("name").GetNonWhiteSpaceString();
var iconHash = json.GetPropertyOrNull("icon")?.GetNonWhiteSpaceStringOrNull();
var name = json.GetProperty("name").GetNonNullString();
var iconHash = json.GetPropertyOrNull("icon")?.GetNonWhiteSpaceStringOrNull();
var iconUrl = !string.IsNullOrWhiteSpace(iconHash)
? GetIconUrl(id, iconHash)
: GetDefaultIconUrl();
@@ -21,23 +21,82 @@ public record Message(
string Content,
IReadOnlyList<Attachment> Attachments,
IReadOnlyList<Embed> Embeds,
IReadOnlyList<Sticker> Stickers,
IReadOnlyList<Reaction> Reactions,
IReadOnlyList<User> MentionedUsers,
MessageReference? Reference,
Message? ReferencedMessage) : IHasId
{
private static IReadOnlyList<Embed> NormalizeEmbeds(IReadOnlyList<Embed> embeds)
{
if (embeds.Count <= 1)
return embeds;
// Discord API doesn't support embeds with multiple images, even though the Discord client does.
// To work around this, it seems that the API returns multiple consecutive embeds with different images,
// which are then merged together on the client. We need to replicate the same behavior ourselves.
// Currently, only known case where this workaround is required is Twitter embeds.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/695
var normalizedEmbeds = new List<Embed>();
for (var i = 0; i < embeds.Count; i++)
{
var embed = embeds[i];
if (embed.Url?.Contains("://twitter.com/") == true)
{
// Find embeds with the same URL that only contain a single image and nothing else
var trailingEmbeds = embeds
.Skip(i + 1)
.TakeWhile(e =>
e.Url == embed.Url &&
e.Timestamp is null &&
e.Author is null &&
e.Color is null &&
string.IsNullOrWhiteSpace(e.Description) &&
!e.Fields.Any() &&
e.Images.Count == 1 &&
e.Footer is null
)
.ToArray();
if (trailingEmbeds.Any())
{
// Concatenate all images into one embed
var images = embed.Images.Concat(trailingEmbeds.SelectMany(e => e.Images)).ToArray();
normalizedEmbeds.Add(embed with { Images = images });
i += trailingEmbeds.Length;
}
else
{
normalizedEmbeds.Add(embed);
}
}
else
{
normalizedEmbeds.Add(embed);
}
}
return normalizedEmbeds;
}
public static Message Parse(JsonElement json)
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var kind = (MessageKind)json.GetProperty("type").GetInt32();
var author = json.GetProperty("author").Pipe(User.Parse);
var timestamp = json.GetProperty("timestamp").GetDateTimeOffset();
var editedTimestamp = json.GetPropertyOrNull("edited_timestamp")?.GetDateTimeOffset();
var callEndedTimestamp = json.GetPropertyOrNull("call")?.GetPropertyOrNull("ended_timestamp")
?.GetDateTimeOffset();
var kind = (MessageKind)json.GetProperty("type").GetInt32();
var callEndedTimestamp = json
.GetPropertyOrNull("call")?
.GetPropertyOrNull("ended_timestamp")?
.GetDateTimeOffset();
var isPinned = json.GetPropertyOrNull("pinned")?.GetBooleanOrNull() ?? false;
var messageReference = json.GetPropertyOrNull("message_reference")?.Pipe(MessageReference.Parse);
var referencedMessage = json.GetPropertyOrNull("referenced_message")?.Pipe(Parse);
var content = kind switch
{
@@ -56,9 +115,14 @@ public record Message(
json.GetPropertyOrNull("attachments")?.EnumerateArrayOrNull()?.Select(Attachment.Parse).ToArray() ??
Array.Empty<Attachment>();
var embeds =
var embeds = NormalizeEmbeds(
json.GetPropertyOrNull("embeds")?.EnumerateArrayOrNull()?.Select(Embed.Parse).ToArray() ??
Array.Empty<Embed>();
Array.Empty<Embed>()
);
var stickers =
json.GetPropertyOrNull("sticker_items")?.EnumerateArrayOrNull()?.Select(Sticker.Parse).ToArray() ??
Array.Empty<Sticker>();
var reactions =
json.GetPropertyOrNull("reactions")?.EnumerateArrayOrNull()?.Select(Reaction.Parse).ToArray() ??
@@ -68,6 +132,9 @@ public record Message(
json.GetPropertyOrNull("mentions")?.EnumerateArrayOrNull()?.Select(User.Parse).ToArray() ??
Array.Empty<User>();
var messageReference = json.GetPropertyOrNull("message_reference")?.Pipe(MessageReference.Parse);
var referencedMessage = json.GetPropertyOrNull("referenced_message")?.Pipe(Parse);
return new Message(
id,
kind,
@@ -79,6 +146,7 @@ public record Message(
content,
attachments,
embeds,
stickers,
reactions,
mentionedUsers,
messageReference,
@@ -9,9 +9,20 @@ public record MessageReference(Snowflake? MessageId, Snowflake? ChannelId, Snowf
{
public static MessageReference Parse(JsonElement json)
{
var messageId = json.GetPropertyOrNull("message_id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse);
var channelId = json.GetPropertyOrNull("channel_id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse);
var guildId = json.GetPropertyOrNull("guild_id")?.GetNonWhiteSpaceStringOrNull()?.Pipe(Snowflake.Parse);
var messageId = json
.GetPropertyOrNull("message_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var channelId = json
.GetPropertyOrNull("channel_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
var guildId = json
.GetPropertyOrNull("guild_id")?
.GetNonWhiteSpaceStringOrNull()?
.Pipe(Snowflake.Parse);
return new MessageReference(messageId, channelId, guildId);
}
@@ -12,7 +12,7 @@ public record Role(Snowflake Id, string Name, int Position, Color? Color) : IHas
public static Role Parse(JsonElement json)
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var name = json.GetProperty("name").GetNonWhiteSpaceString();
var name = json.GetProperty("name").GetNonNullString();
var position = json.GetProperty("position").GetInt32();
var color = json
@@ -0,0 +1,25 @@
using System.Text.Json;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data;
// https://discord.com/developers/docs/resources/sticker#sticker-resource
public record Sticker(Snowflake Id, string Name, StickerFormat Format, string SourceUrl)
{
private static string GetSourceUrl(Snowflake id, StickerFormat format)
{
var extension = format == StickerFormat.Lottie ? "json" : "png";
return $"https://discord.com/stickers/{id}.{extension}";
}
public static Sticker Parse(JsonElement json)
{
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var name = json.GetProperty("name").GetNonNullString();
var format = (StickerFormat)json.GetProperty("format_type").GetInt32();
var sourceUrl = GetSourceUrl(id, format);
return new Sticker(id, name, format, sourceUrl);
}
}
@@ -0,0 +1,8 @@
namespace DiscordChatExporter.Core.Discord.Data;
public enum StickerFormat
{
Png = 1,
PngAnimated = 2,
Lottie = 3
}
@@ -30,7 +30,7 @@ public partial record User
? "gif"
: "png";
return $"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.{extension}?size=128";
return $"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.{extension}?size=512";
}
public static User Parse(JsonElement json)
@@ -38,9 +38,9 @@ public partial record User
var id = json.GetProperty("id").GetNonWhiteSpaceString().Pipe(Snowflake.Parse);
var isBot = json.GetPropertyOrNull("bot")?.GetBooleanOrNull() ?? false;
var discriminator = json.GetProperty("discriminator").GetNonWhiteSpaceString().Pipe(int.Parse);
var name = json.GetProperty("username").GetNonWhiteSpaceString();
var avatarHash = json.GetPropertyOrNull("avatar")?.GetNonWhiteSpaceStringOrNull();
var name = json.GetProperty("username").GetNonNullString();
var avatarHash = json.GetPropertyOrNull("avatar")?.GetNonWhiteSpaceStringOrNull();
var avatarUrl = !string.IsNullOrWhiteSpace(avatarHash)
? GetAvatarUrl(id, avatarHash)
: GetDefaultAvatarUrl(discriminator);
@@ -11,6 +11,7 @@ using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions;
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions;
using Gress;
using JsonExtensions.Http;
using JsonExtensions.Reading;
@@ -18,25 +19,67 @@ namespace DiscordChatExporter.Core.Discord;
public class DiscordClient
{
private readonly AuthToken _token;
private readonly Uri _baseUri = new("https://discord.com/api/v8/", UriKind.Absolute);
private readonly string _token;
private readonly Uri _baseUri = new("https://discord.com/api/v9/", UriKind.Absolute);
public DiscordClient(AuthToken token) => _token = token;
private TokenKind _tokenKind = TokenKind.Unknown;
public DiscordClient(string token) => _token = token;
private async ValueTask<HttpResponseMessage> GetResponseAsync(
string url,
bool isBot,
CancellationToken cancellationToken = default)
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
// Don't validate because token can have invalid characters
// https://github.com/Tyrrrz/DiscordChatExporter/issues/828
request.Headers.TryAddWithoutValidation(
"Authorization",
isBot ? $"Bot {_token}" : _token
);
return await Http.Client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
cancellationToken
);
}
private async ValueTask<HttpResponseMessage> GetResponseAsync(
string url,
CancellationToken cancellationToken = default)
{
return await Http.ResponsePolicy.ExecuteAsync(async innerCancellationToken =>
return await Http.ResponseResiliencePolicy.ExecuteAsync(async innerCancellationToken =>
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
request.Headers.Authorization = _token.GetAuthenticationHeader();
if (_tokenKind == TokenKind.User)
return await GetResponseAsync(url, false, innerCancellationToken);
return await Http.Client.SendAsync(
request,
HttpCompletionOption.ResponseHeadersRead,
innerCancellationToken
);
if (_tokenKind == TokenKind.Bot)
return await GetResponseAsync(url, true, innerCancellationToken);
// Try to authenticate as user
var userResponse = await GetResponseAsync(url, false, innerCancellationToken);
if (userResponse.StatusCode != HttpStatusCode.Unauthorized)
{
_tokenKind = TokenKind.User;
return userResponse;
}
userResponse.Dispose();
// Otherwise, try to authenticate as bot
var botResponse = await GetResponseAsync(url, true, innerCancellationToken);
if (botResponse.StatusCode != HttpStatusCode.Unauthorized)
{
_tokenKind = TokenKind.Bot;
return botResponse;
}
// The token is probably invalid altogether.
// Return the last response anyway, upstream should handle the error.
return botResponse;
}, cancellationToken);
}
@@ -138,6 +181,8 @@ public class DiscordClient
.Select((j, index) => ChannelCategory.Parse(j, index + 1))
.ToDictionary(j => j.Id.ToString(), StringComparer.Ordinal);
// Discord positions are not deterministic, so we need to normalize them
// because the user may refer to the channel position via file name template.
var position = 0;
foreach (var channelJson in responseOrdered)
@@ -233,7 +278,7 @@ public class DiscordClient
Snowflake channelId,
Snowflake? after = null,
Snowflake? before = null,
IProgress<double>? progress = null,
IProgress<Percentage>? progress = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
// Get the last message in the specified range.
@@ -282,16 +327,13 @@ public class DiscordClient
var exportedDuration = (message.Timestamp - firstMessage.Timestamp).Duration();
var totalDuration = (lastMessage.Timestamp - firstMessage.Timestamp).Duration();
if (totalDuration > TimeSpan.Zero)
{
progress.Report(exportedDuration / totalDuration);
}
// Avoid division by zero if all messages have the exact same timestamp
// (which may be the case if there's only one message in the channel)
else
{
progress.Report(1);
}
progress.Report(Percentage.FromFraction(
// Avoid division by zero if all messages have the exact same timestamp
// (which may be the case if there's only one message in the channel)
totalDuration > TimeSpan.Zero
? exportedDuration / totalDuration
: 1
));
}
yield return message;
@@ -49,7 +49,15 @@ public partial record struct Snowflake
public static Snowflake Parse(string str) => Parse(str, null);
}
public partial record struct Snowflake : IComparable<Snowflake>
public partial record struct Snowflake : IComparable<Snowflake>, IComparable
{
public int CompareTo(Snowflake other) => Value.CompareTo(other.Value);
public int CompareTo(object? obj)
{
if (obj is not Snowflake other)
throw new ArgumentException($"Object must be of type {nameof(Snowflake)}.");
return Value.CompareTo(other.Value);
}
}
@@ -1,7 +1,8 @@
namespace DiscordChatExporter.Core.Discord;
public enum AuthTokenKind
public enum TokenKind
{
Unknown,
User,
Bot
}
@@ -5,10 +5,12 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Gress" Version="2.0.1" />
<PackageReference Include="JsonExtensions" Version="1.2.0" />
<PackageReference Include="MiniRazor.CodeGen" Version="2.2.0" />
<PackageReference Include="Polly" Version="7.2.2" />
<PackageReference Include="MiniRazor.CodeGen" Version="2.2.1" />
<PackageReference Include="Polly" Version="7.2.3" />
<PackageReference Include="Superpower" Version="3.0.0" />
<PackageReference Include="WebMarkupMin.Core" Version="2.12.0" />
</ItemGroup>
<ItemGroup>
@@ -8,6 +8,7 @@ using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Exceptions;
using DiscordChatExporter.Core.Utils.Extensions;
using Gress;
namespace DiscordChatExporter.Core.Exporting;
@@ -17,11 +18,9 @@ public class ChannelExporter
public ChannelExporter(DiscordClient discord) => _discord = discord;
public ChannelExporter(AuthToken token) : this(new DiscordClient(token)) {}
public async ValueTask ExportChannelAsync(
ExportRequest request,
IProgress<double>? progress = null,
IProgress<Percentage>? progress = null,
CancellationToken cancellationToken = default)
{
// Build context
@@ -60,8 +60,9 @@ public partial record ExportRequest
"%C" => channel.Name,
"%p" => channel.Position?.ToString() ?? "0",
"%P" => channel.Category.Position?.ToString() ?? "0",
"%a" => (after ?? Snowflake.Zero).ToDate().ToString("yyyy-MM-dd"),
"%b" => (before?.ToDate() ?? DateTime.Now).ToString("yyyy-MM-dd"),
"%a" => after?.ToDate().ToString("yyyy-MM-dd") ?? "",
"%b" => before?.ToDate().ToString("yyyy-MM-dd") ?? "",
"%d" => DateTimeOffset.Now.ToString("yyyy-MM-dd"),
"%%" => "%",
_ => m.Value
})
@@ -93,7 +94,7 @@ public partial record ExportRequest
// Date range
if (after is not null || before is not null)
{
buffer.Append(" (");
buffer.Append(' ').Append('(');
// Both 'after' and 'before' are set
if (after is not null && before is not null)
@@ -111,11 +112,11 @@ public partial record ExportRequest
buffer.Append($"before {before.Value.ToDate():yyyy-MM-dd}");
}
buffer.Append(")");
buffer.Append(')');
}
// File extension
buffer.Append($".{format.GetFileExtension()}");
buffer.Append('.').Append(format.GetFileExtension());
return PathEx.EscapeFileName(buffer.ToString());
}
@@ -10,11 +10,19 @@ internal class ContainsMessageFilter : MessageFilter
public ContainsMessageFilter(string text) => _text = text;
// Match content within word boundaries, between spaces, or as the whole input.
// For example, "max" shouldn't match on content "our maximum effort",
// but should match on content "our max effort".
// Also, "(max)" should match on content "our (max) effort", even though
// parentheses are not considered word characters.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/909
private bool IsMatch(string? content) =>
!string.IsNullOrWhiteSpace(content) &&
Regex.IsMatch(
content,
"\\b" + Regex.Escape(_text) + "\\b",
@"(?:\b|\s|^)" +
Regex.Escape(_text) +
@"(?:\b|\s|$)",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant
);
@@ -19,6 +19,7 @@ internal class HasMessageFilter : MessageFilter
MessageContentMatchKind.Video => message.Attachments.Any(file => file.IsVideo),
MessageContentMatchKind.Image => message.Attachments.Any(file => file.IsImage),
MessageContentMatchKind.Sound => message.Attachments.Any(file => file.IsAudio),
MessageContentMatchKind.Pin => message.IsPinned,
_ => throw new InvalidOperationException($"Unknown message content match kind '{_kind}'.")
};
}
@@ -7,5 +7,6 @@ internal enum MessageContentMatchKind
File,
Video,
Image,
Sound
Sound,
Pin
}
@@ -15,87 +15,107 @@ internal static class FilterGrammar
from close in Character.EqualTo(open)
select value;
private static readonly TextParser<char> FreeCharacter =
Character.Matching(c =>
!char.IsWhiteSpace(c) &&
// Avoid all special tokens used by the grammar
c is not ('(' or ')' or '"' or '\'' or '-' or '|' or '&'),
"any character except whitespace or `(`, `)`, `\"`, `'`, `-`, `|`, `&`"
);
private static readonly TextParser<string> UnquotedString =
Parse.OneOf(EscapedCharacter, FreeCharacter).AtLeastOnce().Text();
Parse.OneOf(
EscapedCharacter,
// Avoid whitespace as it's treated as an implicit 'and' operator.
// Also avoid all special tokens used by other parsers.
Character.ExceptIn(' ', '(', ')', '"', '\'', '-', '~', '|', '&')
).AtLeastOnce().Text();
private static readonly TextParser<string> String =
Parse.OneOf(QuotedString, UnquotedString).Named("text string");
private static readonly TextParser<MessageFilter> ContainsFilter =
String.Select(v => (MessageFilter) new ContainsMessageFilter(v));
String.Select(v => (MessageFilter)new ContainsMessageFilter(v));
private static readonly TextParser<MessageFilter> FromFilter = Span
.EqualToIgnoreCase("from:")
.IgnoreThen(String)
.Select(v => (MessageFilter) new FromMessageFilter(v))
.Named("from:<value>");
private static readonly TextParser<MessageFilter> FromFilter =
Span
.EqualToIgnoreCase("from:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new FromMessageFilter(v))
.Named("from:<value>");
private static readonly TextParser<MessageFilter> MentionsFilter = Span
.EqualToIgnoreCase("mentions:")
.IgnoreThen(String)
.Select(v => (MessageFilter) new MentionsMessageFilter(v))
.Named("mentions:<value>");
private static readonly TextParser<MessageFilter> MentionsFilter =
Span
.EqualToIgnoreCase("mentions:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new MentionsMessageFilter(v))
.Named("mentions:<value>");
private static readonly TextParser<MessageFilter> HasFilter = Span
.EqualToIgnoreCase("has:")
.IgnoreThen(Parse.OneOf(
Span.EqualToIgnoreCase("link").IgnoreThen(Parse.Return(MessageContentMatchKind.Link)),
Span.EqualToIgnoreCase("embed").IgnoreThen(Parse.Return(MessageContentMatchKind.Embed)),
Span.EqualToIgnoreCase("file").IgnoreThen(Parse.Return(MessageContentMatchKind.File)),
Span.EqualToIgnoreCase("video").IgnoreThen(Parse.Return(MessageContentMatchKind.Video)),
Span.EqualToIgnoreCase("image").IgnoreThen(Parse.Return(MessageContentMatchKind.Image)),
Span.EqualToIgnoreCase("sound").IgnoreThen(Parse.Return(MessageContentMatchKind.Sound))
))
.Select(k => (MessageFilter) new HasMessageFilter(k))
.Named("has:<value>");
private static readonly TextParser<MessageFilter> ReactionFilter =
Span
.EqualToIgnoreCase("reaction:")
.Try()
.IgnoreThen(String)
.Select(v => (MessageFilter)new ReactionMessageFilter(v))
.Named("reaction:<value>");
private static readonly TextParser<MessageFilter> NegatedFilter = Character
.EqualTo('-')
.IgnoreThen(Parse.Ref(() => StandaloneFilter!))
.Select(f => (MessageFilter) new NegatedMessageFilter(f));
private static readonly TextParser<MessageFilter> HasFilter =
Span
.EqualToIgnoreCase("has:")
.Try()
.IgnoreThen(Parse.OneOf(
Span.EqualToIgnoreCase("link").IgnoreThen(Parse.Return(MessageContentMatchKind.Link)),
Span.EqualToIgnoreCase("embed").IgnoreThen(Parse.Return(MessageContentMatchKind.Embed)),
Span.EqualToIgnoreCase("file").IgnoreThen(Parse.Return(MessageContentMatchKind.File)),
Span.EqualToIgnoreCase("video").IgnoreThen(Parse.Return(MessageContentMatchKind.Video)),
Span.EqualToIgnoreCase("image").IgnoreThen(Parse.Return(MessageContentMatchKind.Image)),
Span.EqualToIgnoreCase("sound").IgnoreThen(Parse.Return(MessageContentMatchKind.Sound)),
Span.EqualToIgnoreCase("pin").IgnoreThen(Parse.Return(MessageContentMatchKind.Pin))
))
.Select(k => (MessageFilter)new HasMessageFilter(k))
.Named("has:<value>");
// Make sure that property-based filters like 'has:link' don't prevent text like 'hello' from being parsed.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/909#issuecomment-1227575455
private static readonly TextParser<MessageFilter> PrimitiveFilter =
Parse.OneOf(
FromFilter,
MentionsFilter,
ReactionFilter,
HasFilter,
ContainsFilter
);
private static readonly TextParser<MessageFilter> GroupedFilter =
from open in Character.EqualTo('(')
from content in Parse.Ref(() => BinaryExpressionFilter!).Token()
from content in Parse.Ref(() => ChainedFilter!).Token()
from close in Character.EqualTo(')')
select content;
private static readonly TextParser<MessageFilter> StandaloneFilter = Parse.OneOf(
GroupedFilter,
FromFilter,
MentionsFilter,
HasFilter,
ContainsFilter
);
private static readonly TextParser<MessageFilter> NegatedFilter =
Character
// Dash is annoying to use from CLI due to conflicts with options, so we provide tilde as an alias
.In('-', '~')
.IgnoreThen(Parse.OneOf(GroupedFilter, PrimitiveFilter))
.Select(f => (MessageFilter)new NegatedMessageFilter(f));
private static readonly TextParser<MessageFilter> UnaryExpressionFilter = Parse.OneOf(
NegatedFilter,
StandaloneFilter
);
private static readonly TextParser<MessageFilter> BinaryExpressionFilter = Parse.Chain(
Parse.OneOf(
// Explicit operator
Character.In('|', '&').Token().Try(),
// Implicit operator (resolves to 'and')
Character.WhiteSpace.AtLeastOnce().IgnoreThen(Parse.Return(' '))
),
UnaryExpressionFilter,
(op, left, right) => op switch
{
'|' => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.Or),
_ => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.And)
}
);
private static readonly TextParser<MessageFilter> ChainedFilter =
Parse.Chain(
// Operator
Parse.OneOf(
// Explicit operator
Character.In('|', '&').Token().Try(),
// Implicit operator (resolves to 'and')
Character.EqualTo(' ').AtLeastOnce().IgnoreThen(Parse.Return(' '))
),
// Operand
Parse.OneOf(
NegatedFilter,
GroupedFilter,
PrimitiveFilter
),
// Reducer
(op, left, right) => op switch
{
'|' => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.Or),
_ => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.And)
}
);
public static readonly TextParser<MessageFilter> Filter =
BinaryExpressionFilter.Token().AtEnd();
ChainedFilter.Token().AtEnd();
}
@@ -0,0 +1,18 @@
using System;
using System.Linq;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Core.Exporting.Filtering;
internal class ReactionMessageFilter : MessageFilter
{
private readonly string _value;
public ReactionMessageFilter(string value) => _value = value;
public override bool IsMatch(Message message) => message.Reactions.Any(r =>
string.Equals(_value, r.Emoji.Id?.ToString(), StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, r.Emoji.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, r.Emoji.Code, StringComparison.OrdinalIgnoreCase)
);
}
@@ -35,45 +35,42 @@ internal partial class MediaDownloader
var filePath = Path.Combine(_workingDirPath, fileName);
// Reuse existing files if we're allowed to
if (_reuseMedia && File.Exists(filePath))
return _pathCache[url] = filePath;
Directory.CreateDirectory(_workingDirPath);
// This retries on IOExceptions which is dangerous as we're also working with files
await Http.ExceptionPolicy.ExecuteAsync(async () =>
if (!_reuseMedia || !File.Exists(filePath))
{
// Download the file
using var response = await Http.Client.GetAsync(url, cancellationToken);
await using (var output = File.Create(filePath))
{
await response.Content.CopyToAsync(output, cancellationToken);
}
Directory.CreateDirectory(_workingDirPath);
// Try to set the file date according to the last-modified header
try
await Http.ResiliencePolicy.ExecuteAsync(async () =>
{
var lastModified = response.Content.Headers.TryGetValue("Last-Modified")?.Pipe(s =>
DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)
? date
: (DateTimeOffset?) null
);
// Download the file
using var response = await Http.Client.GetAsync(url, cancellationToken);
await using (var output = File.Create(filePath))
await response.Content.CopyToAsync(output, cancellationToken);
if (lastModified is not null)
// Try to set the file date according to the last-modified header
try
{
File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime);
var lastModified = response.Content.Headers.TryGetValue("Last-Modified")?.Pipe(s =>
DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)
? date
: (DateTimeOffset?) null
);
if (lastModified is not null)
{
File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime);
}
}
}
catch
{
// This can apparently fail for some reason.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/585
// Updating file dates is not a critical task, so we'll just
// ignore exceptions thrown here.
}
});
catch
{
// This can apparently fail for some reason.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/585
// Updating file dates is not a critical task, so we'll just
// ignore exceptions thrown here.
}
});
}
return _pathCache[url] = filePath;
}
@@ -104,6 +101,14 @@ internal partial class MediaDownloader
var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
var fileExtension = Path.GetExtension(fileName);
// Probably not a file extension, just a dot in a long file name
// https://github.com/Tyrrrz/DiscordChatExporter/issues/708
if (fileExtension.Length > 41)
{
fileNameWithoutExtension = fileName;
fileExtension = "";
}
return PathEx.EscapeFileName(fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension);
}
}
@@ -1,60 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Core.Exporting.Writers.Html;
// Used for grouping contiguous messages in HTML export
internal partial class MessageGroup
{
public User Author { get; }
public DateTimeOffset Timestamp { get; }
public IReadOnlyList<Message> Messages { get; }
public MessageReference? Reference { get; }
public Message? ReferencedMessage {get; }
public MessageGroup(
User author,
DateTimeOffset timestamp,
MessageReference? reference,
Message? referencedMessage,
IReadOnlyList<Message> messages)
{
Author = author;
Timestamp = timestamp;
Reference = reference;
ReferencedMessage = referencedMessage;
Messages = messages;
}
}
internal partial class MessageGroup
{
public static bool CanJoin(Message message1, Message message2) =>
// Must be from the same author
message1.Author.Id == message2.Author.Id &&
// Author's name must not have changed between messages
string.Equals(message1.Author.FullName, message2.Author.FullName, StringComparison.Ordinal) &&
// Duration between messages must be 7 minutes or less
(message2.Timestamp - message1.Timestamp).Duration().TotalMinutes <= 7 &&
// Other message must not be a reply
message2.Reference is null;
public static MessageGroup Join(IReadOnlyList<Message> messages)
{
var first = messages.First();
return new MessageGroup(
first.Author,
first.Timestamp,
first.Reference,
first.ReferencedMessage,
messages
);
}
}
@@ -1,134 +1,166 @@
@using System
@using System.Linq
@using System.Threading.Tasks
@using DiscordChatExporter.Core.Discord.Data
@using DiscordChatExporter.Core.Discord.Data.Embeds
@using DiscordChatExporter.Core.Exporting.Writers.Html;
@using DiscordChatExporter.Core.Utils.Extensions
@namespace DiscordChatExporter.Core.Exporting.Writers.Html
@inherits MiniRazor.TemplateBase<MessageGroupTemplateContext>
@{
var firstMessage = Model.Messages.First();
ValueTask<string> ResolveUrlAsync(string url) => Model.ExportContext.ResolveMediaUrlAsync(url);
string FormatDate(DateTimeOffset date) => Model.ExportContext.FormatDate(date);
string FormatMarkdown(string markdown) => Model.FormatMarkdown(markdown);
string FormatEmbedMarkdown(string markdown) => Model.FormatMarkdown(markdown, false);
ValueTask<string> ResolveUrlAsync(string url) => Model.ExportContext.ResolveMediaUrlAsync(url);
var userMember = Model.ExportContext.TryGetMember(firstMessage.Author.Id);
var userMember = Model.ExportContext.TryGetMember(Model.MessageGroup.Author.Id);
var userColor = Model.ExportContext.TryGetUserColor(firstMessage.Author.Id);
var userColor = Model.ExportContext.TryGetUserColor(Model.MessageGroup.Author.Id);
var userNick = firstMessage.Author.IsBot
? firstMessage.Author.Name
: userMember?.Nick ?? firstMessage.Author.Name;
var userColorStyle = userColor is not null
? $"color: rgb({userColor?.R},{userColor?.G},{userColor?.B})"
var referencedUserMember = firstMessage.ReferencedMessage is not null
? Model.ExportContext.TryGetMember(firstMessage.ReferencedMessage.Author.Id)
: null;
var userNick = Model.MessageGroup.Author.IsBot
? Model.MessageGroup.Author.Name
: userMember?.Nick ?? Model.MessageGroup.Author.Name;
var referencedUserMember = Model.MessageGroup.ReferencedMessage is not null
? Model.ExportContext.TryGetMember(Model.MessageGroup.ReferencedMessage.Author.Id)
var referencedUserColor = firstMessage.ReferencedMessage is not null
? Model.ExportContext.TryGetUserColor(firstMessage.ReferencedMessage.Author.Id)
: null;
var referencedUserColor = Model.MessageGroup.ReferencedMessage is not null
? Model.ExportContext.TryGetUserColor(Model.MessageGroup.ReferencedMessage.Author.Id)
: null;
var referencedUserColorStyle = referencedUserColor is not null
? $"color: rgb({referencedUserColor?.R},{referencedUserColor?.G},{referencedUserColor?.B})"
: null;
var referencedUserNick = Model.MessageGroup.ReferencedMessage is not null
? Model.MessageGroup.ReferencedMessage.Author.IsBot
? Model.MessageGroup.ReferencedMessage.Author.Name
: referencedUserMember?.Nick ?? Model.MessageGroup.ReferencedMessage.Author.Name
var referencedUserNick = firstMessage.ReferencedMessage is not null
? firstMessage.ReferencedMessage.Author.IsBot
? firstMessage.ReferencedMessage.Author.Name
: referencedUserMember?.Nick ?? firstMessage.ReferencedMessage.Author.Name
: null;
}
<div class="chatlog__message-group">
@{/* Referenced message */}
@if (Model.MessageGroup.Reference is not null)
{
<div class="chatlog__reference-symbol"></div>
<div class="chatlog__reference">
@if (Model.MessageGroup.ReferencedMessage is not null)
{
<img class="chatlog__reference-avatar" src="@await ResolveUrlAsync(Model.MessageGroup.ReferencedMessage.Author.AvatarUrl)" alt="Avatar" loading="lazy">
<span class="chatlog__reference-name" title="@Model.MessageGroup.ReferencedMessage.Author.FullName" style="@referencedUserColorStyle">@referencedUserNick</span>
<div class="chatlog__reference-content">
<span class="chatlog__reference-link" onclick="scrollToMessage(event, '@Model.MessageGroup.ReferencedMessage.Id')">
@if (!string.IsNullOrWhiteSpace(Model.MessageGroup.ReferencedMessage.Content))
{
@Raw(FormatEmbedMarkdown(Model.MessageGroup.ReferencedMessage.Content))
}
else if (Model.MessageGroup.ReferencedMessage.Attachments.Any() || Model.MessageGroup.ReferencedMessage.Embeds.Any())
{
<em>Click to see attachment</em> <span>🖼️</span>
}
else
{
<em>Click to see original message</em>
}
</span>
@foreach (var (message, i) in Model.Messages.WithIndex())
{
var isFirst = i == 0;
@if (Model.MessageGroup.ReferencedMessage.EditedTimestamp is not null)
{
<span class="chatlog__reference-edited-timestamp" title="@FormatDate(Model.MessageGroup.ReferencedMessage.EditedTimestamp.Value)">(edited)</span>
}
</div>
}
else
{
<span class="chatlog__reference-unknown">
Original message was deleted or could not be loaded.
</span>
}
</div>
}
// Hide message content if it only contains a link to an image which is embedded, and nothing else
var isContentHidden =
message.Embeds.Count == 1 &&
message.Content.Trim() == PlainImageEmbedProjection.TryResolve(message.Embeds.Single())?.Url;
@{/* Avatar */}
<div class="chatlog__author-avatar-container">
<img class="chatlog__author-avatar" src="@await ResolveUrlAsync(Model.MessageGroup.Author.AvatarUrl)" alt="Avatar" loading="lazy">
</div>
var isReferencedContentHidden =
message.ReferencedMessage is not null &&
message.ReferencedMessage.Embeds.Count == 1 &&
message.ReferencedMessage.Content.Trim() == PlainImageEmbedProjection.TryResolve(message.ReferencedMessage.Embeds.Single())?.Url;
<div class="chatlog__messages">
@{/* Author name */}
<span class="chatlog__author-name" title="@Model.MessageGroup.Author.FullName" data-user-id="@Model.MessageGroup.Author.Id" style="@userColorStyle">@userNick</span>
@{/* Bot tag */}
@if (Model.MessageGroup.Author.IsBot)
{
<span class="chatlog__bot-tag">BOT</span>
}
@{/* Message timestamp */}
<span class="chatlog__timestamp">@FormatDate(Model.MessageGroup.Timestamp)</span>
@{/* Messages in a group */}
@foreach (var message in Model.MessageGroup.Messages)
{
<div class="chatlog__message @(message.IsPinned ? "chatlog__message--pinned" : null)" data-message-id="@message.Id" id="message-@message.Id" title="Message sent: @FormatDate(message.Timestamp)">
@if (!string.IsNullOrWhiteSpace(message.Content) || message.EditedTimestamp is not null)
<div id="chatlog__message-container-@message.Id" class="chatlog__message-container @(message.IsPinned ? "chatlog__message-container--pinned" : null)" data-message-id="@message.Id">
<div class="chatlog__message">
@{/* Left side */}
<div class="chatlog__message-aside">
@if (isFirst)
{
<div class="chatlog__content">
<div class="markdown">
@{/* Message content */}
<span class="preserve-whitespace">@Raw(FormatMarkdown(message.Content))</span>
// Reference symbol
if (message.Reference is not null)
{
<div class="chatlog__reference-symbol"></div>
}
@{/* Edit timestamp */}
@if (message.EditedTimestamp is not null)
// Avatar
<img class="chatlog__avatar" src="@await ResolveUrlAsync(message.Author.AvatarUrl)" alt="Avatar" loading="lazy">
}
else
{
<div class="chatlog__short-timestamp" title="@FormatDate(message.Timestamp)">@message.Timestamp.ToLocalString("t")</div>
}
</div>
@{/* Right side */}
<div class="chatlog__message-primary">
@if (isFirst)
{
// Reference
if (message.Reference is not null)
{
<div class="chatlog__reference">
@if (message.ReferencedMessage is not null)
{
<span class="chatlog__edited-timestamp" title="@FormatDate(message.EditedTimestamp.Value)">(edited)</span>
<img class="chatlog__reference-avatar" src="@await ResolveUrlAsync(message.ReferencedMessage.Author.AvatarUrl)" alt="Avatar" loading="lazy">
<div class="chatlog__reference-author" style="@(referencedUserColor is not null ? $"color: rgb({referencedUserColor.Value.R}, {referencedUserColor.Value.G}, {referencedUserColor.Value.B})" : null)" title="@message.ReferencedMessage.Author.FullName">@referencedUserNick</div>
<div class="chatlog__reference-content">
<span class="chatlog__reference-link" onclick="scrollToMessage(event, '@message.ReferencedMessage.Id')">
@if (!string.IsNullOrWhiteSpace(message.ReferencedMessage.Content) && !isReferencedContentHidden)
{
<!--wmm:ignore-->@Raw(FormatEmbedMarkdown(message.ReferencedMessage.Content))<!--/wmm:ignore-->
}
else if (message.ReferencedMessage.Attachments.Any() || message.ReferencedMessage.Embeds.Any())
{
<em>Click to see attachment</em>
<span>🖼️</span>
}
else
{
<em>Click to see original message</em>
}
</span>
@if (message.ReferencedMessage.EditedTimestamp is not null)
{
<span class="chatlog__reference-edited-timestamp" title="@FormatDate(message.ReferencedMessage.EditedTimestamp.Value)">(edited)</span>
}
</div>
}
else
{
<div class="chatlog__reference-unknown">
Original message was deleted or could not be loaded.
</div>
}
</div>
}
// Header
<div class="chatlog__header">
@{/* Author name */}
<span class="chatlog__author" style="@(userColor is not null ? $"color: rgb({userColor.Value.R}, {userColor.Value.G}, {userColor.Value.B})" : null)" title="@message.Author.FullName" data-user-id="@message.Author.Id">@userNick</span>
@{/* Bot label */}
@if (message.Author.IsBot)
{
<span class="chatlog__bot-label">BOT</span>
}
@{/* Timestamp */}
<span class="chatlog__timestamp"><a href="#chatlog__message-container-@message.Id">@FormatDate(message.Timestamp)</a></span>
</div>
}
@{/* Content */}
@if (!string.IsNullOrWhiteSpace(message.Content) || message.EditedTimestamp is not null)
{
<div class="chatlog__content chatlog__markdown">
@{/* Text */}
@if (!isContentHidden)
{
<span class="chatlog__markdown-preserve"><!--wmm:ignore-->@Raw(FormatMarkdown(message.Content))<!--/wmm:ignore--></span>
}
@{/* Edited timestamp */}
@if (message.EditedTimestamp is not null)
{
<span class="chatlog__edited-timestamp" title="@FormatDate(message.EditedTimestamp.Value)">(edited)</span>
}
</div>
}
@{/* Attachments */}
@foreach (var attachment in message.Attachments)
{
<div class="chatlog__attachment @(attachment.IsSpoiler ? "chatlog__attachment--hidden" : "")" onclick="@(attachment.IsSpoiler ? "showSpoiler(event, this)" : "")">
<div class="chatlog__attachment @(attachment.IsSpoiler ? "chatlog__attachment--hidden" : null)" onclick="@(attachment.IsSpoiler ? "showSpoiler(event, this)" : null)">
@{/* Spoiler caption */}
@if (attachment.IsSpoiler)
{
@@ -139,26 +171,26 @@
@if (attachment.IsImage)
{
<a href="@await ResolveUrlAsync(attachment.Url)">
<img class="chatlog__attachment-media" src="@await ResolveUrlAsync(attachment.Url)" alt="Image attachment" title="@($"Image: {attachment.FileName} ({attachment.FileSize})")" loading="lazy">
<img class="chatlog__attachment-media" src="@await ResolveUrlAsync(attachment.Url)" alt="@(attachment.Description ?? "Image attachment")" title="Image: @attachment.FileName (@attachment.FileSize)" loading="lazy">
</a>
}
else if (attachment.IsVideo)
{
<video class="chatlog__attachment-media" controls>
<source src="@await ResolveUrlAsync(attachment.Url)" alt="Video attachment" title="@($"Video: {attachment.FileName} ({attachment.FileSize})")">
<source src="@await ResolveUrlAsync(attachment.Url)" alt="@(attachment.Description ?? "Video attachment")" title="Video: @attachment.FileName (@attachment.FileSize)">
</video>
}
else if (attachment.IsAudio)
{
<audio class="chatlog__attachment-media" controls>
<source src="@await ResolveUrlAsync(attachment.Url)" alt="Audio attachment" title="@($"Audio: {attachment.FileName} ({attachment.FileSize})")">
<source src="@await ResolveUrlAsync(attachment.Url)" alt="@(attachment.Description ?? "Audio attachment")" title="Audio: @attachment.FileName (@attachment.FileSize)">
</audio>
}
else
{
<div class="chatlog__attachment-generic">
<svg class="chatlog__attachment-generic-icon">
<use href="#icon-attachment" />
<use href="#attachment-icon"/>
</svg>
<div class="chatlog__attachment-generic-name">
<a href="@await ResolveUrlAsync(attachment.Url)">
@@ -201,7 +233,7 @@
@{/* Color pill */}
@if (embed.Color is not null)
{
<div class="chatlog__embed-color-pill" style="background-color: rgba(@embed.Color?.R,@embed.Color?.G,@embed.Color?.B,@embed.Color?.A)"></div>
<div class="chatlog__embed-color-pill" style="background-color: rgba(@embed.Color.Value.R, @embed.Color.Value.G, @embed.Color.Value.B, @embed.Color.Value.A)"></div>
}
else
{
@@ -214,7 +246,7 @@
@{/* Embed author */}
@if (embed.Author is not null)
{
<div class="chatlog__embed-author">
<div class="chatlog__embed-author-container">
@if (!string.IsNullOrWhiteSpace(embed.Author.IconUrl))
{
<img class="chatlog__embed-author-icon" src="@await ResolveUrlAsync(embed.Author.IconProxyUrl ?? embed.Author.IconUrl)" alt="Author icon" loading="lazy" onerror="this.style.visibility='hidden'">
@@ -222,16 +254,16 @@
@if (!string.IsNullOrWhiteSpace(embed.Author.Name))
{
<span class="chatlog__embed-author-name">
@if (!string.IsNullOrWhiteSpace(embed.Author.Url))
{
<a class="chatlog__embed-author-name-link" href="@embed.Author.Url">@embed.Author.Name</a>
}
else
{
@embed.Author.Name
}
</span>
if (!string.IsNullOrWhiteSpace(embed.Author.Url))
{
<a class="chatlog__embed-author-link" href="@embed.Author.Url">
<div class="chatlog__embed-author">@embed.Author.Name</div>
</a>
}
else
{
<div class="chatlog__embed-author">@embed.Author.Name</div>
}
}
</div>
}
@@ -243,12 +275,12 @@
@if (!string.IsNullOrWhiteSpace(embed.Url))
{
<a class="chatlog__embed-title-link" href="@embed.Url">
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(embed.Title))</div>
<div class="chatlog__markdown chatlog__markdown-preserve"><!--wmm:ignore-->@Raw(FormatEmbedMarkdown(embed.Title))<!--/wmm:ignore--></div>
</a>
}
else
{
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(embed.Title))</div>
<div class="chatlog__markdown chatlog__markdown-preserve"><!--wmm:ignore-->@Raw(FormatEmbedMarkdown(embed.Title))<!--/wmm:ignore--></div>
}
</div>
}
@@ -269,7 +301,7 @@
@{/* Color pill */}
@if (embed.Color is not null)
{
<div class="chatlog__embed-color-pill" style="background-color: rgba(@embed.Color?.R,@embed.Color?.G,@embed.Color?.B,@embed.Color?.A)"></div>
<div class="chatlog__embed-color-pill" style="background-color: rgba(@embed.Color.Value.R, @embed.Color.Value.G, @embed.Color.Value.B, @embed.Color.Value.A)"></div>
}
else
{
@@ -282,7 +314,7 @@
@{/* Embed author */}
@if (embed.Author is not null)
{
<div class="chatlog__embed-author">
<div class="chatlog__embed-author-container">
@if (!string.IsNullOrWhiteSpace(embed.Author.IconUrl))
{
<img class="chatlog__embed-author-icon" src="@await ResolveUrlAsync(embed.Author.IconProxyUrl ?? embed.Author.IconUrl)" alt="Author icon" loading="lazy" onerror="this.style.visibility='hidden'">
@@ -290,16 +322,16 @@
@if (!string.IsNullOrWhiteSpace(embed.Author.Name))
{
<span class="chatlog__embed-author-name">
@if (!string.IsNullOrWhiteSpace(embed.Author.Url))
{
<a class="chatlog__embed-author-name-link" href="@embed.Author.Url">@embed.Author.Name</a>
}
else
{
@embed.Author.Name
}
</span>
if (!string.IsNullOrWhiteSpace(embed.Author.Url))
{
<a class="chatlog__embed-author-link" href="@embed.Author.Url">
<div class="chatlog__embed-author">@embed.Author.Name</div>
</a>
}
else
{
<div class="chatlog__embed-author">@embed.Author.Name</div>
}
}
</div>
}
@@ -311,12 +343,12 @@
@if (!string.IsNullOrWhiteSpace(embed.Url))
{
<a class="chatlog__embed-title-link" href="@embed.Url">
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(embed.Title))</div>
<div class="chatlog__markdown chatlog__markdown-preserve"><!--wmm:ignore-->@Raw(FormatEmbedMarkdown(embed.Title))<!--/wmm:ignore--></div>
</a>
}
else
{
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(embed.Title))</div>
<div class="chatlog__markdown chatlog__markdown-preserve"><!--wmm:ignore-->@Raw(FormatEmbedMarkdown(embed.Title))<!--/wmm:ignore--></div>
}
</div>
}
@@ -325,7 +357,7 @@
@if (!string.IsNullOrWhiteSpace(embed.Description))
{
<div class="chatlog__embed-description">
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(embed.Description))</div>
<div class="chatlog__markdown chatlog__markdown-preserve"><!--wmm:ignore-->@Raw(FormatEmbedMarkdown(embed.Description))<!--/wmm:ignore--></div>
</div>
}
@@ -339,14 +371,14 @@
@if (!string.IsNullOrWhiteSpace(field.Name))
{
<div class="chatlog__embed-field-name">
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(field.Name))</div>
<div class="chatlog__markdown chatlog__markdown-preserve"><!--wmm:ignore-->@Raw(FormatEmbedMarkdown(field.Name))<!--/wmm:ignore--></div>
</div>
}
@if (!string.IsNullOrWhiteSpace(field.Value))
{
<div class="chatlog__embed-field-value">
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(field.Value))</div>
<div class="chatlog__markdown chatlog__markdown-preserve"><!--wmm:ignore-->@Raw(FormatEmbedMarkdown(field.Value))<!--/wmm:ignore--></div>
</div>
}
</div>
@@ -366,13 +398,21 @@
}
</div>
@{/* Embed image */}
@if (embed.Image is not null && !string.IsNullOrWhiteSpace(embed.Image.Url))
@{/* Embed images */}
@if (embed.Images.Any())
{
<div class="chatlog__embed-image-container">
<a class="chatlog__embed-image-link" href="@await ResolveUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url)">
<img class="chatlog__embed-image" src="@await ResolveUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url)" alt="Image" loading="lazy">
</a>
<div class="chatlog__embed-images @(embed.Images.Count == 1 ? "chatlog__embed-images--single" : null)">
@foreach (var image in embed.Images)
{
if (!string.IsNullOrWhiteSpace(image.Url))
{
<div class="chatlog__embed-image-container">
<a class="chatlog__embed-image-link" href="@await ResolveUrlAsync(image.ProxyUrl ?? image.Url)">
<img class="chatlog__embed-image" src="@await ResolveUrlAsync(image.ProxyUrl ?? image.Url)" alt="Image" loading="lazy">
</a>
</div>
}
}
</div>
}
@@ -411,6 +451,21 @@
}
}
@{/* Stickers */}
@foreach (var sticker in message.Stickers)
{
<div class="chatlog__sticker" title="@sticker.Name">
@if (sticker.Format is StickerFormat.Png or StickerFormat.PngAnimated)
{
<img class="chatlog__sticker--media" src="@await ResolveUrlAsync(sticker.SourceUrl)" alt="Sticker">
}
else if (sticker.Format == StickerFormat.Lottie)
{
<div class="chatlog__sticker--media" data-source="@await ResolveUrlAsync(sticker.SourceUrl)"></div>
}
</div>
}
@{/* Message reactions */}
@if (message.Reactions.Any())
{
@@ -418,13 +473,14 @@
@foreach (var reaction in message.Reactions)
{
<div class="chatlog__reaction" title="@reaction.Emoji.Code">
<img class="emoji emoji--small" alt="@reaction.Emoji.Name" src="@await ResolveUrlAsync(reaction.Emoji.ImageUrl)" loading="lazy">
<img class="chatlog__emoji chatlog__emoji--small" alt="@reaction.Emoji.Name" src="@await ResolveUrlAsync(reaction.Emoji.ImageUrl)" loading="lazy">
<span class="chatlog__reaction-count">@reaction.Count</span>
</div>
}
</div>
}
</div>
}
</div>
</div>
}
</div>
@@ -1,4 +1,6 @@
using DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors;
using System.Collections.Generic;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors;
namespace DiscordChatExporter.Core.Exporting.Writers.Html;
@@ -6,12 +8,12 @@ internal class MessageGroupTemplateContext
{
public ExportContext ExportContext { get; }
public MessageGroup MessageGroup { get; }
public IReadOnlyList<Message> Messages { get; }
public MessageGroupTemplateContext(ExportContext exportContext, MessageGroup messageGroup)
public MessageGroupTemplateContext(ExportContext exportContext, IReadOnlyList<Message> messages)
{
ExportContext = exportContext;
MessageGroup = messageGroup;
Messages = messages;
}
public string FormatMarkdown(string? markdown, bool isJumboAllowed = true) =>
@@ -4,7 +4,9 @@
@inherits MiniRazor.TemplateBase<PostambleTemplateContext>
@{/* Close elements opened by preamble */}
<!--wmm:ignore-->
</div>
<!--/wmm:ignore-->
<div class="postamble">
<div class="postamble__entry">Exported @Model.MessagesWritten.ToString("N0") message(s)</div>
File diff suppressed because it is too large Load Diff
@@ -1,10 +1,12 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.Html;
using WebMarkupMin.Core;
namespace DiscordChatExporter.Core.Exporting.Writers;
@@ -13,7 +15,8 @@ internal class HtmlMessageWriter : MessageWriter
private readonly TextWriter _writer;
private readonly string _themeName;
private readonly List<Message> _messageGroupBuffer = new();
private readonly HtmlMinifier _minifier = new();
private readonly List<Message> _messageGroup = new();
public HtmlMessageWriter(Stream stream, ExportContext context, string themeName)
: base(stream, context)
@@ -22,6 +25,23 @@ internal class HtmlMessageWriter : MessageWriter
_themeName = themeName;
}
private bool CanJoinGroup(Message message) =>
// First message in the group can always join
_messageGroup.LastOrDefault() is not { } lastMessage ||
// Must be from the same author
lastMessage.Author.Id == message.Author.Id &&
// Author's name must not have changed between messages
string.Equals(lastMessage.Author.FullName, message.Author.FullName, StringComparison.Ordinal) &&
// Duration between messages must be 7 minutes or less
(message.Timestamp - lastMessage.Timestamp).Duration().TotalMinutes <= 7 &&
// Other message must not be a reply
message.Reference is null;
// Use <!--wmm:ignore--> to preserve blocks of code inside the templates
private string Minify(string html) => _minifier
.Minify(html, false)
.MinifiedContent;
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
{
var templateContext = new PreambleTemplateContext(Context, _themeName);
@@ -29,20 +49,24 @@ internal class HtmlMessageWriter : MessageWriter
// We are not writing directly to output because Razor
// does not actually do asynchronous writes to stream.
await _writer.WriteLineAsync(
await PreambleTemplate.RenderAsync(templateContext, cancellationToken)
Minify(
await PreambleTemplate.RenderAsync(templateContext, cancellationToken)
)
);
}
private async ValueTask WriteMessageGroupAsync(
MessageGroup messageGroup,
IReadOnlyList<Message> messages,
CancellationToken cancellationToken = default)
{
var templateContext = new MessageGroupTemplateContext(Context, messageGroup);
var templateContext = new MessageGroupTemplateContext(Context, messages);
// We are not writing directly to output because Razor
// does not actually do asynchronous writes to stream.
await _writer.WriteLineAsync(
await MessageGroupTemplate.RenderAsync(templateContext, cancellationToken)
Minify(
await MessageGroupTemplate.RenderAsync(templateContext, cancellationToken)
)
);
}
@@ -52,38 +76,35 @@ internal class HtmlMessageWriter : MessageWriter
{
await base.WriteMessageAsync(message, cancellationToken);
// If message group is empty or the given message can be grouped, buffer the given message
if (!_messageGroupBuffer.Any() || MessageGroup.CanJoin(_messageGroupBuffer.Last(), message))
// If the message can be grouped, buffer it for now
if (CanJoinGroup( message))
{
_messageGroupBuffer.Add(message);
_messageGroup.Add(message);
}
// Otherwise, flush the group and render messages
else
{
await WriteMessageGroupAsync(MessageGroup.Join(_messageGroupBuffer), cancellationToken);
await WriteMessageGroupAsync(_messageGroup, cancellationToken);
_messageGroupBuffer.Clear();
_messageGroupBuffer.Add(message);
_messageGroup.Clear();
_messageGroup.Add(message);
}
}
public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
{
// Flush current message group
if (_messageGroupBuffer.Any())
{
await WriteMessageGroupAsync(
MessageGroup.Join(_messageGroupBuffer),
cancellationToken
);
}
if (_messageGroup.Any())
await WriteMessageGroupAsync(_messageGroup, cancellationToken);
var templateContext = new PostambleTemplateContext(Context, MessagesWritten);
// We are not writing directly to output because Razor
// does not actually do asynchronous writes to stream.
await _writer.WriteLineAsync(
await PostambleTemplate.RenderAsync(templateContext, cancellationToken)
Minify(
await PostambleTemplate.RenderAsync(templateContext, cancellationToken)
)
);
}
@@ -20,6 +20,7 @@ internal class JsonMessageWriter : MessageWriter
{
_writer = new Utf8JsonWriter(stream, new JsonWriterOptions
{
// https://github.com/Tyrrrz/DiscordChatExporter/issues/450
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = true,
// Validation errors may mask actual failures
@@ -50,7 +51,7 @@ internal class JsonMessageWriter : MessageWriter
EmbedAuthor embedAuthor,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject("author");
_writer.WriteStartObject();
_writer.WriteString("name", embedAuthor.Name);
_writer.WriteString("url", embedAuthor.Url);
@@ -62,27 +63,11 @@ internal class JsonMessageWriter : MessageWriter
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteEmbedThumbnailAsync(
EmbedImage embedThumbnail,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject("thumbnail");
if (!string.IsNullOrWhiteSpace(embedThumbnail.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedThumbnail.ProxyUrl ?? embedThumbnail.Url, cancellationToken));
_writer.WriteNumber("width", embedThumbnail.Width);
_writer.WriteNumber("height", embedThumbnail.Height);
_writer.WriteEndObject();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteEmbedImageAsync(
EmbedImage embedImage,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject("image");
_writer.WriteStartObject();
if (!string.IsNullOrWhiteSpace(embedImage.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedImage.ProxyUrl ?? embedImage.Url, cancellationToken));
@@ -98,7 +83,7 @@ internal class JsonMessageWriter : MessageWriter
EmbedFooter embedFooter,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject("footer");
_writer.WriteStartObject();
_writer.WriteString("text", embedFooter.Text);
@@ -138,16 +123,37 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteString("color", embed.Color.Value.ToHex());
if (embed.Author is not null)
{
_writer.WritePropertyName("author");
await WriteEmbedAuthorAsync(embed.Author, cancellationToken);
}
if (embed.Thumbnail is not null)
await WriteEmbedThumbnailAsync(embed.Thumbnail, cancellationToken);
{
_writer.WritePropertyName("thumbnail");
await WriteEmbedImageAsync(embed.Thumbnail, cancellationToken);
}
if (embed.Image is not null)
await WriteEmbedImageAsync(embed.Image, cancellationToken);
// Legacy: backwards-compatibility for old embeds with a single image
if (embed.Images.Count > 0)
{
_writer.WritePropertyName("image");
await WriteEmbedImageAsync(embed.Images[0], cancellationToken);
}
if (embed.Footer is not null)
{
_writer.WritePropertyName("footer");
await WriteEmbedFooterAsync(embed.Footer, cancellationToken);
}
// Images
_writer.WriteStartArray("images");
foreach (var image in embed.Images)
await WriteEmbedImageAsync(image, cancellationToken);
_writer.WriteEndArray();
// Fields
_writer.WriteStartArray("fields");
@@ -161,6 +167,21 @@ internal class JsonMessageWriter : MessageWriter
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteStickerAsync(
Sticker sticker,
CancellationToken cancellationToken = default)
{
_writer.WriteStartObject();
_writer.WriteString("id", sticker.Id.ToString());
_writer.WriteString("name", sticker.Name);
_writer.WriteString("format", sticker.Format.ToString());
_writer.WriteString("sourceUrl", await Context.ResolveMediaUrlAsync(sticker.SourceUrl, cancellationToken));
_writer.WriteEndObject();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteReactionAsync(
Reaction reaction,
CancellationToken cancellationToken = default)
@@ -169,7 +190,7 @@ internal class JsonMessageWriter : MessageWriter
// Emoji
_writer.WriteStartObject("emoji");
_writer.WriteString("id", reaction.Emoji.Id);
_writer.WriteString("id", reaction.Emoji.Id.ToString());
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", await Context.ResolveMediaUrlAsync(reaction.Emoji.ImageUrl, cancellationToken));
@@ -276,6 +297,14 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteEndArray();
// Stickers
_writer.WriteStartArray("stickers");
foreach (var sticker in message.Stickers)
await WriteStickerAsync(sticker, cancellationToken);
_writer.WriteEndArray();
// Reactions
_writer.WriteStartArray("reactions");
@@ -3,7 +3,6 @@ using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Parsing;
@@ -34,14 +33,37 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
{
var (tagOpen, tagClose) = formatting.Kind switch
{
FormattingKind.Bold => ("<strong>", "</strong>"),
FormattingKind.Italic => ("<em>", "</em>"),
FormattingKind.Underline => ("<u>", "</u>"),
FormattingKind.Strikethrough => ("<s>", "</s>"),
FormattingKind.Bold => (
"<strong>",
"</strong>"
),
FormattingKind.Italic => (
"<em>",
"</em>"
),
FormattingKind.Underline => (
"<u>",
"</u>"
),
FormattingKind.Strikethrough => (
"<s>",
"</s>"
),
FormattingKind.Spoiler => (
"<span class=\"spoiler-text spoiler-text--hidden\" onclick=\"showSpoiler(event, this)\">", "</span>"),
FormattingKind.Quote => ("<div class=\"quote\">", "</div>"),
_ => throw new ArgumentOutOfRangeException(nameof(formatting.Kind))
"<span class=\"chatlog__markdown-spoiler chatlog__markdown-spoiler--hidden\" onclick=\"showSpoiler(event, this)\">",
"</span>"
),
FormattingKind.Quote => (
"<div class=\"chatlog__markdown-quote\"><div class=\"chatlog__markdown-quote-border\"></div><div class=\"chatlog__markdown-quote-content\">",
"</div></div>"
),
_ => throw new InvalidOperationException($"Unknown formatting kind '{formatting.Kind}'.")
};
_buffer.Append(tagOpen);
@@ -54,9 +76,9 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override MarkdownNode VisitInlineCodeBlock(InlineCodeBlockNode inlineCodeBlock)
{
_buffer
.Append("<span class=\"pre pre--inline\">")
.Append("<code class=\"chatlog__markdown-pre chatlog__markdown-pre--inline\">")
.Append(HtmlEncode(inlineCodeBlock.Code))
.Append("</span>");
.Append("</code>");
return base.VisitInlineCodeBlock(inlineCodeBlock);
}
@@ -68,9 +90,9 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
: "nohighlight";
_buffer
.Append($"<div class=\"pre pre--multiline {highlightCssClass}\">")
.Append($"<code class=\"chatlog__markdown-pre chatlog__markdown-pre--multiline {highlightCssClass}\">")
.Append(HtmlEncode(multiLineCodeBlock.Code))
.Append("</div>");
.Append("</code>");
return base.VisitMultiLineCodeBlock(multiLineCodeBlock);
}
@@ -85,8 +107,8 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
_buffer.Append(
!string.IsNullOrWhiteSpace(linkedMessageId)
? $"<a href=\"{Uri.EscapeUriString(link.Url)}\" onclick=\"scrollToMessage(event, '{linkedMessageId}')\">"
: $"<a href=\"{Uri.EscapeUriString(link.Url)}\">"
? $"<a href=\"{HtmlEncode(link.Url)}\" onclick=\"scrollToMessage(event, '{linkedMessageId}')\">"
: $"<a href=\"{HtmlEncode(link.Url)}\">"
);
var result = base.VisitLink(link);
@@ -98,59 +120,66 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override MarkdownNode VisitEmoji(EmojiNode emoji)
{
var emojiImageUrl = Emoji.GetImageUrl(emoji.Id, emoji.Name, emoji.IsAnimated);
var jumboClass = _isJumbo ? "emoji--large" : "";
var jumboClass = _isJumbo ? "chatlog__emoji--large" : "";
_buffer
.Append($"<img loading=\"lazy\" class=\"emoji {jumboClass}\" alt=\"{emoji.Name}\" title=\"{emoji.Code}\" src=\"{emojiImageUrl}\">");
.Append($"<img loading=\"lazy\" class=\"chatlog__emoji {jumboClass}\" alt=\"{emoji.Name}\" title=\"{emoji.Code}\" src=\"{emojiImageUrl}\">");
return base.VisitEmoji(emoji);
}
protected override MarkdownNode VisitMention(MentionNode mention)
{
var mentionId = Snowflake.TryParse(mention.Id);
if (mention.Kind == MentionKind.Meta)
if (mention.Kind == MentionKind.Everyone)
{
_buffer
.Append("<span class=\"mention\">")
.Append("@").Append(HtmlEncode(mention.Id))
.Append("<span class=\"chatlog__markdown-mention\">")
.Append("@everyone")
.Append("</span>");
}
else if (mention.Kind == MentionKind.Here)
{
_buffer
.Append("<span class=\"chatlog__markdown-mention\">")
.Append("@here")
.Append("</span>");
}
else if (mention.Kind == MentionKind.User)
{
var member = mentionId?.Pipe(_context.TryGetMember);
var member = mention.TargetId?.Pipe(_context.TryGetMember);
var fullName = member?.User.FullName ?? "Unknown";
var nick = member?.Nick ?? "Unknown";
_buffer
.Append($"<span class=\"mention\" title=\"{HtmlEncode(fullName)}\">")
.Append("@").Append(HtmlEncode(nick))
.Append($"<span class=\"chatlog__markdown-mention\" title=\"{HtmlEncode(fullName)}\">")
.Append('@').Append(HtmlEncode(nick))
.Append("</span>");
}
else if (mention.Kind == MentionKind.Channel)
{
var channel = mentionId?.Pipe(_context.TryGetChannel);
var symbol = channel?.IsVoiceChannel == true ? "🔊" : "#";
var channel = mention.TargetId?.Pipe(_context.TryGetChannel);
var symbol = channel?.SupportsVoice == true ? "🔊" : "#";
var name = channel?.Name ?? "deleted-channel";
_buffer
.Append("<span class=\"mention\">")
.Append("<span class=\"chatlog__markdown-mention\">")
.Append(symbol).Append(HtmlEncode(name))
.Append("</span>");
}
else if (mention.Kind == MentionKind.Role)
{
var role = mentionId?.Pipe(_context.TryGetRole);
var role = mention.TargetId?.Pipe(_context.TryGetRole);
var name = role?.Name ?? "deleted-role";
var color = role?.Color;
var style = color is not null
? $"color: rgb({color?.R}, {color?.G}, {color?.B}); background-color: rgba({color?.R}, {color?.G}, {color?.B}, 0.1);"
? $"color: rgb({color.Value.R}, {color.Value.G}, {color.Value.B}); " +
$"background-color: rgba({color.Value.R}, {color.Value.G}, {color.Value.B}, 0.1);"
: "";
_buffer
.Append($"<span class=\"mention\" style=\"{style}\">")
.Append("@").Append(HtmlEncode(name))
.Append($"<span class=\"chatlog__markdown-mention\" style=\"{style}\">")
.Append('@').Append(HtmlEncode(name))
.Append("</span>");
}
@@ -159,12 +188,18 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
protected override MarkdownNode VisitUnixTimestamp(UnixTimestampNode timestamp)
{
var dateString = timestamp.Date is not null
? _context.FormatDate(timestamp.Date.Value)
: "Invalid date";
// Timestamp tooltips always use full date regardless of the configured format
var longDateString = timestamp.Value.ToLocalString("dddd, MMMM d, yyyy h:mm tt");
var longDateString = timestamp.Date is not null
? timestamp.Date.Value.ToLocalString("dddd, MMMM d, yyyy h:mm tt")
: "Invalid date";
_buffer
.Append($"<span class=\"timestamp\" title=\"{HtmlEncode(longDateString)}\">")
.Append(HtmlEncode(_context.FormatDate(timestamp.Value)))
.Append($"<span class=\"chatlog__markdown-timestamp\" title=\"{HtmlEncode(longDateString)}\">")
.Append(HtmlEncode(dateString))
.Append("</span>");
return base.VisitUnixTimestamp(timestamp);
@@ -1,5 +1,4 @@
using System.Text;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Parsing;
using DiscordChatExporter.Core.Utils.Extensions;
@@ -36,32 +35,35 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
protected override MarkdownNode VisitMention(MentionNode mention)
{
var mentionId = Snowflake.TryParse(mention.Id);
if (mention.Kind == MentionKind.Meta)
if (mention.Kind == MentionKind.Everyone)
{
_buffer.Append($"@{mention.Id}");
_buffer.Append("@everyone");
}
else if (mention.Kind == MentionKind.Here)
{
_buffer.Append("@here");
}
else if (mention.Kind == MentionKind.User)
{
var member = mentionId?.Pipe(_context.TryGetMember);
var member = mention.TargetId?.Pipe(_context.TryGetMember);
var name = member?.User.Name ?? "Unknown";
_buffer.Append($"@{name}");
}
else if (mention.Kind == MentionKind.Channel)
{
var channel = mentionId?.Pipe(_context.TryGetChannel);
var channel = mention.TargetId?.Pipe(_context.TryGetChannel);
var name = channel?.Name ?? "deleted-channel";
_buffer.Append($"#{name}");
// Voice channel marker
if (channel?.IsVoiceChannel == true)
if (channel?.SupportsVoice == true)
_buffer.Append(" [voice]");
}
else if (mention.Kind == MentionKind.Role)
{
var role = mentionId?.Pipe(_context.TryGetRole);
var role = mention.TargetId?.Pipe(_context.TryGetRole);
var name = role?.Name ?? "deleted-role";
_buffer.Append($"@{name}");
@@ -73,7 +75,9 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
protected override MarkdownNode VisitUnixTimestamp(UnixTimestampNode timestamp)
{
_buffer.Append(
_context.FormatDate(timestamp.Value)
timestamp.Date is not null
? _context.FormatDate(timestamp.Date.Value)
: "Invalid date"
);
return base.VisitUnixTimestamp(timestamp);
@@ -88,8 +88,11 @@ internal class PlainTextMessageWriter : MessageWriter
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Thumbnail.ProxyUrl ?? embed.Thumbnail.Url, cancellationToken));
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url, cancellationToken));
foreach (var image in embed.Images)
{
if (!string.IsNullOrWhiteSpace(image.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(image.ProxyUrl ?? image.Url, cancellationToken));
}
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
await _writer.WriteLineAsync(embed.Footer.Text);
@@ -98,6 +101,25 @@ internal class PlainTextMessageWriter : MessageWriter
}
}
private async ValueTask WriteStickersAsync(
IReadOnlyList<Sticker> stickers,
CancellationToken cancellationToken = default)
{
if (!stickers.Any())
return;
await _writer.WriteLineAsync("{Stickers}");
foreach (var sticker in stickers)
{
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(sticker.SourceUrl, cancellationToken));
}
await _writer.WriteLineAsync();
}
private async ValueTask WriteReactionsAsync(
IReadOnlyList<Reaction> reactions,
CancellationToken cancellationToken = default)
@@ -156,9 +178,10 @@ internal class PlainTextMessageWriter : MessageWriter
await _writer.WriteLineAsync();
// Attachments, embeds, reactions
// Attachments, embeds, reactions, etc.
await WriteAttachmentsAsync(message.Attachments, cancellationToken);
await WriteEmbedsAsync(message.Embeds, cancellationToken);
await WriteStickersAsync(message.Stickers, cancellationToken);
await WriteReactionsAsync(message.Reactions, cancellationToken);
await _writer.WriteLineAsync();
@@ -1,21 +1,22 @@
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Utils;
namespace DiscordChatExporter.Core.Markdown;
internal record EmojiNode(
// Only present on custom emoji
string? Id,
// Name of custom emoji (e.g. LUL) or actual representation of standard emoji (e.g. 🙂)
Snowflake? Id,
// Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂)
string Name,
bool IsAnimated) : MarkdownNode
{
// Name of custom emoji (e.g. LUL) or name of standard emoji (e.g. slight_smile)
public string Code => !string.IsNullOrWhiteSpace(Id)
public bool IsCustomEmoji => Id is not null;
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => IsCustomEmoji
? Name
: EmojiIndex.TryGetCode(Name) ?? Name;
public bool IsCustomEmoji => !string.IsNullOrWhiteSpace(Id);
public EmojiNode(string name)
: this(null, name, false)
{
@@ -2,7 +2,8 @@
internal enum MentionKind
{
Meta,
Everyone,
Here,
User,
Channel,
Role
@@ -1,3 +1,6 @@
namespace DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Discord;
internal record MentionNode(string Id, MentionKind Kind) : MarkdownNode;
namespace DiscordChatExporter.Core.Markdown;
// Null ID means it's a meta mention or an invalid mention
internal record MentionNode(Snowflake? TargetId, MentionKind Kind) : MarkdownNode;
@@ -16,7 +16,7 @@ internal class AggregateMatcher<T> : IMatcher<T>
{
}
public ParsedMatch<T>? TryMatch(StringPart stringPart)
public ParsedMatch<T>? TryMatch(StringSegment segment)
{
ParsedMatch<T>? earliestMatch = null;
@@ -24,19 +24,19 @@ internal class AggregateMatcher<T> : IMatcher<T>
foreach (var matcher in _matchers)
{
// Try to match
var match = matcher.TryMatch(stringPart);
var match = matcher.TryMatch(segment);
// If there's no match - continue
if (match is null)
continue;
// If this match is earlier than previous earliest - replace
if (earliestMatch is null || match.StringPart.StartIndex < earliestMatch.StringPart.StartIndex)
if (earliestMatch is null || match.Segment.StartIndex < earliestMatch.Segment.StartIndex)
earliestMatch = match;
// If the earliest match starts at the very beginning - break,
// because it's impossible to find a match earlier than that
if (earliestMatch.StringPart.StartIndex == stringPart.StartIndex)
if (earliestMatch.Segment.StartIndex == segment.StartIndex)
break;
}
@@ -5,44 +5,57 @@ namespace DiscordChatExporter.Core.Markdown.Parsing;
internal interface IMatcher<T>
{
ParsedMatch<T>? TryMatch(StringPart stringPart);
ParsedMatch<T>? TryMatch(StringSegment segment);
}
internal static class MatcherExtensions
{
public static IEnumerable<ParsedMatch<T>> MatchAll<T>(this IMatcher<T> matcher,
StringPart stringPart, Func<StringPart, T> transformFallback)
public static IEnumerable<ParsedMatch<T>> MatchAll<T>(
this IMatcher<T> matcher,
StringSegment segment,
Func<StringSegment, T> transformFallback)
{
// Loop through segments divided by individual matches
var currentIndex = stringPart.StartIndex;
while (currentIndex < stringPart.EndIndex)
var currentIndex = segment.StartIndex;
while (currentIndex < segment.EndIndex)
{
// Find a match within this segment
var match = matcher.TryMatch(stringPart.Slice(currentIndex, stringPart.EndIndex - currentIndex));
var match = matcher.TryMatch(
segment.Relocate(
currentIndex,
segment.EndIndex - currentIndex
)
);
// If there's no match - break
if (match is null)
break;
// If this match doesn't start immediately at current index - transform and yield fallback first
if (match.StringPart.StartIndex > currentIndex)
// If this match doesn't start immediately at the current position - transform and yield fallback first
if (match.Segment.StartIndex > currentIndex)
{
var fallbackPart = stringPart.Slice(currentIndex, match.StringPart.StartIndex - currentIndex);
yield return new ParsedMatch<T>(fallbackPart, transformFallback(fallbackPart));
var fallbackSegment = segment.Relocate(
currentIndex,
match.Segment.StartIndex - currentIndex
);
yield return new ParsedMatch<T>(fallbackSegment, transformFallback(fallbackSegment));
}
// Yield match
yield return match;
// Shift current index to the end of the match
currentIndex = match.StringPart.StartIndex + match.StringPart.Length;
currentIndex = match.Segment.StartIndex + match.Segment.Length;
}
// If EOL wasn't reached - transform and yield remaining part as fallback
if (currentIndex < stringPart.EndIndex)
// If EOL hasn't been reached - transform and yield remaining part as fallback
if (currentIndex < segment.EndIndex)
{
var fallbackPart = stringPart.Slice(currentIndex);
yield return new ParsedMatch<T>(fallbackPart, transformFallback(fallbackPart));
var fallbackSegment = segment.Relocate(
currentIndex,
segment.EndIndex - currentIndex
);
yield return new ParsedMatch<T>(fallbackSegment, transformFallback(fallbackSegment));
}
}
}
@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Utils;
namespace DiscordChatExporter.Core.Markdown.Parsing;
@@ -23,7 +24,7 @@ internal static partial class MarkdownParser
// Capture any character until the earliest double asterisk not followed by an asterisk
private static readonly IMatcher<MarkdownNode> BoldFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\*\\*(.+?)\\*\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattingNode(FormattingKind.Bold, Parse(p.Slice(m.Groups[1])))
(s, m) => new FormattingNode(FormattingKind.Bold, Parse(s.Relocate(m.Groups[1])))
);
// Capture any character until the earliest single asterisk not preceded or followed by an asterisk
@@ -31,61 +32,62 @@ internal static partial class MarkdownParser
// Closing asterisk must not be preceded by whitespace
private static readonly IMatcher<MarkdownNode> ItalicFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\*(?!\\s)(.+?)(?<!\\s|\\*)\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattingNode(FormattingKind.Italic, Parse(p.Slice(m.Groups[1])))
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
// Capture any character until the earliest triple asterisk not followed by an asterisk
private static readonly IMatcher<MarkdownNode> ItalicBoldFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\*(\\*\\*.+?\\*\\*)\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattingNode(FormattingKind.Italic, Parse(p.Slice(m.Groups[1]), BoldFormattingNodeMatcher))
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1]), BoldFormattingNodeMatcher))
);
// Capture any character except underscore until an underscore
// Closing underscore must not be followed by a word character
private static readonly IMatcher<MarkdownNode> ItalicAltFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("_([^_]+)_(?!\\w)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattingNode(FormattingKind.Italic, Parse(p.Slice(m.Groups[1])))
(s, m) => new FormattingNode(FormattingKind.Italic, Parse(s.Relocate(m.Groups[1])))
);
// Capture any character until the earliest double underscore not followed by an underscore
private static readonly IMatcher<MarkdownNode> UnderlineFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("__(.+?)__(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattingNode(FormattingKind.Underline, Parse(p.Slice(m.Groups[1])))
(s, m) => new FormattingNode(FormattingKind.Underline, Parse(s.Relocate(m.Groups[1])))
);
// Capture any character until the earliest triple underscore not followed by an underscore
private static readonly IMatcher<MarkdownNode> ItalicUnderlineFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex("_(__.+?__)_(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattingNode(FormattingKind.Italic,
Parse(p.Slice(m.Groups[1]), UnderlineFormattingNodeMatcher))
(s, m) => new FormattingNode(FormattingKind.Italic,
Parse(s.Relocate(m.Groups[1]), UnderlineFormattingNodeMatcher))
);
// Capture any character until the earliest double tilde
private static readonly IMatcher<MarkdownNode> StrikethroughFormattingNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex("~~(.+?)~~", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattingNode(FormattingKind.Strikethrough, Parse(p.Slice(m.Groups[1])))
(s, m) => new FormattingNode(FormattingKind.Strikethrough, Parse(s.Relocate(m.Groups[1])))
);
// Capture any character until the earliest double pipe
private static readonly IMatcher<MarkdownNode> SpoilerFormattingNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\|\\|(.+?)\\|\\|", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattingNode(FormattingKind.Spoiler, Parse(p.Slice(m.Groups[1])))
(s, m) => new FormattingNode(FormattingKind.Spoiler, Parse(s.Relocate(m.Groups[1])))
);
// Capture any character until the end of the line
// Opening 'greater than' character must be followed by whitespace
// Text content is optional
private static readonly IMatcher<MarkdownNode> SingleLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("^>\\s(.+\n?)", DefaultRegexOptions),
(p, m) => new FormattingNode(FormattingKind.Quote, Parse(p.Slice(m.Groups[1])))
new Regex("^>\\s(.*\n?)", DefaultRegexOptions),
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
// Repeatedly capture any character until the end of the line
// This one is tricky as it ends up producing multiple separate captures which need to be joined
private static readonly IMatcher<MarkdownNode> RepeatedSingleLineQuoteNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex("(?:^>\\s(.+\n?)){2,}", DefaultRegexOptions),
new Regex("(?:^>\\s(.*\n?)){2,}", DefaultRegexOptions),
(_, m) =>
{
var content = string.Concat(m.Groups[1].Captures.Select(c => c.Value));
@@ -97,7 +99,7 @@ internal static partial class MarkdownParser
// Opening 'greater than' characters must be followed by whitespace
private static readonly IMatcher<MarkdownNode> MultiLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("^>>>\\s(.+)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattingNode(FormattingKind.Quote, Parse(p.Slice(m.Groups[1])))
(s, m) => new FormattingNode(FormattingKind.Quote, Parse(s.Relocate(m.Groups[1])))
);
/* Code blocks */
@@ -123,31 +125,31 @@ internal static partial class MarkdownParser
// Capture @everyone
private static readonly IMatcher<MarkdownNode> EveryoneMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@everyone",
_ => new MentionNode("everyone", MentionKind.Meta)
_ => new MentionNode(null, MentionKind.Everyone)
);
// Capture @here
private static readonly IMatcher<MarkdownNode> HereMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@here",
_ => new MentionNode("here", MentionKind.Meta)
_ => new MentionNode(null, MentionKind.Here)
);
// Capture <@123456> or <@!123456>
private static readonly IMatcher<MarkdownNode> UserMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<@!?(\\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(m.Groups[1].Value, MentionKind.User)
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.User)
);
// Capture <#123456>
private static readonly IMatcher<MarkdownNode> ChannelMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<#!?(\\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(m.Groups[1].Value, MentionKind.Channel)
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Channel)
);
// Capture <@&123456>
private static readonly IMatcher<MarkdownNode> RoleMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<@&(\\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(m.Groups[1].Value, MentionKind.Role)
(_, m) => new MentionNode(Snowflake.TryParse(m.Groups[1].Value), MentionKind.Role)
);
/* Emoji */
@@ -177,7 +179,11 @@ internal static partial class MarkdownParser
// Capture <:lul:123456> or <a:lul:123456>
private static readonly IMatcher<MarkdownNode> CustomEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<(a)?:(.+?):(\\d+?)>", DefaultRegexOptions),
(_, m) => new EmojiNode(m.Groups[3].Value, m.Groups[2].Value, !string.IsNullOrWhiteSpace(m.Groups[1].Value))
(_, m) => new EmojiNode(
Snowflake.TryParse(m.Groups[3].Value),
m.Groups[2].Value,
!string.IsNullOrWhiteSpace(m.Groups[1].Value)
)
);
/* Links */
@@ -185,7 +191,7 @@ internal static partial class MarkdownParser
// Capture [title](link)
private static readonly IMatcher<MarkdownNode> TitledLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\[(.+?)\\]\\((.+?)\\)", DefaultRegexOptions),
(p, m) => new LinkNode(m.Groups[2].Value, Parse(p.Slice(m.Groups[1])))
(s, m) => new LinkNode(m.Groups[2].Value, Parse(s.Relocate(m.Groups[1])))
);
// Capture any non-whitespace character after http:// or https://
@@ -207,7 +213,7 @@ internal static partial class MarkdownParser
// This escapes it from matching for formatting
private static readonly IMatcher<MarkdownNode> ShrugTextNodeMatcher = new StringMatcher<MarkdownNode>(
@"¯\_(ツ)_/¯",
p => new TextNode(p.ToString())
s => new TextNode(s.ToString())
);
// Capture some specific emoji that don't get rendered
@@ -235,7 +241,7 @@ internal static partial class MarkdownParser
// Capture <t:12345678> or <t:12345678:R>
private static readonly IMatcher<MarkdownNode> UnixTimestampNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<t:(\\d+)(?::\\w)?>", DefaultRegexOptions),
new Regex("<t:(-?\\d+)(?::\\w)?>", DefaultRegexOptions),
(_, m) =>
{
// TODO: support formatting parameters
@@ -244,17 +250,19 @@ internal static partial class MarkdownParser
if (!long.TryParse(m.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture,
out var offset))
{
return null;
return new UnixTimestampNode(null);
}
// Bound check
// https://github.com/Tyrrrz/DiscordChatExporter/issues/681
if (offset < TimeSpan.MinValue.TotalSeconds || offset > TimeSpan.MaxValue.TotalSeconds)
try
{
return null;
return new UnixTimestampNode(DateTimeOffset.UnixEpoch + TimeSpan.FromSeconds(offset));
}
// https://github.com/Tyrrrz/DiscordChatExporter/issues/681
// https://github.com/Tyrrrz/DiscordChatExporter/issues/766
catch (Exception ex) when (ex is ArgumentOutOfRangeException or OverflowException)
{
return new UnixTimestampNode(null);
}
return new UnixTimestampNode(DateTimeOffset.UnixEpoch + TimeSpan.FromSeconds(offset));
}
);
@@ -321,24 +329,24 @@ internal static partial class MarkdownParser
UnixTimestampNodeMatcher
);
private static IReadOnlyList<MarkdownNode> Parse(StringPart stringPart, IMatcher<MarkdownNode> matcher) =>
private static IReadOnlyList<MarkdownNode> Parse(StringSegment segment, IMatcher<MarkdownNode> matcher) =>
matcher
.MatchAll(stringPart, p => new TextNode(p.ToString()))
.MatchAll(segment, s => new TextNode(s.ToString()))
.Select(r => r.Value)
.ToArray();
}
internal static partial class MarkdownParser
{
private static IReadOnlyList<MarkdownNode> Parse(StringPart stringPart) =>
Parse(stringPart, AggregateNodeMatcher);
private static IReadOnlyList<MarkdownNode> Parse(StringSegment segment) =>
Parse(segment, AggregateNodeMatcher);
private static IReadOnlyList<MarkdownNode> ParseMinimal(StringPart stringPart) =>
Parse(stringPart, MinimalAggregateNodeMatcher);
private static IReadOnlyList<MarkdownNode> ParseMinimal(StringSegment segment) =>
Parse(segment, MinimalAggregateNodeMatcher);
public static IReadOnlyList<MarkdownNode> Parse(string input) =>
Parse(new StringPart(input));
Parse(new StringSegment(input));
public static IReadOnlyList<MarkdownNode> ParseMinimal(string input) =>
ParseMinimal(new StringPart(input));
ParseMinimal(new StringSegment(input));
}
@@ -2,13 +2,13 @@
internal class ParsedMatch<T>
{
public StringPart StringPart { get; }
public StringSegment Segment { get; }
public T Value { get; }
public ParsedMatch(StringPart stringPart, T value)
public ParsedMatch(StringSegment segment, T value)
{
StringPart = stringPart;
Segment = segment;
Value = value;
}
}
@@ -6,17 +6,17 @@ namespace DiscordChatExporter.Core.Markdown.Parsing;
internal class RegexMatcher<T> : IMatcher<T>
{
private readonly Regex _regex;
private readonly Func<StringPart, Match, T?> _transform;
private readonly Func<StringSegment, Match, T?> _transform;
public RegexMatcher(Regex regex, Func<StringPart, Match, T?> transform)
public RegexMatcher(Regex regex, Func<StringSegment, Match, T?> transform)
{
_regex = regex;
_transform = transform;
}
public ParsedMatch<T>? TryMatch(StringPart stringPart)
public ParsedMatch<T>? TryMatch(StringSegment segment)
{
var match = _regex.Match(stringPart.Target, stringPart.StartIndex, stringPart.Length);
var match = _regex.Match(segment.Source, segment.StartIndex, segment.Length);
if (!match.Success)
return null;
@@ -25,14 +25,14 @@ internal class RegexMatcher<T> : IMatcher<T>
// Which is super weird because regex.Match(string, int) takes the whole input in context.
// So in order to properly account for ^/$ regex tokens, we need to make sure that
// the expression also matches on the bigger part of the input.
if (!_regex.IsMatch(stringPart.Target[..stringPart.EndIndex], stringPart.StartIndex))
if (!_regex.IsMatch(segment.Source[..segment.EndIndex], segment.StartIndex))
return null;
var stringPartMatch = stringPart.Slice(match.Index, match.Length);
var value = _transform(stringPartMatch, match);
var segmentMatch = segment.Relocate(match);
var value = _transform(segmentMatch, match);
return value is not null
? new ParsedMatch<T>(stringPartMatch, value)
? new ParsedMatch<T>(segmentMatch, value)
: null;
}
}
@@ -6,31 +6,31 @@ internal class StringMatcher<T> : IMatcher<T>
{
private readonly string _needle;
private readonly StringComparison _comparison;
private readonly Func<StringPart, T?> _transform;
private readonly Func<StringSegment, T?> _transform;
public StringMatcher(string needle, StringComparison comparison, Func<StringPart, T?> transform)
public StringMatcher(string needle, StringComparison comparison, Func<StringSegment, T?> transform)
{
_needle = needle;
_comparison = comparison;
_transform = transform;
}
public StringMatcher(string needle, Func<StringPart, T> transform)
public StringMatcher(string needle, Func<StringSegment, T> transform)
: this(needle, StringComparison.Ordinal, transform)
{
}
public ParsedMatch<T>? TryMatch(StringPart stringPart)
public ParsedMatch<T>? TryMatch(StringSegment segment)
{
var index = stringPart.Target.IndexOf(_needle, stringPart.StartIndex, stringPart.Length, _comparison);
var index = segment.Source.IndexOf(_needle, segment.StartIndex, segment.Length, _comparison);
if (index < 0)
return null;
var stringPartMatch = stringPart.Slice(index, _needle.Length);
var value = _transform(stringPartMatch);
var segmentMatch = segment.Relocate(index, _needle.Length);
var value = _transform(segmentMatch);
return value is not null
? new ParsedMatch<T>(stringPartMatch, value)
? new ParsedMatch<T>(segmentMatch, value)
: null;
}
}
@@ -1,21 +0,0 @@
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Core.Markdown.Parsing;
internal readonly record struct StringPart(string Target, int StartIndex, int Length)
{
public int EndIndex => StartIndex + Length;
public StringPart(string target)
: this(target, 0, target.Length)
{
}
public StringPart Slice(int newStartIndex, int newLength) => new(Target, newStartIndex, newLength);
public StringPart Slice(int newStartIndex) => Slice(newStartIndex, EndIndex - newStartIndex);
public StringPart Slice(Capture capture) => Slice(capture.Index, capture.Length);
public override string ToString() => Target.Substring(StartIndex, Length);
}
@@ -0,0 +1,19 @@
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Core.Markdown.Parsing;
internal readonly record struct StringSegment(string Source, int StartIndex, int Length)
{
public int EndIndex => StartIndex + Length;
public StringSegment(string target)
: this(target, 0, target.Length)
{
}
public StringSegment Relocate(int newStartIndex, int newLength) => new(Source, newStartIndex, newLength);
public StringSegment Relocate(Capture capture) => Relocate(capture.Index, capture.Length);
public override string ToString() => Source.Substring(StartIndex, Length);
}
@@ -2,4 +2,5 @@
namespace DiscordChatExporter.Core.Markdown;
internal record UnixTimestampNode(DateTimeOffset Value) : MarkdownNode;
// Null date means invalid timestamp
internal record UnixTimestampNode(DateTimeOffset? Date) : MarkdownNode;
@@ -0,0 +1,18 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Utils.Extensions;
public static class CollectionExtensions
{
public static IEnumerable<T> Enumerate<T>(this T obj)
{
yield return obj;
}
public static IEnumerable<(T value, int index)> WithIndex<T>(this IEnumerable<T> source)
{
var i = 0;
foreach (var o in source)
yield return (o, i++);
}
}
@@ -0,0 +1,44 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Net;
using System.Net.Http;
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Core.Utils.Extensions;
public static class ExceptionExtensions
{
private static void PopulateChildren(this Exception exception, ICollection<Exception> children)
{
if (exception is AggregateException aggregateException)
{
foreach (var innerException in aggregateException.InnerExceptions)
{
children.Add(innerException);
PopulateChildren(innerException, children);
}
}
else if (exception.InnerException is not null)
{
children.Add(exception.InnerException);
PopulateChildren(exception.InnerException, children);
}
}
public static IReadOnlyList<Exception> GetSelfAndChildren(this Exception exception)
{
var children = new List<Exception> {exception};
PopulateChildren(exception, children);
return children;
}
public static HttpStatusCode? TryGetStatusCode(this HttpRequestException ex) =>
// This is extremely frail, but there's no other way
Regex
.Match(ex.Message, @": (\d+) \(")
.Groups[1]
.Value
.NullIfWhiteSpace()?
.Pipe(s => (HttpStatusCode) int.Parse(s, CultureInfo.InvariantCulture));
}
+22 -29
View File
@@ -1,9 +1,8 @@
using System;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Net.Sockets;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Utils.Extensions;
using Polly;
@@ -14,13 +13,27 @@ public static class Http
{
public static HttpClient Client { get; } = new();
public static IAsyncPolicy<HttpResponseMessage> ResponsePolicy { get; } =
private static bool IsRetryableStatusCode(HttpStatusCode statusCode) =>
statusCode is HttpStatusCode.TooManyRequests or HttpStatusCode.RequestTimeout ||
// Treat all server-side errors as retryable.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/908
(int)statusCode >= 500;
private static bool IsRetryableException(Exception exception) =>
exception.GetSelfAndChildren().Any(ex =>
ex is TimeoutException or SocketException ||
ex is HttpRequestException hrex && IsRetryableStatusCode(hrex.TryGetStatusCode() ?? HttpStatusCode.OK)
);
public static IAsyncPolicy ResiliencePolicy { get; } =
Policy
.Handle<IOException>()
.Or<HttpRequestException>()
.OrResult<HttpResponseMessage>(m => m.StatusCode == HttpStatusCode.TooManyRequests)
.OrResult(m => m.StatusCode == HttpStatusCode.RequestTimeout)
.OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError)
.Handle<Exception>(IsRetryableException)
.WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1));
public static IAsyncPolicy<HttpResponseMessage> ResponseResiliencePolicy { get; } =
Policy
.Handle<Exception>(IsRetryableException)
.OrResult<HttpResponseMessage>(m => IsRetryableStatusCode(m.StatusCode))
.WaitAndRetryAsync(
8,
(i, result, _) =>
@@ -43,24 +56,4 @@ public static class Http
},
(_, _, _, _) => Task.CompletedTask
);
private static HttpStatusCode? TryGetStatusCodeFromException(HttpRequestException ex) =>
// This is extremely frail, but there's no other way
Regex
.Match(ex.Message, @": (\d+) \(")
.Groups[1]
.Value
.NullIfWhiteSpace()?
.Pipe(s => (HttpStatusCode) int.Parse(s, CultureInfo.InvariantCulture));
public static IAsyncPolicy ExceptionPolicy { get; } =
Policy
.Handle<IOException>() // dangerous
.Or<HttpRequestException>(ex =>
TryGetStatusCodeFromException(ex) is
HttpStatusCode.TooManyRequests or
HttpStatusCode.RequestTimeout or
HttpStatusCode.InternalServerError
)
.WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1));
}

Some files were not shown because too many files have changed in this diff Show More