Compare commits

...

133 Commits

Author SHA1 Message Date
Tyrrrz cba5a934df Update version 2021-06-19 20:50:22 +03:00
Tyrrrz cd05058cf7 Prefer proxy URLs for embed content everywhere 2021-06-19 20:25:50 +03:00
Tyrrrz e9c3872701 [HTML] Prevent click events on elements inside referenced message
Closes #545
2021-06-19 20:17:36 +03:00
Tyrrrz 88c4330a66 [HTML] Link to pre-resized avatar image for better image quality 2021-06-19 20:15:18 +03:00
Tyrrrz 998bdaf2d4 [HTML] Use svg variants of Twemoji for better resize quality 2021-06-19 19:57:29 +03:00
Tyrrrz 24a80f915f [HTML] Recognize standard emoji by code and show emoji code in tooltips
Closes #549
Closes #599
2021-06-19 19:49:53 +03:00
Tyrrrz de57cd714d [HTML] Prepend "Message sent: " to message timestamp tooltips 2021-06-19 18:56:47 +03:00
Tyrrrz f9ab269b29 [HTML] Don't jumbo emojis in referenced messages
Closes #544
2021-06-19 15:49:16 +03:00
Tyrrrz 33feaccee7 Only use DotnetRuntimeBootstrapper in release build 2021-06-17 22:08:11 +03:00
Tyrrrz 4fd6f8272e [HTML] Update styling to match Discord 2021-06-16 18:15:58 +03:00
Tyrrrz b28f029382 Refactor YouTube embeds 2021-06-16 17:54:35 +03:00
Tyrrrz 29552be6e5 Cleanup 2021-06-16 16:52:18 +03:00
Tyrrrz 9491e18e2f Update NuGet packages 2021-06-15 22:41:06 +03:00
quentinmay 4b1c9660be Render YouTube iframe for applicable video embeds (#607) 2021-06-15 07:19:19 -07:00
Tyrrrz 950625e671 [HTML] Show timestamps of individual messages
Closes #536
2021-06-14 22:55:42 +03:00
Tyrrrz ce2dedff6b [HTML] Use a clickable <div> instead of <a> for reply reference link
Fixes #546
2021-06-14 21:33:20 +03:00
Tyrrrz baac419538 [HTML] Fix embed footer styling 2021-06-14 20:48:20 +03:00
Tyrrrz bd40dbde48 Don't throw if file dates cannot be modified when downloading media
Fixes #585
2021-06-14 20:43:44 +03:00
Tyrrrz cbe4bc593c [HTML] Prefer proxy URLs for resolving embed media
Closes #590
2021-06-14 20:38:19 +03:00
Tyrrrz cb32cee5f5 Update DotnetRuntimeBootstrapper 2021-06-14 04:51:32 +03:00
Tyrrrz 82f364a887 Update readme 2021-06-08 18:04:19 +03:00
Tyrrrz 152e8c8dcb Update version 2021-06-08 17:54:58 +03:00
John Zabroski 260139ab25 Fix .NET v3.1 to say .NET Core v3.1 in readme (#605) 2021-06-08 07:25:12 -07:00
Tyrrrz 11e9c0f15c Downgrade MDIX 2021-06-08 01:32:21 +03:00
Tyrrrz ac809d1a3f Update packages 2021-06-08 01:23:00 +03:00
Tyrrrz 36fa346299 Use DotnetRuntimeBootstrapper 2021-06-08 00:53:41 +03:00
Tyrrrz 8c3a5bd096 Update issue templates 2021-05-12 21:38:12 +03:00
Tyrrrz 021682899a [CLI] Move Markup.Escape(...) to ConsoleExtensions 2021-04-26 19:47:25 +03:00
Lucas LaBuff 0ec6027a93 Escape brackets in category name in CLI (#558) 2021-04-26 09:37:37 -07:00
Tyrrrz eb49d0bd16 Update version 2021-04-24 21:26:49 +03:00
Tyrrrz e7c2564f6f Update NuGet packages 2021-04-24 21:08:57 +03:00
Tyrrrz d1b70cf5f6 Add wiki to issue templates 2021-04-22 21:48:45 +03:00
Tyrrrz b73d6a78a9 Fix issue templates 2021-04-22 21:47:05 +03:00
Tyrrrz b9f04bc4d9 Use issue forms 2021-04-22 21:40:47 +03:00
Tyrrrz d01ed13041 Update NuGet packages 2021-04-18 19:32:59 +03:00
Tyrrrz ed35a36eaa Update NuGet.config 2021-04-17 21:25:52 +03:00
Tyrrrz 2ddfb46c31 Add IFormatProvider to FileSize.TryParse(...) and PartitionLimit.Parse(...) 2021-04-17 14:07:07 +03:00
rtm516 3c5beeba79 Export channel category id to JSON (#554) 2021-04-17 13:56:27 +03:00
Tyrrrz ed915443e6 Update contribution guidelines 2021-04-17 00:21:23 +03:00
Tyrrrz f8d55f406c [GUI] Cleanup ExportSetupViewModel & RootViewModel 2021-04-16 23:39:36 +03:00
Tyrrrz 15b605f5a7 Fix typo 2021-04-16 23:29:19 +03:00
Tyrrrz 511af1e35c Refactor 2021-04-16 23:09:08 +03:00
rtm516 2fea455c64 [JSON] Write user color (#552) 2021-04-15 19:21:16 +03:00
Tyrrrz c64133f717 Add contribution guidelines 2021-04-14 21:42:41 +03:00
Alexey Golub d18d2e0307 Update readme 2021-04-14 15:04:42 +03:00
Alexey Golub 86633b858c Update readme
Fixes #551
2021-04-14 15:02:06 +03:00
Tyrrrz c390c64316 Remove GHA workarounds 2021-04-13 22:33:06 +03:00
Tyrrrz af3a59ce0f Add NuGet.config 2021-04-13 22:23:14 +03:00
Andrew Kolos eb89ea5b40 Add partition by file size (#497) 2021-04-12 13:50:32 +03:00
Tyrrrz ad3655396f [CLI] Remove redundant whitespace 2021-03-25 00:44:21 +02:00
Tyrrrz 1b8af364d3 [CLI] Cleanup 2021-03-24 22:39:27 +02:00
Tyrrrz 58464d8708 Improve error messages 2021-03-24 22:39:01 +02:00
Tyrrrz 026927265a [CLI] Remove ExportMultipleCommandBase and allow multiple channel IDs in ExportChannelCommand 2021-03-24 03:04:46 +02:00
Tyrrrz 390735032e [CLI] Hack Spectre.Console so that it properly supports CliFx encoding 2021-03-24 02:51:11 +02:00
Tyrrrz 461d474498 [CLI] Update CliFx 2021-03-24 02:50:54 +02:00
Tyrrrz 017ed5ae6d [CLI] Update CliFx and use Spectre.Console for progress reporting 2021-03-23 22:38:44 +02:00
Tyrrrz 6f90c367b9 Clamp progress in DiscordClient.GetMessagesAsync(...) 2021-03-23 21:41:37 +02:00
Tyrrrz f866c16355 Update NuGet packages 2021-03-23 18:43:09 +02:00
Tyrrrz 072d4c1fe6 Use MiniRazor.CodeGen instead of MiniRazor 2021-03-23 00:48:56 +02:00
Lucas LaBuff 6365e0a080 Fix JSON discriminator and nickname fields (#524) 2021-03-19 16:58:28 +02:00
Tyrrrz 7b140fd450 Try to fix nullability warning 2021-03-18 02:13:15 +02:00
Tyrrrz e237e00600 Fix CI/CD Windows issue 2021-03-16 01:24:00 +02:00
Tyrrrz 3aef7faa1e Use is null/is not null 2021-03-09 21:57:26 +02:00
Tyrrrz ebe4d58a42 Cleanup 2021-02-22 03:15:22 +02:00
Lucas LaBuff bed0ade732 [JSON] Skip validation (#499) 2021-02-15 01:07:28 +02:00
Alexey Golub 06e89538cb Drop xs:code 2021-02-10 08:28:27 +02:00
Alexey Golub 7903869f9f Update Readme.md 2021-02-10 02:44:55 +02:00
Tyrrrz 32443c5a5b Update readme 2021-02-10 02:26:58 +02:00
Tyrrrz d477159907 Update readme 2021-02-10 02:05:08 +02:00
Tyrrrz 93578abe24 Normalize snackbar style 2021-02-09 00:50:18 +02:00
Tyrrrz f22b048d32 Update version 2021-02-06 21:22:42 +02:00
Lucas LaBuff a8031ad3aa Fix crash when parsing position on DM channels (#496) 2021-02-06 17:00:29 +02:00
Frederic Portaria-Janicki e49cf997ea [CLI] Fix missing channel category bug when API returns empty string (#492) 2021-02-02 14:42:04 +02:00
Lucas LaBuff 8b9afe45b9 [CLI] Fix sorting by channel/category position (#490) 2021-01-31 17:23:46 +02:00
Lucas LaBuff 77b7977324 Formatting output paths (#472) 2021-01-28 22:01:13 +02:00
Lucas LaBuff 915f4c8d9f Fix crash on encountering invalid snowflake (#479) 2021-01-18 19:21:22 +02:00
Tyrrrz 0872d6d44b Update screenshots 2020-12-29 21:06:45 +02:00
Tyrrrz 3aa90bb27b Update version 2020-12-29 21:02:46 +02:00
Tyrrrz 51acffc92e [HTML] Fix up replies 2020-12-29 20:49:34 +02:00
Tyrrrz 39a33538fe Cleanup last PR 2020-12-28 22:10:46 +02:00
Sanqui 07ac805677 Add support for replies (#455) 2020-12-28 22:03:30 +02:00
Alexey Golub 097a75e326 Update readme 2020-12-28 18:38:22 +02:00
Tyrrrz 3d9ee3b339 Embrace Snowflake as first class citizen 2020-12-27 19:42:21 +02:00
multisignature 4ff7990967 Fix grammar in readme (#466) 2020-12-23 21:00:33 +02:00
Andrew Kolos 730fed075f [CLI] Allow passing after and before message ids instead of dates (#462) 2020-12-22 17:41:40 +02:00
Alexey Golub 600c354ab2 Use Discord API v8 2020-12-19 18:37:38 +02:00
Alexey Golub d8dc12b3f7 Update readme 2020-12-19 18:22:03 +02:00
Alexey Golub 33a6d49d76 Use .NET 5.0 SDK in Docker 2020-12-19 18:11:31 +02:00
Tyrrrz 3861dba60c Refactor last PR 2020-12-17 17:49:35 +02:00
Andrew Kolos 69acb5bf76 Set last write time on downloaded media to the value provided in the Last-Modified header (#459) 2020-12-17 17:13:47 +02:00
Tyrrrz 0e67dcbf2e Update readme 2020-12-15 17:47:00 +02:00
Tyrrrz 45d2be549d Install dotnet 5 on CI/CD so that source generators work 2020-12-15 16:17:37 +02:00
Tyrrrz 9394824943 Update MiniRazor to use build-time template compilation 2020-12-15 03:44:13 +02:00
Sanqui 43c50324d8 [JSON] Don't escape Unicode characters when possible (#453)
While UnsafeRelaxedJsonEscaping sounds scary, there is no risk
here since the JSON data is saved directly into a file.  See #450
2020-12-14 21:57:48 +02:00
Alexey Golub a6353de2a7 Use preview language version 2020-12-14 18:48:34 +02:00
Sanqui d020652f50 [CLI] Display channel name for errors during ExportMultipleAsync (#452) 2020-12-14 18:48:11 +02:00
Tyrrrz 63803f98aa Use C#9 features 2020-12-14 17:45:42 +02:00
Tyrrrz 9dda9cfc27 Use floating dotnet versions on CI/CD 2020-12-14 17:37:49 +02:00
Tyrrrz 61d0ddeced Update project structure 2020-12-10 16:52:05 +02:00
Tyrrrz 4c64977b71 Expect null result in WaitAndRetry logic in http retry handler
Fixes #447
2020-12-07 16:24:33 +02:00
Tyrrrz 8bc5b547ef Remove gitattributes and simplify gitignore 2020-12-07 16:23:53 +02:00
Lucas LaBuff 286981d9f8 [HTML] Prettier attachment preview for non-media files (#448) 2020-12-07 16:20:38 +02:00
Tyrrrz 5a62ce6af9 Update JsonExtensions to 1.0.1 2020-11-29 00:42:05 +02:00
Tyrrrz d72fe594b4 Use JsonExtensions package 2020-11-29 00:17:58 +02:00
Tyrrrz 9df98b0405 Update version 2020-11-23 17:04:08 +02:00
Tyrrrz 0d2b9e8bc0 Refactor last PR 2020-11-23 16:53:22 +02:00
Andrew Kolos 53cebd82eb [CLI] Print error and exit if reuse-media option is used without media option (#433) 2020-11-22 20:45:19 +02:00
Tyrrrz 37fcee2f21 Fix relative path extraction in ExportContext when base directory is empty
Fixes #437
Fixes #434
2020-11-20 16:56:30 +02:00
Tyrrrz 047dccef7f Refactor last PR 2020-11-18 19:04:53 +02:00
sas41 cb47902d10 [HTML] Rich rendering support for audio and video embeds (#432) 2020-11-16 15:49:57 +02:00
Alexey Golub dca8b8ceb2 Update .NET runtime links 2020-11-11 15:23:43 +02:00
Tyrrrz a8e7c85b99 Update docker image references 2020-11-11 01:56:53 +02:00
Tyrrrz edd51e0838 Revert last change, .net5 doesn't work 2020-11-11 00:02:56 +02:00
Tyrrrz 5e04accdff Update build infra 2020-11-10 23:58:05 +02:00
Andrew Kolos 547bf0a463 [JSON] Add core user mentions to output (#416) 2020-10-25 23:00:30 +02:00
Tyrrrz 948fab7c86 Update readme 2020-10-25 01:53:32 +03:00
Tyrrrz 5038b8541e Update version 2020-10-24 21:26:34 +03:00
Tyrrrz 1da80956dd Refactor 2020-10-24 21:15:58 +03:00
Tyrrrz 0763a99765 [JSON] Fix incorrect embed color output 2020-10-24 17:45:54 +03:00
Tyrrrz eda5b21684 Update github actions 2020-10-24 00:02:32 +03:00
Tyrrrz 2c7b054c24 Update nuget packages 2020-10-24 00:02:32 +03:00
Andrew Kolos 520e023aff Stop redundantly downloading media when re-exporting (#395) 2020-10-23 16:38:15 +03:00
Alexey Golub 949c9d3f1e Update readme 2020-10-17 20:25:21 +03:00
Andrew Kolos fa825fce47 [GUI] Fix content bg color on first screen to match the second screen (#411) 2020-10-16 14:20:58 +03:00
Andrew Kolos 09acfcff59 Add retry policy to MediaDownloader to accommodate short network outages during download (#404) 2020-10-12 23:03:14 +03:00
Andrew Kolos ddb4856516 Add "unix" and "unixms" options for date format (#405) 2020-10-12 13:41:49 +03:00
Andrew Kolos c4007d4a80 [JSON] Add embed color (#403) 2020-10-10 16:10:56 +03:00
Alexey Golub 2a432c0ed2 Treat nullability warnings as errors 2020-10-07 15:33:59 +03:00
Alexey Golub c5eeb5f464 Update nuget packages 2020-10-07 15:33:47 +03:00
Alexey Golub a6bf546e79 [GUI] Clean up the UI 2020-10-05 17:41:09 +03:00
Alexey Golub 239c47c06e [GUI] Clean up theme management 2020-10-05 17:10:00 +03:00
Andrew Kolos 19f678ca01 [GUI] Add dark theme (#388) 2020-10-05 16:31:37 +03:00
Pegasis 08442f21b7 Add AUR package to README (#379) 2020-09-20 15:52:44 +03:00
159 changed files with 11925 additions and 2001 deletions
-63
View File
@@ -1,63 +0,0 @@
###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
* text=auto
###############################################################################
# Set default behavior for command prompt diff.
#
# This is need for earlier builds of msysgit that does not have it on by
# default for csharp files.
# Note: This is only used by command line
###############################################################################
#*.cs diff=csharp
###############################################################################
# Set the merge driver for project and solution files
#
# Merging from the command prompt will add diff markers to the files if there
# are conflicts (Merging from VS is not affected by the settings below, in VS
# the diff markers are never inserted). Diff markers may cause the following
# file extensions to fail to load in VS. An alternative would be to treat
# these files as binary and thus will always conflict and require user
# intervention with every merge. To do so, just uncomment the entries below
###############################################################################
#*.sln merge=binary
#*.csproj merge=binary
#*.vbproj merge=binary
#*.vcxproj merge=binary
#*.vcproj merge=binary
#*.dbproj merge=binary
#*.fsproj merge=binary
#*.lsproj merge=binary
#*.wixproj merge=binary
#*.modelproj merge=binary
#*.sqlproj merge=binary
#*.wwaproj merge=binary
###############################################################################
# behavior for image files
#
# image files are treated as binary by default.
###############################################################################
#*.jpg binary
#*.png binary
#*.gif binary
###############################################################################
# diff behavior for common document formats
#
# Convert binary document formats to text before diffing them. This feature
# is only available from the command line. Turn it on by uncommenting the
# entries below.
###############################################################################
#*.doc diff=astextplain
#*.DOC diff=astextplain
#*.docx diff=astextplain
#*.DOCX diff=astextplain
#*.dot diff=astextplain
#*.DOT diff=astextplain
#*.pdf diff=astextplain
#*.PDF diff=astextplain
#*.rtf diff=astextplain
#*.RTF diff=astextplain
+55
View File
@@ -0,0 +1,55 @@
name: "\U0001F41E Bug report"
description: Report broken functionality.
labels: [bug]
body:
- type: markdown
attributes:
value: |
Important:
- Please check existing issues (both opened and closed) to ensure that this bug hasn't been reported before.
- Refer to the [contribution guidelines](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/Contributing.md) for info on how to file a good issue.
- If you want to ask a question instead of reporting a bug, please use [discussions](https://github.com/Tyrrrz/DiscordChatExporter/discussions/new) instead.
- type: input
attributes:
label: Version
description: "Which version(s) of DiscordChatExporter does this bug affect?"
validations:
required: true
- type: dropdown
attributes:
label: Flavor
description: "Which flavor(s) of DiscordChatExporter does this bug affect?"
multiple: true
options:
- GUI (Graphical User Interface)
- CLI (Command Line Interface)
validations:
required: true
- type: dropdown
attributes:
label: Export format
description: "Which export format(s) does this bug affect? If this bug is not related to any specific export format, leave this blank."
multiple: true
options:
- HTML
- TXT
- JSON
- CSV
- type: textarea
attributes:
label: Details
description: "Clear and thorough explanation of the bug. If relevant, include screenshots or screen recordings."
validations:
required: true
- type: textarea
attributes:
label: Steps to reproduce
description: "Minimum steps required to reproduce the bug."
validations:
required: true
+8
View File
@@ -0,0 +1,8 @@
blank_issues_enabled: false
contact_links:
- name: "\U0001F4D6 Read the wiki"
url: https://github.com/Tyrrrz/DiscordChatExporter/wiki
about: Find answers to most common questions.
- name: "\U0001F5E8 Ask a question"
url: https://github.com/Tyrrrz/DiscordChatExporter/discussions/new
about: Please ask and answer questions here.
@@ -0,0 +1,39 @@
name: "\U00002728 Feature request"
description: Request a new feature.
labels: [enhancement]
body:
- type: markdown
attributes:
value: |
Important:
- Please check existing issues (both opened and closed) to ensure that this feature hasn't been requested before.
- Refer to the [contribution guidelines](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/Contributing.md) for info on how to file a good issue.
- If you want to ask a question instead of requesting a feature, please use [discussions](https://github.com/Tyrrrz/DiscordChatExporter/discussions/new) instead.
- type: dropdown
attributes:
label: Flavor
description: "Which flavor(s) of DiscordChatExporter does this feature request apply to? If this feature is not related to any specific flavor, leave this blank."
multiple: true
options:
- GUI (Graphical User Interface)
- CLI (Command Line Interface)
- type: dropdown
attributes:
label: Export format
description: "Which export format(s) does this feature request apply to? If this feature is not related to any specific export format, leave this blank."
multiple: true
options:
- HTML
- TXT
- JSON
- CSV
- type: textarea
attributes:
label: Details
description: "Clear and thorough explanation of the feature you have in mind. If relevant, include screenshots or screen recordings."
validations:
required: true
+52 -47
View File
@@ -3,64 +3,69 @@ name: CD
on:
push:
tags:
- '*'
- "*"
jobs:
build:
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Checkout
uses: actions/checkout@v2.3.3
- name: Install .NET Core
uses: actions/setup-dotnet@v1.4.0
with:
dotnet-version: 3.1.100
- name: Install .NET (v3.1)
uses: actions/setup-dotnet@v1.7.2
with:
dotnet-version: 3.1.x
- name: Build & publish (CLI)
run: dotnet publish DiscordChatExporter.Cli/ -o DiscordChatExporter.Cli/bin/Publish/ --configuration Release
- name: Install .NET (v5.0)
uses: actions/setup-dotnet@v1.7.2
with:
dotnet-version: 5.0.x
- name: Build & publish (GUI)
run: dotnet publish DiscordChatExporter.Gui/ -o DiscordChatExporter.Gui/bin/Publish/ --configuration Release
- name: Build & publish (CLI)
run: dotnet publish DiscordChatExporter.Cli/ -o DiscordChatExporter.Cli/bin/Publish/ --configuration Release
- name: Pack (CLI)
run: Compress-Archive -Path DiscordChatExporter.Cli/bin/Publish/* -DestinationPath DiscordChatExporter.Cli/bin/Publish/Archive.zip -Force
shell: pwsh
- name: Build & publish (GUI)
run: dotnet publish DiscordChatExporter.Gui/ -o DiscordChatExporter.Gui/bin/Publish/ --configuration Release
- name: Pack (GUI)
run: Compress-Archive -Path DiscordChatExporter.Gui/bin/Publish/* -DestinationPath DiscordChatExporter.Gui/bin/Publish/Archive.zip -Force
shell: pwsh
- name: Pack (CLI)
run: Compress-Archive -Path DiscordChatExporter.Cli/bin/Publish/* -DestinationPath DiscordChatExporter.Cli/bin/Publish/Archive.zip -Force
shell: pwsh
- name: Create release
id: create_release
uses: actions/create-release@v1.0.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: ${{ github.ref }}
body: |
[Changelog](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/Changelog.md)
draft: false
prerelease: false
- name: Pack (GUI)
run: Compress-Archive -Path DiscordChatExporter.Gui/bin/Publish/* -DestinationPath DiscordChatExporter.Gui/bin/Publish/Archive.zip -Force
shell: pwsh
- name: Upload release asset (CLI)
uses: actions/upload-release-asset@v1.0.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: DiscordChatExporter.Cli/bin/Publish/Archive.zip
asset_name: DiscordChatExporter.CLI.zip
asset_content_type: application/zip
- name: Create release
id: create_release
uses: actions/create-release@v1.0.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: ${{ github.ref }}
body: |
[Changelog](https://github.com/Tyrrrz/DiscordChatExporter/blob/master/Changelog.md)
draft: false
prerelease: false
- name: Upload release asset (GUI)
uses: actions/upload-release-asset@v1.0.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: DiscordChatExporter.Gui/bin/Publish/Archive.zip
asset_name: DiscordChatExporter.zip
asset_content_type: application/zip
- name: Upload release asset (CLI)
uses: actions/upload-release-asset@v1.0.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: DiscordChatExporter.Cli/bin/Publish/Archive.zip
asset_name: DiscordChatExporter.CLI.zip
asset_content_type: application/zip
- name: Upload release asset (GUI)
uses: actions/upload-release-asset@v1.0.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: DiscordChatExporter.Gui/bin/Publish/Archive.zip
asset_name: DiscordChatExporter.zip
asset_content_type: application/zip
+25 -20
View File
@@ -7,28 +7,33 @@ jobs:
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Checkout
uses: actions/checkout@v2.3.3
- name: Install .NET Core
uses: actions/setup-dotnet@v1.4.0
with:
dotnet-version: 3.1.100
- name: Install .NET (v3.1)
uses: actions/setup-dotnet@v1.7.2
with:
dotnet-version: 3.1.x
- name: Build & publish (CLI)
run: dotnet publish DiscordChatExporter.Cli/ -o DiscordChatExporter.Cli/bin/Publish/ --configuration Release
- name: Install .NET (v5.0)
uses: actions/setup-dotnet@v1.7.2
with:
dotnet-version: 5.0.x
- name: Build & publish (GUI)
run: dotnet publish DiscordChatExporter.Gui/ -o DiscordChatExporter.Gui/bin/Publish/ --configuration Release
- name: Build & publish (CLI)
run: dotnet publish DiscordChatExporter.Cli/ -o DiscordChatExporter.Cli/bin/Publish/ --configuration Release
- name: Upload build artifacts (CLI)
uses: actions/upload-artifact@v1
with:
name: DiscordChatExporter.CLI
path: DiscordChatExporter.Cli/bin/Publish/
- name: Build & publish (GUI)
run: dotnet publish DiscordChatExporter.Gui/ -o DiscordChatExporter.Gui/bin/Publish/ --configuration Release
- name: Upload build artifact (GUI)
uses: actions/upload-artifact@v1
with:
name: DiscordChatExporter
path: DiscordChatExporter.Gui/bin/Publish/
- name: Upload build artifacts (CLI)
uses: actions/upload-artifact@v1
with:
name: DiscordChatExporter.CLI
path: DiscordChatExporter.Cli/bin/Publish/
- name: Upload build artifact (GUI)
uses: actions/upload-artifact@v1
with:
name: DiscordChatExporter
path: DiscordChatExporter.Gui/bin/Publish/
+7 -246
View File
@@ -1,261 +1,22 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
.idea/
.vs/
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
[Xx]64/
[Xx]86/
[Bb]uild/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# DNX
project.lock.json
project.fragment.lock.json
artifacts/
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Comment the next line if you want to checkin your web deploy settings
# but database connection strings (with potential passwords) will be unencrypted
#*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignoreable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
# CodeRush
.cr/
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Coverage
*.opencover.xml
Binary file not shown.

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 33 KiB

+69
View File
@@ -1,3 +1,72 @@
### v2.28 (19-Jun-2021)
- [HTML] Added special casing for YouTube video embeds to render them directly using an embedded player. (Thanks [@quentinmay](https://github.com/quentinmay))
- [HTML] Added support for rendering standard emoji by code. Such emoji may sometimes be found in messages sent by bots or through webhooks. (Thanks [@CanePlayz](https://github.com/CanePlayz) and [@Lucas LaBuff](https://github.com/96-LB))
- [HTML] Changed tooltips on standard emoji to show emoji code instead of their raw string representation. (Thanks [@CanePlayz](https://github.com/CanePlayz) and [@Lucas LaBuff](https://github.com/96-LB))
- [HTML] Added tooltips on individual messages to show when those messages were sent.
- [HTML] Updated colors and minor styling elements to match Discord's new style direction after rebranding.
- [HTML] Updated Twemoji URLs to use SVG image variants for better rendering quality.
- [HTML] Changed user avatar URLs to include predetermined size query parameter for better rendering quality.
- Changed embed rendering logic to prefer Discord-proxied URLs for external content, in order to avoid unnecessary HTTP requests to third parties.
- Fixed an issue where exporting with media sometimes resulted in a crash due to lack of permissions to change file attributes. These errors are now ignored, which means that the creation, write, and access dates of downloaded files may sometimes not match the dates provided by Discord's CDN.
- [HTML] Fixed styling issues when rendering embed footers.
- [HTML] Fixed an issue where links didn't render properly inside referenced messages.
- [HTML] Fixed an issue where standalone emoji were incorrectly enlarged inside referenced messages.
- [HTML] Fixed an issue where clicking on a referenced message revealed spoilers inside of it.
- [GUI] Fixed an issue where the list of missing components, reported by the prerequisite check at application startup, included Windows updates that were not applicable to the user's system due to the fact that other superseding updates have already been installed. This issue only affected users running DiscordChatExporter on Windows 7.
- [GUI] Fixed a few other minor issues related to the prerequisite installation window.
### v2.27.1 (08-Jun-2021)
- [GUI] Application will now detect if the required .NET Runtime or any of its prerequisites are missing and prompt the user to download and install them automatically. **Experimental feature, please test it out and report any issues you may find!**
- [CLI] Fixed an issue where the application crashed when exporting a channel that had square brackets as part of its name or category name. (Thanks [@Lucas LaBuff](https://github.com/96-LB))
### v2.27 (24-Apr-2021)
- Added partitioning by file size. You can now use values such as `10mb` to indicate a size-based cut off point, in addition to values like `10` to indicate a number of messages. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
- [CLI] Improved formatting and progress reporting.
- [CLI] Changed `export` command so that it can accept multiple channel IDs passed to `-c|--channel` option. This makes it possible to export multiple channels at once while benefiting from parallelization.
- [JSON] Added `color` field to message author. This is a hex string that represents user's color, inherited from their roles. (Thanks [@rtm516](https://github.com/rtm516))
- [JSON] Added `categoryId` field to the root. This represents the ID of the category channel that the currently exported channel belongs to. (Thanks [@rtm516](https://github.com/rtm516))
- [JSON] Fixed an issue where user discriminator was written without leading zeroes. (Thanks [@Lucas LaBuff](https://github.com/96-LB))
- [JSON] Fixed an issue where the application sometimes crashed with an error saying `']' invalid without a matching open`, which was just a validation message that masked the actual error behind the failure. (Thanks [@Lucas LaBuff](https://github.com/96-LB))
- Fixed an issue where progress was not reported correctly if the channel didn't have any messages in the specified time range.
### v2.26.1 (06-Feb-2021)
- [CLI] Added support for file name templates, which allow you to dynamically generate output file names based on channel and guild metadata. (Thanks [@Lucas LaBuff](https://github.com/96-LB))
- Fixed an issue where the application sometimes crashed with `Invalid snowflake` message when exporting chat logs that contained invalid or outdated mentions. (Thanks [@Lucas LaBuff](https://github.com/96-LB))
### v2.26 (29-Dec-2020)
- [HTML] Added support for replies. (Thanks [@Sanqui](https://github.com/Sanqui))
- [HTML] Improved styling of non-media attachments (regular files, i.e. not video, audio, or image). (Thanks [@Lucas LaBuff](https://github.com/96-LB))
- [HTML] Improved initial export performance.
- [JSON] Relaxed escaping rules for unicode characters. (Thanks [@Sanqui](https://github.com/Sanqui))
- [JSON] Added support for replies. (Thanks [@Sanqui](https://github.com/Sanqui))
- [CLI] Added channel names to errors shown on failures when exporting multiple channels. (Thanks [@Sanqui](https://github.com/Sanqui))
- [CLI] Added support for passing message IDs (snowflakes) to `--after` and `--before` options. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
- Files downloaded by the "download referenced media" option now have their dates set according to the `Last-Modified` header provided by Discord CDN. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
- Fixed an issue where the application sometimes crashed with `NullReferenceException` when retrying a failed HTTP request.
- Migrated to Discord API v8.
### v2.25 (23-Nov-2020)
- [JSON] Added `mentions` property to the output. This contains users referenced inside the message text, but doesn't include mentions in embeds. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
- [HTML] Added better previews for attached video and audio. They are now rendered using a browser native player element. The actual file path can still be retrieved by right clicking and selecting "Copy file path". (Thanks [@sas41](https://github.com/sas41))
- [CLI] Fixed an issue where using `--media` option without providing a rooted path via `--output` option triggered an error.
- [CLI] Fixed an issue where using `--reuse-media` option without also providing `--media` option was accepted, but didn't actually do anything. Now this produces an error instead. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
### v2.24 (24-Oct-2020)
- [CLI] Updated to [CliFx v1.5](https://github.com/Tyrrrz/CliFx), which brings some UX improvements.
- [GUI] Added dark mode, which can be enabled in settings. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
- [GUI] Improved the UX in a few minor places.
- [JSON] Added `embed.color` property to the output. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
- Added additional date formats: `unix` to write dates as a unix timestamp and `unixms` for the milliseconds version. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
- Added retry policy for downloading media content. This is mostly a workaround for transient failures that sometimes may occur when downloading assets referenced from embeds. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
- Added an option to reuse already downloaded media to avoid redundant network requests. It can be enabled in settings in GUI, or by passing `--reuse-media` in CLI. To facilitate this feature, the file name strategy has been changed as well -- now instead of relying on the original file name and adding `(n)` to the end in case of duplicates, a URL hash is injected in every file name. This also means that when the "reuse media" option is disabled, existing files inside the media directory may be overwritten. (Thanks [@Andrew Kolos](https://github.com/andrewkolos))
### v2.23 (14-Sep-2020)
- [CLI] Added a command to export all channels across all servers. Use `exportall` to do it. (Thanks [@wyattscarpenter](https://github.com/wyattscarpenter))
+56
View File
@@ -0,0 +1,56 @@
# Contributing
DiscordChatExporter accepts contributions in the form of issues and pull requests.
## Creating issues
If you have a feature suggestion or want to report a bug, you are welcome to create an issue.
Guidelines:
- Avoid creating an issue if a similar one already exists. Look through existing open and closed issues first.
- Keep your issue focused on one specific problem. If you have multiple suggestions or bug reports, please create separate issues for them.
- Provide a descriptive title for your issue. Don't use generic titles like "A couple suggestions" or "Not working".
- Provide more context in the body of the issue. If relevant, attach screenshots or screen recordings.
- Remain civil and respectful when participating in discussions.
## Creating pull requests
If you want to contribute code to the project, you can do so by creating a pull request.
Guidelines:
- Make sure that there is an existing issue that describes the problem solved by your pull request. If there isn't one, please create it first.
- Add a comment to the issue indicating that you want to submit a pull request for it. This can be a good opportunity to discuss possible solutions, finalize requirements, or ask questions.
- Keep your pull request focused and as small as possible. If you want to contribute multiple unrelated changes, please create separate pull requests for them.
- Follow the coding style and conventions already established by the project. When in doubt which style to use, ask in comments to your pull request.
- If you want to start a discussion regarding a specific change you made, add a review comment to your own code. This can be used to highlight something important or to seek input from others.
## Building the project locally
Prerequisites:
- [.NET 5.0 SDK](https://dotnet.microsoft.com/download/dotnet/5.0)
- [.NET 3.1 SDK](https://dotnet.microsoft.com/download/dotnet/3.1) (temporarily as the app still targets .NET 3.1, but requires .NET 5.0 to build)
- _(Optional)_ C#/.NET IDE, such as [JetBrains Rider](https://www.jetbrains.com/rider), [VS Code](https://code.visualstudio.com/docs/languages/csharp), or [Visual Studio](https://visualstudio.microsoft.com).
To build the entire solution run the following command in the root of the repository:
```sh
> dotnet build
```
This will generate runtime artifacts for each project:
```plaintext
./DiscordChatExporter.Gui/bin/[Debug|Release]/[runtime]/*
./DiscordChatExporter.Cli/bin/[Debug|Release]/[runtime]/*
```
You can also build and run a specific project directly.
To do that, navigate to its directory and use `dotnet run`:
```sh
> cd DiscordChatExporter.Gui
> dotnet run
```
@@ -2,11 +2,12 @@
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<Version>2.23</Version>
<Version>2.28</Version>
<Company>Tyrrrz</Company>
<Copyright>Copyright (c) Alexey Golub</Copyright>
<LangVersion>latest</LangVersion>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<WarningsAsErrors>nullable</WarningsAsErrors>
</PropertyGroup>
</Project>
@@ -1,72 +1,145 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Utilities;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting;
using CliFx.Exceptions;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Utils.Extensions;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions;
using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Exporting.Partitioning;
using DiscordChatExporter.Core.Utils.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Cli.Commands.Base
{
public abstract class ExportCommandBase : TokenCommandBase
{
[CommandOption("output", 'o', Description = "Output file or directory path.")]
public string OutputPath { get; set; } = Directory.GetCurrentDirectory();
public string OutputPath { get; init; } = Directory.GetCurrentDirectory();
[CommandOption("format", 'f', Description = "Output file format.")]
public ExportFormat ExportFormat { get; set; } = ExportFormat.HtmlDark;
[CommandOption("format", 'f', Description = "Export format.")]
public ExportFormat ExportFormat { get; init; } = ExportFormat.HtmlDark;
[CommandOption("after", Description = "Limit to messages sent after this date.")]
public DateTimeOffset? After { get; set; }
[CommandOption("after", Description = "Only include messages sent after this date or message ID.")]
public Snowflake? After { get; init; }
[CommandOption("before", Description = "Limit to messages sent before this date.")]
public DateTimeOffset? Before { get; set; }
[CommandOption("before", Description = "Only include messages sent before this date or message ID.")]
public Snowflake? Before { get; init; }
[CommandOption("partition", 'p', Description = "Split output into partitions limited to this number of messages.")]
public int? PartitionLimit { get; set; }
[CommandOption("partition", 'p', Description = "Split output into partitions, each limited to this number of messages (e.g. 100) or file size (e.g. 10mb).")]
public PartitionLimit PartitionLimit { get; init; } = NullPartitionLimit.Instance;
[CommandOption("parallel", Description = "Limits how many channels can be exported in parallel.")]
public int ParallelLimit { get; init; } = 1;
[CommandOption("media", Description = "Download referenced media content.")]
public bool ShouldDownloadMedia { get; set; }
public bool ShouldDownloadMedia { get; init; }
[CommandOption("dateformat", Description = "Date format used in output.")]
public string DateFormat { get; set; } = "dd-MMM-yy hh:mm tt";
[CommandOption("reuse-media", Description = "Reuse already existing media content to skip redundant downloads.")]
public bool ShouldReuseMedia { get; init; }
protected ChannelExporter GetChannelExporter() => new ChannelExporter(GetDiscordClient());
[CommandOption("dateformat", Description = "Format used when writing dates.")]
public string DateFormat { get; init; } = "dd-MMM-yy hh:mm tt";
protected async ValueTask ExportAsync(IConsole console, Guild guild, Channel channel)
private ChannelExporter? _channelExporter;
protected ChannelExporter Exporter => _channelExporter ??= new ChannelExporter(Discord);
protected async ValueTask ExportAsync(IConsole console, IReadOnlyList<Channel> channels)
{
console.Output.Write($"Exporting channel '{channel.Category} / {channel.Name}'... ");
var progress = console.CreateProgressTicker();
await console.Output.WriteLineAsync($"Exporting {channels.Count} channel(s)...");
var request = new ExportRequest(
guild,
channel,
OutputPath,
ExportFormat,
After,
Before,
PartitionLimit,
ShouldDownloadMedia,
DateFormat
);
var errors = new ConcurrentDictionary<Channel, string>();
await GetChannelExporter().ExportChannelAsync(request, progress);
// Wrap everything in a progress ticker
await console.CreateProgressTicker().StartAsync(async progressContext =>
{
await channels.ParallelForEachAsync(async channel =>
{
// Export
try
{
await progressContext.StartTaskAsync($"{channel.Category} / {channel.Name}", async progress =>
{
var guild = await Discord.GetGuildAsync(channel.GuildId);
console.Output.WriteLine();
console.Output.WriteLine("Done.");
var request = new ExportRequest(
guild,
channel,
OutputPath,
ExportFormat,
After,
Before,
PartitionLimit,
ShouldDownloadMedia,
ShouldReuseMedia,
DateFormat
);
await Exporter.ExportChannelAsync(request, progress);
});
}
catch (DiscordChatExporterException ex) when (!ex.IsCritical)
{
errors[channel] = ex.Message;
}
}, ParallelLimit.ClampMin(1));
});
// Print result
using (console.WithForegroundColor(ConsoleColor.White))
{
await console.Output.WriteLineAsync(
$"Successfully exported {channels.Count - errors.Count} channel(s)."
);
}
// Print errors
if (errors.Any())
{
await console.Output.WriteLineAsync();
using (console.WithForegroundColor(ConsoleColor.Red))
{
await console.Output.WriteLineAsync(
$"Failed to export {errors.Count} channel(s):"
);
}
foreach (var (channel, error) in errors)
{
await console.Output.WriteAsync($"{channel.Category} / {channel.Name}: ");
using (console.WithForegroundColor(ConsoleColor.Red))
await console.Output.WriteLineAsync(error);
}
await console.Output.WriteLineAsync();
}
// Fail the command if ALL channels failed to export.
// Having some of the channels fail to export is fine and expected.
if (errors.Count >= channels.Count)
{
throw new CommandException("Export failed.");
}
await console.Output.WriteLineAsync("Done.");
}
protected async ValueTask ExportAsync(IConsole console, Channel channel)
public override ValueTask ExecuteAsync(IConsole console)
{
var guild = await GetDiscordClient().GetGuildAsync(channel.GuildId);
await ExportAsync(console, guild, channel);
}
if (ShouldReuseMedia && !ShouldDownloadMedia)
{
throw new CommandException("Option --reuse-media cannot be used without --media.");
}
protected async ValueTask ExportAsync(IConsole console, string channelId)
{
var channel = await GetDiscordClient().GetChannelAsync(channelId);
await ExportAsync(console, channel);
return default;
}
}
}
@@ -1,76 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Utilities;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exceptions;
using DiscordChatExporter.Domain.Exporting;
using DiscordChatExporter.Domain.Utilities;
using Gress;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Cli.Commands.Base
{
public abstract class ExportMultipleCommandBase : ExportCommandBase
{
[CommandOption("parallel", Description = "Export this number of channels in parallel.")]
public int ParallelLimit { get; set; } = 1;
protected async ValueTask ExportMultipleAsync(IConsole console, IReadOnlyList<Channel> channels)
{
// HACK: this uses a separate route from ExportCommandBase because the progress ticker is not thread-safe
console.Output.Write($"Exporting {channels.Count} channels... ");
var progress = console.CreateProgressTicker();
var operations = progress.Wrap().CreateOperations(channels.Count);
var errors = new List<string>();
var successfulExportCount = 0;
await channels.Zip(operations).ParallelForEachAsync(async tuple =>
{
var (channel, operation) = tuple;
try
{
var guild = await GetDiscordClient().GetGuildAsync(channel.GuildId);
var request = new ExportRequest(
guild,
channel,
OutputPath,
ExportFormat,
After,
Before,
PartitionLimit,
ShouldDownloadMedia,
DateFormat
);
await GetChannelExporter().ExportChannelAsync(request, operation);
Interlocked.Increment(ref successfulExportCount);
}
catch (DiscordChatExporterException ex) when (!ex.IsCritical)
{
errors.Add(ex.Message);
}
finally
{
operation.Dispose();
}
}, ParallelLimit.ClampMin(1));
console.Output.WriteLine();
foreach (var error in errors)
console.Error.WriteLine(error);
console.Output.WriteLine($"Successfully exported {successfulExportCount} channel(s).");
}
}
}
@@ -1,23 +1,28 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Domain.Discord;
using CliFx.Infrastructure;
using DiscordChatExporter.Core.Discord;
namespace DiscordChatExporter.Cli.Commands.Base
{
public abstract class TokenCommandBase : ICommand
{
[CommandOption("token", 't', IsRequired = true, EnvironmentVariableName = "DISCORD_TOKEN",
Description = "Authorization token.")]
public string TokenValue { get; set; } = "";
[CommandOption("token", 't', IsRequired = true, EnvironmentVariable = "DISCORD_TOKEN", Description = "Authentication token.")]
public string TokenValue { get; init; } = "";
[CommandOption("bot", 'b', EnvironmentVariableName = "DISCORD_TOKEN_BOT",
Description = "Whether this authorization token belongs to a bot.")]
public bool IsBotToken { get; set; }
[CommandOption("bot", 'b', EnvironmentVariable = "DISCORD_TOKEN_BOT", Description = "Authenticate as a bot.")]
public bool IsBotToken { get; init; }
protected AuthToken GetAuthToken() => new AuthToken(IsBotToken ? AuthTokenType.Bot : AuthTokenType.User, TokenValue);
private AuthToken GetAuthToken() => new(
IsBotToken
? AuthTokenType.Bot
: AuthTokenType.User,
TokenValue
);
protected DiscordClient GetDiscordClient() => new DiscordClient(GetAuthToken());
private DiscordClient? _discordClient;
protected DiscordClient Discord => _discordClient ??= new DiscordClient(GetAuthToken());
public abstract ValueTask ExecuteAsync(IConsole console);
}
@@ -1,36 +1,42 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Cli.Commands
{
[Command("exportall", Description = "Export all accessible channels.")]
public class ExportAllCommand : ExportMultipleCommandBase
public class ExportAllCommand : ExportCommandBase
{
[CommandOption("include-dm", Description = "Whether to also export direct message channels.")]
public bool IncludeDirectMessages { get; set; } = true;
[CommandOption("include-dm", Description = "Include direct message channels.")]
public bool IncludeDirectMessages { get; init; } = true;
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
// Get channel metadata
await console.Output.WriteLineAsync("Fetching channels...");
var channels = new List<Channel>();
// Aggregate channels from all guilds
await foreach (var guild in GetDiscordClient().GetUserGuildsAsync())
await foreach (var guild in Discord.GetUserGuildsAsync())
{
// Skip DMs if instructed to
if (!IncludeDirectMessages && guild.Id == Guild.DirectMessages.Id)
continue;
await foreach (var channel in GetDiscordClient().GetGuildChannelsAsync(guild.Id))
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id))
{
channels.Add(channel);
}
}
await ExportMultipleAsync(console, channels);
// Export
await ExportAsync(console, channels);
}
}
}
}
@@ -1,16 +0,0 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Cli.Commands.Base;
namespace DiscordChatExporter.Cli.Commands
{
[Command("export", Description = "Export a channel.")]
public class ExportChannelCommand : ExportCommandBase
{
[CommandOption("channel", 'c', IsRequired = true, Description = "Channel ID.")]
public string ChannelId { get; set; } = "";
public override async ValueTask ExecuteAsync(IConsole console) => await ExportAsync(console, ChannelId);
}
}
@@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Cli.Commands
{
[Command("export", Description = "Export one or multiple channels.")]
public class ExportChannelsCommand : ExportCommandBase
{
// TODO: change this to plural (breaking change)
[CommandOption("channel", 'c', IsRequired = true, Description = "Channel ID(s).")]
public IReadOnlyList<Snowflake> ChannelIds { get; init; } = Array.Empty<Snowflake>();
public override async ValueTask ExecuteAsync(IConsole console)
{
await base.ExecuteAsync(console);
// Get channel metadata
await console.Output.WriteLineAsync("Fetching channel(s)...");
var channels = new List<Channel>();
foreach (var channelId in ChannelIds)
{
var channel = await Discord.GetChannelAsync(channelId);
channels.Add(channel);
}
// Export
await ExportAsync(console, channels);
}
}
}
@@ -1,19 +1,25 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Utilities;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands
{
[Command("exportdm", Description = "Export all direct message channels.")]
public class ExportDirectMessagesCommand : ExportMultipleCommandBase
public class ExportDirectMessagesCommand : ExportCommandBase
{
public override async ValueTask ExecuteAsync(IConsole console)
{
var channels = await GetDiscordClient().GetGuildChannelsAsync(Guild.DirectMessages.Id);
await ExportMultipleAsync(console, channels);
await base.ExecuteAsync(console);
// Get channel metadata
await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id);
// Export
await ExportAsync(console, channels);
}
}
}
@@ -1,21 +1,28 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Domain.Utilities;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands
{
[Command("exportguild", Description = "Export all channels within specified guild.")]
public class ExportGuildCommand : ExportMultipleCommandBase
public class ExportGuildCommand : ExportCommandBase
{
[CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")]
public string GuildId { get; set; } = "";
public Snowflake GuildId { get; init; }
public override async ValueTask ExecuteAsync(IConsole console)
{
var channels = await GetDiscordClient().GetGuildChannelsAsync(GuildId);
await ExportMultipleAsync(console, channels);
await base.ExecuteAsync(console);
// Get channel metadata
await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(GuildId);
// Export
await ExportAsync(console, channels);
}
}
}
@@ -1,9 +1,11 @@
using System.Linq;
using System;
using System.Linq;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Domain.Utilities;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands
{
@@ -11,14 +13,27 @@ namespace DiscordChatExporter.Cli.Commands
public class GetChannelsCommand : TokenCommandBase
{
[CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")]
public string GuildId { get; set; } = "";
public Snowflake GuildId { get; init; }
public override async ValueTask ExecuteAsync(IConsole console)
{
var channels = await GetDiscordClient().GetGuildChannelsAsync(GuildId);
var channels = await Discord.GetGuildChannelsAsync(GuildId);
foreach (var channel in channels.OrderBy(c => c.Category).ThenBy(c => c.Name))
console.Output.WriteLine($"{channel.Id} | {channel.Category} / {channel.Name}");
foreach (var channel in channels.OrderBy(c => c.Category.Position).ThenBy(c => c.Name))
{
// Channel ID
await console.Output.WriteAsync(channel.Id.ToString());
// Separator
using (console.WithForegroundColor(ConsoleColor.DarkGray))
await console.Output.WriteAsync(" | ");
// Channel category / name
using (console.WithForegroundColor(ConsoleColor.White))
await console.Output.WriteAsync($"{channel.Category} / {channel.Name}");
await console.Output.WriteLineAsync();
}
}
}
}
@@ -1,10 +1,11 @@
using System.Linq;
using System;
using System.Linq;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Utilities;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands
{
@@ -13,10 +14,23 @@ namespace DiscordChatExporter.Cli.Commands
{
public override async ValueTask ExecuteAsync(IConsole console)
{
var channels = await GetDiscordClient().GetGuildChannelsAsync(Guild.DirectMessages.Id);
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id);
foreach (var channel in channels.OrderBy(c => c.Category).ThenBy(c => c.Name))
console.Output.WriteLine($"{channel.Id} | {channel.Category} / {channel.Name}");
foreach (var channel in channels.OrderBy(c => c.Name))
{
// Channel ID
await console.Output.WriteAsync(channel.Id.ToString());
// Separator
using (console.WithForegroundColor(ConsoleColor.DarkGray))
await console.Output.WriteAsync(" | ");
// Channel category / name
using (console.WithForegroundColor(ConsoleColor.White))
await console.Output.WriteAsync($"{channel.Category} / {channel.Name}");
await console.Output.WriteLineAsync();
}
}
}
}
@@ -1,9 +1,10 @@
using System.Linq;
using System;
using System.Linq;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Domain.Utilities;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands
{
@@ -12,10 +13,23 @@ namespace DiscordChatExporter.Cli.Commands
{
public override async ValueTask ExecuteAsync(IConsole console)
{
var guilds = await GetDiscordClient().GetUserGuildsAsync();
var guilds = await Discord.GetUserGuildsAsync();
foreach (var guild in guilds.OrderBy(g => g.Name))
console.Output.WriteLine($"{guild.Id} | {guild.Name}");
{
// Guild ID
await console.Output.WriteAsync(guild.Id.ToString());
// Separator
using (console.WithForegroundColor(ConsoleColor.DarkGray))
await console.Output.WriteAsync(" | ");
// Guild name
using (console.WithForegroundColor(ConsoleColor.White))
await console.Output.WriteAsync(guild.Name);
await console.Output.WriteLineAsync();
}
}
}
}
@@ -2,6 +2,7 @@
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Infrastructure;
namespace DiscordChatExporter.Cli.Commands
{
@@ -10,7 +11,9 @@ namespace DiscordChatExporter.Cli.Commands
{
public ValueTask ExecuteAsync(IConsole console)
{
console.WithForegroundColor(ConsoleColor.White, () => console.Output.WriteLine("To get user token:"));
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("To get user token:");
console.Output.WriteLine(" 1. Open Discord");
console.Output.WriteLine(" 2. Press Ctrl+Shift+I to show developer tools");
console.Output.WriteLine(" 3. Navigate to the Application tab");
@@ -20,14 +23,18 @@ namespace DiscordChatExporter.Cli.Commands
console.Output.WriteLine(" * Automating user accounts is technically against TOS, use at your own risk.");
console.Output.WriteLine();
console.WithForegroundColor(ConsoleColor.White, () => console.Output.WriteLine("To get bot token:"));
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("To get bot token:");
console.Output.WriteLine(" 1. Go to Discord developer portal");
console.Output.WriteLine(" 2. Open your application's settings");
console.Output.WriteLine(" 3. Navigate to the Bot section on the left");
console.Output.WriteLine(" 4. Under Token click Copy");
console.Output.WriteLine();
console.WithForegroundColor(ConsoleColor.White, () => console.Output.WriteLine("To get guild ID or guild channel ID:"));
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("To get guild ID or guild channel ID:");
console.Output.WriteLine(" 1. Open Discord");
console.Output.WriteLine(" 2. Open Settings");
console.Output.WriteLine(" 3. Go to Appearance section");
@@ -35,7 +42,9 @@ namespace DiscordChatExporter.Cli.Commands
console.Output.WriteLine(" 5. Right click on the desired guild or channel and click Copy ID");
console.Output.WriteLine();
console.WithForegroundColor(ConsoleColor.White, () => console.Output.WriteLine("To get direct message channel ID:"));
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("To get direct message channel ID:");
console.Output.WriteLine(" 1. Open Discord");
console.Output.WriteLine(" 2. Open the desired direct message channel");
console.Output.WriteLine(" 3. Press Ctrl+Shift+I to show developer tools");
@@ -44,8 +53,11 @@ namespace DiscordChatExporter.Cli.Commands
console.Output.WriteLine(" 6. Copy the first long sequence of numbers inside the URL");
console.Output.WriteLine();
console.Output.WriteLine("For more information, check out the wiki:");
console.Output.WriteLine("https://github.com/Tyrrrz/DiscordChatExporter/wiki");
using (console.WithForegroundColor(ConsoleColor.White))
console.Output.WriteLine("For more information, check out the wiki:");
using (console.WithForegroundColor(ConsoleColor.DarkCyan))
console.Output.WriteLine("https://github.com/Tyrrrz/DiscordChatExporter/wiki");
return default;
}
@@ -1,5 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="../DiscordChatExporter.props" />
<PropertyGroup>
<OutputType>Exe</OutputType>
@@ -7,13 +6,14 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CliFx" Version="1.3.2" />
<PackageReference Include="CliFx" Version="2.0.4" />
<PackageReference Include="Spectre.Console" Version="0.40.0" />
<PackageReference Include="Gress" Version="1.2.0" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DiscordChatExporter.Domain\DiscordChatExporter.Domain.csproj" />
<ProjectReference Include="..\DiscordChatExporter.Core\DiscordChatExporter.Core.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,52 @@
using System;
using System.Threading.Tasks;
using CliFx.Infrastructure;
using Spectre.Console;
namespace DiscordChatExporter.Cli.Utils.Extensions
{
internal static class ConsoleExtensions
{
public static IAnsiConsole CreateAnsiConsole(this IConsole console) => AnsiConsole.Create(
new AnsiConsoleSettings
{
Ansi = AnsiSupport.Detect,
ColorSystem = ColorSystemSupport.Detect,
Out = new AnsiConsoleOutput(console.Output)
}
);
public static Progress CreateProgressTicker(this IConsole console) => console
.CreateAnsiConsole()
.Progress()
.AutoClear(false)
.AutoRefresh(true)
.HideCompleted(false)
.Columns(
new TaskDescriptionColumn {Alignment = Justify.Left},
new ProgressBarColumn(),
new PercentageColumn()
);
public static async ValueTask StartTaskAsync(
this ProgressContext progressContext,
string description,
Func<ProgressTask, ValueTask> performOperationAsync)
{
var progressTask = progressContext.AddTask(
// Don't recognize random square brackets as style tags
Markup.Escape(description),
new ProgressTaskSettings {MaxValue = 1}
);
try
{
await performOperationAsync(progressTask);
}
finally
{
progressTask.StopTask();
}
}
}
}
@@ -0,0 +1,9 @@
// ReSharper disable CheckNamespace
// TODO: remove after moving to .NET 5
namespace System.Runtime.CompilerServices
{
internal static class IsExternalInit
{
}
}
@@ -1,9 +1,7 @@
using System.Net.Http.Headers;
namespace DiscordChatExporter.Domain.Discord
namespace DiscordChatExporter.Core.Discord
{
public enum AuthTokenType { User, Bot }
public class AuthToken
{
public AuthTokenType Type { get; }
@@ -16,7 +14,7 @@ namespace DiscordChatExporter.Domain.Discord
Value = value;
}
public AuthenticationHeaderValue GetAuthorizationHeader() => Type switch
public AuthenticationHeaderValue GetAuthenticationHeader() => Type switch
{
AuthTokenType.Bot => new AuthenticationHeaderValue("Bot", Value),
_ => new AuthenticationHeaderValue(Value)
@@ -0,0 +1,8 @@
namespace DiscordChatExporter.Core.Discord
{
public enum AuthTokenType
{
User,
Bot
}
}
@@ -0,0 +1,77 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.Json;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#attachment-object
public partial class Attachment : IHasId
{
public Snowflake Id { get; }
public string Url { get; }
public string FileName { get; }
public int? Width { get; }
public int? Height { get; }
public bool IsImage => ImageFileExtensions.Contains(Path.GetExtension(FileName));
public bool IsVideo => VideoFileExtensions.Contains(Path.GetExtension(FileName));
public bool IsAudio => AudioFileExtensions.Contains(Path.GetExtension(FileName));
public bool IsSpoiler =>
(IsImage || IsVideo || IsAudio) && FileName.StartsWith("SPOILER_", StringComparison.Ordinal);
public FileSize FileSize { get; }
public Attachment(
Snowflake id,
string url,
string fileName,
int? width,
int? height,
FileSize fileSize)
{
Id = id;
Url = url;
FileName = fileName;
Width = width;
Height = height;
FileSize = fileSize;
}
public override string ToString() => FileName;
}
public partial class Attachment
{
private static readonly HashSet<string> ImageFileExtensions = new(StringComparer.OrdinalIgnoreCase)
{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp"};
private static readonly HashSet<string> VideoFileExtensions = new(StringComparer.OrdinalIgnoreCase)
{".mp4", ".webm"};
private static readonly HashSet<string> AudioFileExtensions = new(StringComparer.OrdinalIgnoreCase)
{".mp3", ".wav", ".ogg", ".flac", ".m4a"};
public static Attachment Parse(JsonElement json)
{
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
var url = json.GetProperty("url").GetString();
var width = json.GetPropertyOrNull("width")?.GetInt32();
var height = json.GetPropertyOrNull("height")?.GetInt32();
var fileName = json.GetProperty("filename").GetString();
var fileSize = json.GetProperty("size").GetInt64().Pipe(FileSize.FromBytes);
return new Attachment(id, url, fileName, width, height, fileSize);
}
}
}
@@ -0,0 +1,97 @@
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#channel-object
public partial class Channel : IHasId
{
public Snowflake Id { get; }
public ChannelType Type { get; }
public bool IsTextChannel => Type is
ChannelType.GuildTextChat or
ChannelType.DirectTextChat or
ChannelType.DirectGroupTextChat or
ChannelType.GuildNews or
ChannelType.GuildStore;
public Snowflake GuildId { get; }
public ChannelCategory Category { get; }
public string Name { get; }
public int? Position { get; }
public string? Topic { get; }
public Channel(
Snowflake id,
ChannelType type,
Snowflake guildId,
ChannelCategory category,
string name,
int? position,
string? topic)
{
Id = id;
Type = type;
GuildId = guildId;
Category = category;
Name = name;
Position = position;
Topic = topic;
}
public override string ToString() => Name;
}
public partial class Channel
{
private static ChannelCategory GetFallbackCategory(ChannelType channelType) => new(
Snowflake.Zero,
channelType switch
{
ChannelType.GuildTextChat => "Text",
ChannelType.DirectTextChat => "Private",
ChannelType.DirectGroupTextChat => "Group",
ChannelType.GuildNews => "News",
ChannelType.GuildStore => "Store",
_ => "Default"
},
null
);
public static Channel Parse(JsonElement json, ChannelCategory? category = null, int? position = null)
{
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
var guildId = json.GetPropertyOrNull("guild_id")?.GetString().Pipe(Snowflake.Parse);
var topic = json.GetPropertyOrNull("topic")?.GetString();
var type = (ChannelType) json.GetProperty("type").GetInt32();
var name =
// Guild channel
json.GetPropertyOrNull("name")?.GetString() ??
// DM channel
json.GetPropertyOrNull("recipients")?.EnumerateArray().Select(User.Parse).Select(u => u.Name).JoinToString(", ") ??
// Fallback
id.ToString();
return new Channel(
id,
type,
guildId ?? Guild.DirectMessages.Id,
category ?? GetFallbackCategory(type),
name,
position ?? json.GetPropertyOrNull("position")?.GetInt32(),
topic
);
}
}
}
@@ -0,0 +1,45 @@
using System.Text.Json;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data
{
public partial class ChannelCategory : IHasId
{
public Snowflake Id { get; }
public string Name { get; }
public int? Position { get; }
public ChannelCategory(Snowflake id, string name, int? position)
{
Id = id;
Name = name;
Position = position;
}
public override string ToString() => Name;
}
public partial class ChannelCategory
{
public static ChannelCategory Parse(JsonElement json, int? position = null)
{
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
var name =
json.GetPropertyOrNull("name")?.GetString() ??
id.ToString();
return new ChannelCategory(
id,
name,
position ?? json.GetPropertyOrNull("position")?.GetInt32()
);
}
public static ChannelCategory Empty { get; } = new(Snowflake.Zero, "<unknown category>", 0);
}
}
@@ -0,0 +1,15 @@
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#channel-object-channel-types
// Order of enum fields needs to match the order in the docs.
public enum ChannelType
{
GuildTextChat = 0,
DirectTextChat,
GuildVoiceChat,
DirectGroupTextChat,
GuildCategory,
GuildNews,
GuildStore
}
}
@@ -1,6 +1,6 @@
using System;
namespace DiscordChatExporter.Domain.Discord.Models.Common
namespace DiscordChatExporter.Core.Discord.Data.Common
{
// Loosely based on https://github.com/omar/ByteSize (MIT license)
public readonly partial struct FileSize
@@ -10,19 +10,11 @@ namespace DiscordChatExporter.Domain.Discord.Models.Common
public double TotalKiloBytes => TotalBytes / 1024.0;
public double TotalMegaBytes => TotalKiloBytes / 1024.0;
public double TotalGigaBytes => TotalMegaBytes / 1024.0;
public double TotalTeraBytes => TotalGigaBytes / 1024.0;
public double TotalPetaBytes => TotalTeraBytes / 1024.0;
public FileSize(long bytes) => TotalBytes = bytes;
private double GetLargestWholeNumberValue()
{
if (Math.Abs(TotalPetaBytes) >= 1)
return TotalPetaBytes;
if (Math.Abs(TotalTeraBytes) >= 1)
return TotalTeraBytes;
if (Math.Abs(TotalGigaBytes) >= 1)
return TotalGigaBytes;
@@ -37,12 +29,6 @@ namespace DiscordChatExporter.Domain.Discord.Models.Common
private string GetLargestWholeNumberSymbol()
{
if (Math.Abs(TotalPetaBytes) >= 1)
return "PB";
if (Math.Abs(TotalTeraBytes) >= 1)
return "TB";
if (Math.Abs(TotalGigaBytes) >= 1)
return "GB";
@@ -52,7 +38,7 @@ namespace DiscordChatExporter.Domain.Discord.Models.Common
if (Math.Abs(TotalKiloBytes) >= 1)
return "KB";
return "B";
return "bytes";
}
public override string ToString() => $"{GetLargestWholeNumberValue():0.##} {GetLargestWholeNumberSymbol()}";
@@ -60,6 +46,6 @@ namespace DiscordChatExporter.Domain.Discord.Models.Common
public partial struct FileSize
{
public static FileSize FromBytes(long bytes) => new FileSize(bytes);
public static FileSize FromBytes(long bytes) => new(bytes);
}
}
@@ -0,0 +1,7 @@
namespace DiscordChatExporter.Core.Discord.Data.Common
{
public interface IHasId
{
Snowflake Id { get; }
}
}
@@ -0,0 +1,16 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Discord.Data.Common
{
public partial class IdBasedEqualityComparer : IEqualityComparer<IHasId>
{
public bool Equals(IHasId? x, IHasId? y) => x?.Id == y?.Id;
public int GetHashCode(IHasId obj) => obj.Id.GetHashCode();
}
public partial class IdBasedEqualityComparer
{
public static IdBasedEqualityComparer Instance { get; } = new();
}
}
@@ -3,9 +3,10 @@ using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Domain.Internal.Extensions;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#embed-object
public partial class Embed
@@ -54,6 +55,8 @@ namespace DiscordChatExporter.Domain.Discord.Models
Footer = footer;
}
public YouTubeVideoEmbedProjection? TryGetYouTubeVideo() => YouTubeVideoEmbedProjection.TryResolve(this);
public override string ToString() => Title ?? "<untitled embed>";
}
@@ -62,10 +65,10 @@ namespace DiscordChatExporter.Domain.Discord.Models
public static Embed Parse(JsonElement json)
{
var title = json.GetPropertyOrNull("title")?.GetString();
var description = json.GetPropertyOrNull("description")?.GetString();
var url = json.GetPropertyOrNull("url")?.GetString();
var timestamp = json.GetPropertyOrNull("timestamp")?.GetDateTimeOffset();
var color = json.GetPropertyOrNull("color")?.GetInt32().Pipe(System.Drawing.Color.FromArgb).ResetAlpha();
var description = json.GetPropertyOrNull("description")?.GetString();
var author = json.GetPropertyOrNull("author")?.Pipe(EmbedAuthor.Parse);
var thumbnail = json.GetPropertyOrNull("thumbnail")?.Pipe(EmbedImage.Parse);
@@ -1,7 +1,7 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#embed-object-embed-author-structure
public partial class EmbedAuthor
@@ -12,11 +12,14 @@ namespace DiscordChatExporter.Domain.Discord.Models
public string? IconUrl { get; }
public EmbedAuthor(string? name, string? url, string? iconUrl)
public string? IconProxyUrl { get; }
public EmbedAuthor(string? name, string? url, string? iconUrl, string? iconProxyUrl)
{
Name = name;
Url = url;
IconUrl = iconUrl;
IconProxyUrl = iconProxyUrl;
}
public override string ToString() => Name ?? "<unnamed author>";
@@ -29,8 +32,9 @@ namespace DiscordChatExporter.Domain.Discord.Models
var name = json.GetPropertyOrNull("name")?.GetString();
var url = json.GetPropertyOrNull("url")?.GetString();
var iconUrl = json.GetPropertyOrNull("icon_url")?.GetString();
var iconProxyUrl = json.GetPropertyOrNull("proxy_icon_url")?.GetString();
return new EmbedAuthor(name, url, iconUrl);
return new EmbedAuthor(name, url, iconUrl, iconProxyUrl);
}
}
}
@@ -1,7 +1,7 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#embed-object-embed-field-structure
public partial class EmbedField
@@ -1,7 +1,7 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#embed-object-embed-footer-structure
public partial class EmbedFooter
@@ -10,10 +10,13 @@ namespace DiscordChatExporter.Domain.Discord.Models
public string? IconUrl { get; }
public EmbedFooter(string text, string? iconUrl)
public string? IconProxyUrl { get; }
public EmbedFooter(string text, string? iconUrl, string? iconProxyUrl)
{
Text = text;
IconUrl = iconUrl;
IconProxyUrl = iconProxyUrl;
}
public override string ToString() => Text;
@@ -25,8 +28,9 @@ namespace DiscordChatExporter.Domain.Discord.Models
{
var text = json.GetProperty("text").GetString();
var iconUrl = json.GetPropertyOrNull("icon_url")?.GetString();
var iconProxyUrl = json.GetPropertyOrNull("proxy_icon_url")?.GetString();
return new EmbedFooter(text, iconUrl);
return new EmbedFooter(text, iconUrl, iconProxyUrl);
}
}
}
@@ -1,20 +1,23 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure
public partial class EmbedImage
{
public string? Url { get; }
public string? ProxyUrl { get; }
public int? Width { get; }
public int? Height { get; }
public EmbedImage(string? url, int? width, int? height)
public EmbedImage(string? url, string? proxyUrl, int? width, int? height)
{
Url = url;
ProxyUrl = proxyUrl;
Height = height;
Width = width;
}
@@ -25,10 +28,11 @@ namespace DiscordChatExporter.Domain.Discord.Models
public static EmbedImage Parse(JsonElement json)
{
var url = json.GetPropertyOrNull("url")?.GetString();
var proxyUrl = json.GetPropertyOrNull("proxy_url")?.GetString();
var width = json.GetPropertyOrNull("width")?.GetInt32();
var height = json.GetPropertyOrNull("height")?.GetInt32();
return new EmbedImage(url, width, height);
return new EmbedImage(url, proxyUrl, width, height);
}
}
}
@@ -0,0 +1,74 @@
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/emoji#emoji-object
public partial class Emoji
{
// Only present on custom emoji
public string? Id { get; }
// Name of custom emoji (e.g. LUL) or actual representation of standard emoji (e.g. 🙂)
public string Name { get; }
// Name of custom emoji (e.g. LUL) or name of standard emoji (e.g. slight_smile)
public string Code => !string.IsNullOrWhiteSpace(Id)
? Name
: EmojiIndex.TryGetCode(Name) ?? Name;
public bool IsAnimated { get; }
public string ImageUrl { get; }
public Emoji(string? id, string name, bool isAnimated, string imageUrl)
{
Id = id;
Name = name;
IsAnimated = isAnimated;
ImageUrl = imageUrl;
}
public override string ToString() => Name;
}
public partial class Emoji
{
private static string GetTwemojiName(string name) => name
.GetRunes()
// Variant selector rune is skipped in Twemoji names
.Where(r => r.Value != 0xfe0f)
.Select(r => r.Value.ToString("x"))
.JoinToString("-");
public static string GetImageUrl(string? id, string name, bool isAnimated)
{
// Custom emoji
if (!string.IsNullOrWhiteSpace(id))
{
return isAnimated
? $"https://cdn.discordapp.com/emojis/{id}.gif"
: $"https://cdn.discordapp.com/emojis/{id}.png";
}
// Standard emoji
var twemojiName = GetTwemojiName(name);
return $"https://twemoji.maxcdn.com/2/svg/{twemojiName}.svg";
}
public static Emoji Parse(JsonElement json)
{
var id = json.GetPropertyOrNull("id")?.GetString();
var name = json.GetProperty("name").GetString();
var isAnimated = json.GetPropertyOrNull("animated")?.GetBoolean() ?? false;
var imageUrl = GetImageUrl(id, name, isAnimated);
return new Emoji(id, name, isAnimated, imageUrl);
}
}
}
@@ -1,18 +1,19 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/guild#guild-object
public partial class Guild : IHasId
{
public string Id { get; }
public Snowflake Id { get; }
public string Name { get; }
public string IconUrl { get; }
public Guild(string id, string name, string iconUrl)
public Guild(Snowflake id, string name, string iconUrl)
{
Id = id;
Name = name;
@@ -24,18 +25,21 @@ namespace DiscordChatExporter.Domain.Discord.Models
public partial class Guild
{
public static Guild DirectMessages { get; } =
new Guild("@me", "Direct Messages", GetDefaultIconUrl());
public static Guild DirectMessages { get; } = new(
Snowflake.Zero,
"Direct Messages",
GetDefaultIconUrl()
);
private static string GetDefaultIconUrl() =>
"https://cdn.discordapp.com/embed/avatars/0.png";
private static string GetIconUrl(string id, string iconHash) =>
private static string GetIconUrl(Snowflake id, string iconHash) =>
$"https://cdn.discordapp.com/icons/{id}/{iconHash}.png";
public static Guild Parse(JsonElement json)
{
var id = json.GetProperty("id").GetString();
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
var name = json.GetProperty("name").GetString();
var iconHash = json.GetProperty("icon").GetString();
@@ -2,26 +2,27 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Domain.Internal.Extensions;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/guild#guild-member-object
public partial class Member : IHasId
{
public string Id => User.Id;
public Snowflake Id => User.Id;
public User User { get; }
public string Nick { get; }
public IReadOnlyList<string> RoleIds { get; }
public IReadOnlyList<Snowflake> RoleIds { get; }
public Member(User user, string? nick, IReadOnlyList<string> roleIds)
public Member(User user, string nick, IReadOnlyList<Snowflake> roleIds)
{
User = user;
Nick = nick ?? user.Name;
Nick = nick;
RoleIds = roleIds;
}
@@ -30,8 +31,11 @@ namespace DiscordChatExporter.Domain.Discord.Models
public partial class Member
{
public static Member CreateForUser(User user) =>
new Member(user, null, Array.Empty<string>());
public static Member CreateForUser(User user) => new(
user,
user.Name,
Array.Empty<Snowflake>()
);
public static Member Parse(JsonElement json)
{
@@ -39,12 +43,12 @@ namespace DiscordChatExporter.Domain.Discord.Models
var nick = json.GetPropertyOrNull("nick")?.GetString();
var roleIds =
json.GetPropertyOrNull("roles")?.EnumerateArray().Select(j => j.GetString()).ToArray() ??
Array.Empty<string>();
json.GetPropertyOrNull("roles")?.EnumerateArray().Select(j => j.GetString()).Select(Snowflake.Parse).ToArray() ??
Array.Empty<Snowflake>();
return new Member(
user,
nick,
nick ?? user.Name,
roleIds
);
}
@@ -2,28 +2,30 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Domain.Internal.Extensions;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#message-object-message-types
public enum MessageType
{
Default,
RecipientAdd,
RecipientRemove,
Call,
ChannelNameChange,
ChannelIconChange,
ChannelPinnedMessage,
GuildMemberJoin
Default = 0,
RecipientAdd = 1,
RecipientRemove = 2,
Call = 3,
ChannelNameChange = 4,
ChannelIconChange = 5,
ChannelPinnedMessage = 6,
GuildMemberJoin = 7,
Reply = 19
}
// https://discord.com/developers/docs/resources/channel#message-object
public partial class Message : IHasId
{
public string Id { get; }
public Snowflake Id { get; }
public MessageType Type { get; }
@@ -47,8 +49,12 @@ namespace DiscordChatExporter.Domain.Discord.Models
public IReadOnlyList<User> MentionedUsers { get; }
public MessageReference? Reference { get; }
public Message? ReferencedMessage { get; }
public Message(
string id,
Snowflake id,
MessageType type,
User author,
DateTimeOffset timestamp,
@@ -59,7 +65,9 @@ namespace DiscordChatExporter.Domain.Discord.Models
IReadOnlyList<Attachment> attachments,
IReadOnlyList<Embed> embeds,
IReadOnlyList<Reaction> reactions,
IReadOnlyList<User> mentionedUsers)
IReadOnlyList<User> mentionedUsers,
MessageReference? messageReference,
Message? referencedMessage)
{
Id = id;
Type = type;
@@ -73,6 +81,8 @@ namespace DiscordChatExporter.Domain.Discord.Models
Embeds = embeds;
Reactions = reactions;
MentionedUsers = mentionedUsers;
Reference = messageReference;
ReferencedMessage = referencedMessage;
}
public override string ToString() => Content;
@@ -82,20 +92,22 @@ namespace DiscordChatExporter.Domain.Discord.Models
{
public static Message Parse(JsonElement json)
{
var id = json.GetProperty("id").GetString();
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
var author = json.GetProperty("author").Pipe(User.Parse);
var timestamp = json.GetProperty("timestamp").GetDateTimeOffset();
var editedTimestamp = json.GetPropertyOrNull("edited_timestamp")?.GetDateTimeOffset();
var callEndedTimestamp = json.GetPropertyOrNull("call")?.GetPropertyOrNull("ended_timestamp")?.GetDateTimeOffset();
var type = (MessageType) json.GetProperty("type").GetInt32();
var isPinned = json.GetPropertyOrNull("pinned")?.GetBoolean() ?? false;
var messageReference = json.GetPropertyOrNull("message_reference")?.Pipe(MessageReference.Parse);
var referencedMessage = json.GetPropertyOrNull("referenced_message")?.Pipe(Parse);
var content = type switch
{
MessageType.RecipientAdd => "Added a recipient.",
MessageType.RecipientRemove => "Removed a recipient.",
MessageType.Call =>
$"Started a call that lasted {callEndedTimestamp?.Pipe(t => t - timestamp).Pipe(t => (int) t.TotalMinutes) ?? 0} minutes.",
$"Started a call that lasted {callEndedTimestamp?.Pipe(t => t - timestamp).Pipe(t => (int) t.TotalMinutes) ?? 0} minutes.",
MessageType.ChannelNameChange => "Changed the channel name.",
MessageType.ChannelIconChange => "Changed the channel icon.",
MessageType.ChannelPinnedMessage => "Pinned a message.",
@@ -131,7 +143,9 @@ namespace DiscordChatExporter.Domain.Discord.Models
attachments,
embeds,
reactions,
mentionedUsers
mentionedUsers,
messageReference,
referencedMessage
);
}
}
@@ -0,0 +1,37 @@
using System.Text.Json;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#message-object-message-reference-structure
public partial class MessageReference
{
public Snowflake? MessageId { get; }
public Snowflake? ChannelId { get; }
public Snowflake? GuildId { get; }
public MessageReference(Snowflake? messageId, Snowflake? channelId, Snowflake? guildId)
{
MessageId = messageId;
ChannelId = channelId;
GuildId = guildId;
}
public override string ToString() => MessageId?.ToString() ?? "<unknown reference>";
}
public partial class MessageReference
{
public static MessageReference Parse(JsonElement json)
{
var messageId = json.GetPropertyOrNull("message_id")?.GetString().Pipe(Snowflake.Parse);
var channelId = json.GetPropertyOrNull("channel_id")?.GetString().Pipe(Snowflake.Parse);
var guildId = json.GetPropertyOrNull("guild_id")?.GetString().Pipe(Snowflake.Parse);
return new MessageReference(messageId, channelId, guildId);
}
}
}
@@ -1,7 +1,7 @@
using System.Text.Json;
using DiscordChatExporter.Domain.Internal.Extensions;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/channel#reaction-object
public partial class Reaction
@@ -23,8 +23,8 @@ namespace DiscordChatExporter.Domain.Discord.Models
{
public static Reaction Parse(JsonElement json)
{
var count = json.GetProperty("count").GetInt32();
var emoji = json.GetProperty("emoji").Pipe(Emoji.Parse);
var count = json.GetProperty("count").GetInt32();
return new Reaction(emoji, count);
}
@@ -1,13 +1,15 @@
using System.Drawing;
using System.Text.Json;
using DiscordChatExporter.Domain.Internal.Extensions;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/topics/permissions#role-object
public partial class Role
public partial class Role : IHasId
{
public string Id { get; }
public Snowflake Id { get; }
public string Name { get; }
@@ -15,7 +17,11 @@ namespace DiscordChatExporter.Domain.Discord.Models
public Color? Color { get; }
public Role(string id, string name, int position, Color? color)
public Role(
Snowflake id,
string name,
int position,
Color? color)
{
Id = id;
Name = name;
@@ -30,11 +36,12 @@ namespace DiscordChatExporter.Domain.Discord.Models
{
public static Role Parse(JsonElement json)
{
var id = json.GetProperty("id").GetString();
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
var name = json.GetProperty("name").GetString();
var position = json.GetProperty("position").GetInt32();
var color = json.GetPropertyOrNull("color")?
var color = json
.GetPropertyOrNull("color")?
.GetInt32()
.Pipe(System.Drawing.Color.FromArgb)
.ResetAlpha()
@@ -1,26 +1,34 @@
using System;
using System.Text.Json;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Domain.Internal.Extensions;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord.Models
namespace DiscordChatExporter.Core.Discord.Data
{
// https://discord.com/developers/docs/resources/user#user-object
public partial class User : IHasId
{
public string Id { get; }
public Snowflake Id { get; }
public bool IsBot { get; }
public int Discriminator { get; }
public string DiscriminatorFormatted => $"{Discriminator:0000}";
public string Name { get; }
public string FullName => $"{Name}#{Discriminator:0000}";
public string FullName => $"{Name}#{DiscriminatorFormatted}";
public string AvatarUrl { get; }
public User(string id, bool isBot, int discriminator, string name, string avatarUrl)
public User(
Snowflake id,
bool isBot,
int discriminator,
string name,
string avatarUrl)
{
Id = id;
IsBot = isBot;
@@ -37,23 +45,23 @@ namespace DiscordChatExporter.Domain.Discord.Models
private static string GetDefaultAvatarUrl(int discriminator) =>
$"https://cdn.discordapp.com/embed/avatars/{discriminator % 5}.png";
private static string GetAvatarUrl(string id, string avatarHash)
private static string GetAvatarUrl(Snowflake id, string avatarHash)
{
// Animated
if (avatarHash.StartsWith("a_", StringComparison.Ordinal))
return $"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.gif";
return $"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.gif?size=40";
// Non-animated
return $"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.png";
return $"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.png?size=40";
}
public static User Parse(JsonElement json)
{
var id = json.GetProperty("id").GetString();
var id = json.GetProperty("id").GetString().Pipe(Snowflake.Parse);
var isBot = json.GetPropertyOrNull("bot")?.GetBoolean() ?? false;
var discriminator = json.GetProperty("discriminator").GetString().Pipe(int.Parse);
var name = json.GetProperty("username").GetString();
var avatarHash = json.GetProperty("avatar").GetString();
var isBot = json.GetPropertyOrNull("bot")?.GetBoolean() ?? false;
var avatarUrl = !string.IsNullOrWhiteSpace(avatarHash)
? GetAvatarUrl(id, avatarHash)
@@ -0,0 +1,61 @@
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Core.Discord.Data
{
public partial class YouTubeVideoEmbedProjection
{
public string VideoId { get; }
public string Url { get; }
public YouTubeVideoEmbedProjection(string videoId, string url)
{
VideoId = videoId;
Url = url;
}
public override string ToString() => Url;
}
public partial class YouTubeVideoEmbedProjection
{
// Adapted from YoutubeExplode
// https://github.com/Tyrrrz/YoutubeExplode/blob/5be164be20019783913f76fcc98f18c65aebe9f0/YoutubeExplode/Videos/VideoId.cs#L34-L64
private static string? TryParseVideoId(string embedUrl)
{
// Regular URL
// https://www.youtube.com/watch?v=yIVRs6YSbOM
var regularMatch = Regex.Match(embedUrl, @"youtube\..+?/watch.*?v=(.*?)(?:&|/|$)").Groups[1].Value;
if (!string.IsNullOrWhiteSpace(regularMatch))
return regularMatch;
// Short URL
// https://youtu.be/yIVRs6YSbOM
var shortMatch = Regex.Match(embedUrl, @"youtu\.be/(.*?)(?:\?|&|/|$)").Groups[1].Value;
if (!string.IsNullOrWhiteSpace(shortMatch))
return shortMatch;
// Embed URL
// https://www.youtube.com/embed/yIVRs6YSbOM
var embedMatch = Regex.Match(embedUrl, @"youtube\..+?/embed/(.*?)(?:\?|&|/|$)").Groups[1].Value;
if (!string.IsNullOrWhiteSpace(embedMatch))
return embedMatch;
return null;
}
public static YouTubeVideoEmbedProjection? TryResolve(Embed embed)
{
if (string.IsNullOrWhiteSpace(embed.Url))
return null;
var videoId = TryParseVideoId(embed.Url);
if (string.IsNullOrWhiteSpace(videoId))
return null;
var url = $"https://www.youtube.com/embed/{videoId}";
return new YouTubeVideoEmbedProjection(videoId, url);
}
}
}
@@ -5,54 +5,41 @@ using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exceptions;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Internal.Extensions;
using Polly;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions;
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Http;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Domain.Discord
namespace DiscordChatExporter.Core.Discord
{
public class DiscordClient
{
private readonly HttpClient _httpClient;
private readonly AuthToken _token;
private readonly HttpClient _httpClient = Singleton.HttpClient;
private readonly IAsyncPolicy<HttpResponseMessage> _httpRequestPolicy;
private readonly Uri _baseUri = new Uri("https://discord.com/api/v6/", UriKind.Absolute);
private readonly Uri _baseUri = new("https://discord.com/api/v8/", UriKind.Absolute);
public DiscordClient(AuthToken token)
public DiscordClient(HttpClient httpClient, AuthToken token)
{
_httpClient = httpClient;
_token = token;
// Discord seems to always respond with 429 on the first request with unreasonable wait time (10+ minutes).
// For that reason the policy will ignore such errors at first, then wait a constant amount of time, and
// finally wait the specified amount of time, based on how many requests have failed in a row.
_httpRequestPolicy = Policy
.HandleResult<HttpResponseMessage>(m => m.StatusCode == HttpStatusCode.TooManyRequests)
.OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError)
.WaitAndRetryAsync(6,
(i, result, ctx) =>
{
if (i <= 3)
return TimeSpan.FromSeconds(2 * i);
if (i <= 5)
return TimeSpan.FromSeconds(5 * i);
return result.Result.Headers.RetryAfter.Delta ?? TimeSpan.FromSeconds(10 * i);
},
(response, timespan, retryCount, context) => Task.CompletedTask
);
}
private async ValueTask<HttpResponseMessage> GetResponseAsync(string url) => await _httpRequestPolicy.ExecuteAsync(async () =>
public DiscordClient(AuthToken token)
: this(Http.Client, token)
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
request.Headers.Authorization = _token.GetAuthorizationHeader();
}
return await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
});
private async ValueTask<HttpResponseMessage> GetResponseAsync(string url) =>
await Http.ResponsePolicy.ExecuteAsync(async () =>
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
request.Headers.Authorization = _token.GetAuthenticationHeader();
return await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
});
private async ValueTask<JsonElement> GetJsonResponseAsync(string url)
{
@@ -78,20 +65,21 @@ namespace DiscordChatExporter.Domain.Discord
return response.IsSuccessStatusCode
? await response.Content.ReadAsJsonAsync()
: (JsonElement?) null;
: null;
}
public async IAsyncEnumerable<Guild> GetUserGuildsAsync()
{
yield return Guild.DirectMessages;
var afterId = "";
var currentAfter = Snowflake.Zero;
while (true)
{
var url = new UrlBuilder()
.SetPath("users/@me/guilds")
.SetQueryParameter("limit", "100")
.SetQueryParameter("after", afterId)
.SetQueryParameter("after", currentAfter.ToString())
.Build();
var response = await GetJsonResponseAsync(url);
@@ -101,7 +89,7 @@ namespace DiscordChatExporter.Domain.Discord
{
yield return guild;
afterId = guild.Id;
currentAfter = guild.Id;
isEmpty = false;
}
@@ -110,7 +98,7 @@ namespace DiscordChatExporter.Domain.Discord
}
}
public async ValueTask<Guild> GetGuildAsync(string guildId)
public async ValueTask<Guild> GetGuildAsync(Snowflake guildId)
{
if (guildId == Guild.DirectMessages.Id)
return Guild.DirectMessages;
@@ -119,7 +107,7 @@ namespace DiscordChatExporter.Domain.Discord
return Guild.Parse(response);
}
public async IAsyncEnumerable<Channel> GetGuildChannelsAsync(string guildId)
public async IAsyncEnumerable<Channel> GetGuildChannelsAsync(Snowflake guildId)
{
if (guildId == Guild.DirectMessages.Id)
{
@@ -131,32 +119,41 @@ namespace DiscordChatExporter.Domain.Discord
{
var response = await GetJsonResponseAsync($"guilds/{guildId}/channels");
var categories = response
var responseOrdered = response
.EnumerateArray()
.ToDictionary(
j => j.GetProperty("id").GetString(),
j => j.GetProperty("name").GetString()
);
.OrderBy(j => j.GetProperty("position").GetInt32())
.ThenBy(j => Snowflake.Parse(j.GetProperty("id").GetString()))
.ToArray();
foreach (var channelJson in response.EnumerateArray())
var categories = responseOrdered
.Where(j => j.GetProperty("type").GetInt32() == (int) ChannelType.GuildCategory)
.Select((j, index) => ChannelCategory.Parse(j, index + 1))
.ToDictionary(j => j.Id.ToString(), StringComparer.Ordinal);
var position = 0;
foreach (var channelJson in responseOrdered)
{
var parentId = channelJson.GetPropertyOrNull("parent_id")?.GetString();
var category = !string.IsNullOrWhiteSpace(parentId)
? categories.GetValueOrDefault(parentId)
: null;
var channel = Channel.Parse(channelJson, category);
var channel = Channel.Parse(channelJson, category, position);
// Skip non-text channels
// We are only interested in text channels
if (!channel.IsTextChannel)
continue;
position++;
yield return channel;
}
}
}
public async IAsyncEnumerable<Role> GetGuildRolesAsync(string guildId)
public async IAsyncEnumerable<Role> GetGuildRolesAsync(Snowflake guildId)
{
if (guildId == Guild.DirectMessages.Id)
yield break;
@@ -167,7 +164,7 @@ namespace DiscordChatExporter.Domain.Discord
yield return Role.Parse(roleJson);
}
public async ValueTask<Member?> TryGetGuildMemberAsync(string guildId, User user)
public async ValueTask<Member?> TryGetGuildMemberAsync(Snowflake guildId, User user)
{
if (guildId == Guild.DirectMessages.Id)
return Member.CreateForUser(user);
@@ -176,30 +173,40 @@ namespace DiscordChatExporter.Domain.Discord
return response?.Pipe(Member.Parse);
}
private async ValueTask<string> GetChannelCategoryAsync(string channelParentId)
public async ValueTask<ChannelCategory> GetChannelCategoryAsync(Snowflake channelId)
{
var response = await GetJsonResponseAsync($"channels/{channelParentId}");
return response.GetProperty("name").GetString();
try
{
var response = await GetJsonResponseAsync($"channels/{channelId}");
return ChannelCategory.Parse(response);
}
// In some cases, the Discord API returns an empty body when requesting channel category.
// Instead, we use an empty channel category as a fallback.
catch (DiscordChatExporterException)
{
return ChannelCategory.Empty;
}
}
public async ValueTask<Channel> GetChannelAsync(string channelId)
public async ValueTask<Channel> GetChannelAsync(Snowflake channelId)
{
var response = await GetJsonResponseAsync($"channels/{channelId}");
var parentId = response.GetPropertyOrNull("parent_id")?.GetString();
var category = !string.IsNullOrWhiteSpace(parentId)
? await GetChannelCategoryAsync(parentId)
var parentId = response.GetPropertyOrNull("parent_id")?.GetString().Pipe(Snowflake.Parse);
var category = parentId is not null
? await GetChannelCategoryAsync(parentId.Value)
: null;
return Channel.Parse(response, category);
}
private async ValueTask<Message?> TryGetLastMessageAsync(string channelId, DateTimeOffset? before = null)
private async ValueTask<Message?> TryGetLastMessageAsync(Snowflake channelId, Snowflake? before = null)
{
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
.SetQueryParameter("limit", "1")
.SetQueryParameter("before", before?.ToSnowflake())
.SetQueryParameter("before", before?.ToString())
.Build();
var response = await GetJsonResponseAsync(url);
@@ -207,29 +214,29 @@ namespace DiscordChatExporter.Domain.Discord
}
public async IAsyncEnumerable<Message> GetMessagesAsync(
string channelId,
DateTimeOffset? after = null,
DateTimeOffset? before = null,
Snowflake channelId,
Snowflake? after = null,
Snowflake? before = null,
IProgress<double>? progress = null)
{
// Get the last message in the specified range.
// This snapshots the boundaries, which means that messages posted after the exported started
// This snapshots the boundaries, which means that messages posted after the export started
// will not appear in the output.
// Additionally, it provides the date of the last message, which is used to calculate progress.
var lastMessage = await TryGetLastMessageAsync(channelId, before);
if (lastMessage == null || lastMessage.Timestamp < after)
if (lastMessage is null || lastMessage.Timestamp < after?.ToDate())
yield break;
// Keep track of first message in range in order to calculate progress
var firstMessage = default(Message);
var afterId = after?.ToSnowflake() ?? "0";
var currentAfter = after ?? Snowflake.Zero;
while (true)
{
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
.SetQueryParameter("limit", "100")
.SetQueryParameter("after", afterId)
.SetQueryParameter("after", currentAfter.ToString())
.Build();
var response = await GetJsonResponseAsync(url);
@@ -252,14 +259,26 @@ namespace DiscordChatExporter.Domain.Discord
if (message.Timestamp > lastMessage.Timestamp)
yield break;
// Report progress based on the duration of parsed messages divided by total
progress?.Report(
(message.Timestamp - firstMessage.Timestamp) /
(lastMessage.Timestamp - firstMessage.Timestamp)
);
// Report progress based on the duration of exported messages divided by total
if (progress is not null)
{
var exportedDuration = (message.Timestamp - firstMessage.Timestamp).Duration();
var totalDuration = (lastMessage.Timestamp - firstMessage.Timestamp).Duration();
if (totalDuration > TimeSpan.Zero)
{
progress.Report(exportedDuration / totalDuration);
}
// Avoid division by zero if all messages have the exact same timestamp
// (which may be the case if there's only one message in the channel)
else
{
progress.Report(1);
}
}
yield return message;
afterId = message.Id;
currentAfter = message.Id;
}
}
}
@@ -0,0 +1,71 @@
using System;
using System.Globalization;
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Core.Discord
{
public readonly partial struct Snowflake
{
public ulong Value { get; }
public Snowflake(ulong value) => Value = value;
public DateTimeOffset ToDate() => DateTimeOffset.FromUnixTimeMilliseconds(
(long) ((Value >> 22) + 1420070400000UL)
).ToLocalTime();
public override string ToString() => Value.ToString(CultureInfo.InvariantCulture);
}
public partial struct Snowflake
{
public static Snowflake Zero { get; } = new(0);
public static Snowflake FromDate(DateTimeOffset date)
{
var value = ((ulong) date.ToUnixTimeMilliseconds() - 1420070400000UL) << 22;
return new Snowflake(value);
}
public static Snowflake? TryParse(string? str, IFormatProvider? formatProvider = null)
{
if (string.IsNullOrWhiteSpace(str))
return null;
// As number
if (Regex.IsMatch(str, @"^\d+$") &&
ulong.TryParse(str, NumberStyles.Number, formatProvider, out var value))
{
return new Snowflake(value);
}
// As date
if (DateTimeOffset.TryParse(str, formatProvider, DateTimeStyles.None, out var date))
{
return FromDate(date);
}
return null;
}
public static Snowflake Parse(string str, IFormatProvider? formatProvider) =>
TryParse(str, formatProvider) ?? throw new FormatException($"Invalid snowflake '{str}'.");
public static Snowflake Parse(string str) => Parse(str, null);
}
public partial struct Snowflake : IComparable<Snowflake>, IEquatable<Snowflake>
{
public int CompareTo(Snowflake other) => Value.CompareTo(other.Value);
public bool Equals(Snowflake other) => CompareTo(other) == 0;
public override bool Equals(object? obj) => obj is Snowflake other && Equals(other);
public override int GetHashCode() => Value.GetHashCode();
public static bool operator ==(Snowflake left, Snowflake right) => left.Equals(right);
public static bool operator !=(Snowflake left, Snowflake right) => !(left == right);
}
}
@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="JsonExtensions" Version="1.1.0" />
<PackageReference Include="MiniRazor.CodeGen" Version="2.1.2" />
<PackageReference Include="Polly" Version="7.2.2" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Exporting\Writers\Html\Core.css" />
<EmbeddedResource Include="Exporting\Writers\Html\Dark.css" />
<EmbeddedResource Include="Exporting\Writers\Html\Light.css" />
<AdditionalFiles Include="Exporting\Writers\Html\PreambleTemplate.cshtml" IsRazorTemplate="true" />
<AdditionalFiles Include="Exporting\Writers\Html\PostambleTemplate.cshtml" IsRazorTemplate="true" />
<AdditionalFiles Include="Exporting\Writers\Html\MessageGroupTemplate.cshtml" IsRazorTemplate="true" />
</ItemGroup>
</Project>
@@ -1,7 +1,7 @@
using System;
using System.Net.Http;
namespace DiscordChatExporter.Domain.Exceptions
namespace DiscordChatExporter.Core.Exceptions
{
public partial class DiscordChatExporterException : Exception
{
@@ -21,35 +21,25 @@ namespace DiscordChatExporter.Domain.Exceptions
var message = $@"
Failed to perform an HTTP request.
[Request]
{response.RequestMessage}
[Response]
{response}";
return new DiscordChatExporterException(message.Trim(), true);
}
internal static DiscordChatExporterException Unauthorized()
{
const string message = "Authentication token is invalid.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException Unauthorized() =>
new("Authentication token is invalid.");
internal static DiscordChatExporterException Forbidden()
{
const string message = "Access is forbidden.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException Forbidden() =>
new("Access is forbidden.");
internal static DiscordChatExporterException NotFound()
{
const string message = "Requested resource does not exist.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException NotFound() =>
new("Requested resource does not exist.");
internal static DiscordChatExporterException ChannelIsEmpty(string channel)
{
var message = $"Channel '{channel}' contains no messages for the specified period.";
return new DiscordChatExporterException(message);
}
internal static DiscordChatExporterException ChannelIsEmpty() =>
new("No messages found for the specified period.");
}
}
@@ -2,13 +2,13 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Discord.Models.Common;
using DiscordChatExporter.Domain.Exceptions;
using DiscordChatExporter.Domain.Utilities;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Exceptions;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Domain.Exporting
namespace DiscordChatExporter.Core.Exporting
{
public class ChannelExporter
{
@@ -59,7 +59,7 @@ namespace DiscordChatExporter.Domain.Exporting
// Throw if no messages were exported
if (!exportedAnything)
throw DiscordChatExporterException.ChannelIsEmpty(request.Channel.Name);
throw DiscordChatExporterException.ChannelIsEmpty();
}
}
}
@@ -5,11 +5,12 @@ using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Internal.Extensions;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Domain.Exporting
namespace DiscordChatExporter.Core.Exporting
{
internal class ExportContext
{
@@ -34,27 +35,29 @@ namespace DiscordChatExporter.Domain.Exporting
Channels = channels;
Roles = roles;
_mediaDownloader = new MediaDownloader(request.OutputMediaDirPath);
_mediaDownloader = new MediaDownloader(request.OutputMediaDirPath, request.ShouldReuseMedia);
}
public string FormatDate(DateTimeOffset date) => date.ToLocalString(Request.DateFormat);
public string FormatDate(DateTimeOffset date) => Request.DateFormat switch
{
"unix" => date.ToUnixTimeSeconds().ToString(),
"unixms" => date.ToUnixTimeMilliseconds().ToString(),
var dateFormat => date.ToLocalString(dateFormat)
};
public Member? TryGetMember(string id) =>
Members.FirstOrDefault(m => m.Id == id);
public Member? TryGetMember(Snowflake id) => Members.FirstOrDefault(m => m.Id == id);
public Channel? TryGetChannel(string id) =>
Channels.FirstOrDefault(c => c.Id == id);
public Channel? TryGetChannel(Snowflake id) => Channels.FirstOrDefault(c => c.Id == id);
public Role? TryGetRole(string id) =>
Roles.FirstOrDefault(r => r.Id == id);
public Role? TryGetRole(Snowflake id) => Roles.FirstOrDefault(r => r.Id == id);
public Color? TryGetUserColor(string id)
public Color? TryGetUserColor(Snowflake id)
{
var member = TryGetMember(id);
var roles = member?.RoleIds.Join(Roles, i => i, r => r.Id, (_, role) => role);
return roles?
.Where(r => r.Color != null)
.Where(r => r.Color is not null)
.OrderByDescending(r => r.Position)
.Select(r => r.Color)
.FirstOrDefault();
@@ -69,11 +72,14 @@ namespace DiscordChatExporter.Domain.Exporting
{
var filePath = await _mediaDownloader.DownloadAsync(url);
// We want relative path so that the output files can be copied around without breaking
var relativeFilePath = Path.GetRelativePath(Request.OutputBaseDirPath, filePath);
// We want relative path so that the output files can be copied around without breaking.
// Base directory path may be null if the file is stored at the root or relative to working directory.
var relativeFilePath = !string.IsNullOrWhiteSpace(Request.OutputBaseDirPath)
? Path.GetRelativePath(Request.OutputBaseDirPath, filePath)
: filePath;
// For HTML, we need to format the URL properly
if (Request.Format == ExportFormat.HtmlDark || Request.Format == ExportFormat.HtmlLight)
// HACK: for HTML, we need to format the URL properly
if (Request.Format is ExportFormat.HtmlDark or ExportFormat.HtmlLight)
{
// Need to escape each path segment while keeping the directory separators intact
return relativeFilePath
@@ -87,8 +93,9 @@ namespace DiscordChatExporter.Domain.Exporting
// Try to catch only exceptions related to failed HTTP requests
// https://github.com/Tyrrrz/DiscordChatExporter/issues/332
// https://github.com/Tyrrrz/DiscordChatExporter/issues/372
catch (Exception ex) when (ex is HttpRequestException || ex is OperationCanceledException)
catch (Exception ex) when (ex is HttpRequestException or OperationCanceledException)
{
// TODO: add logging so we can be more liberal with catching exceptions
// We don't want this to crash the exporting process in case of failure
return url;
}
@@ -1,6 +1,6 @@
using System;
namespace DiscordChatExporter.Domain.Exporting
namespace DiscordChatExporter.Core.Exporting
{
public enum ExportFormat
{
@@ -1,10 +1,13 @@
using System;
using System.IO;
using System.Text;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Internal;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Partitioning;
using DiscordChatExporter.Core.Utils;
namespace DiscordChatExporter.Domain.Exporting
namespace DiscordChatExporter.Core.Exporting
{
public partial class ExportRequest
{
@@ -22,14 +25,16 @@ namespace DiscordChatExporter.Domain.Exporting
public ExportFormat Format { get; }
public DateTimeOffset? After { get; }
public Snowflake? After { get; }
public DateTimeOffset? Before { get; }
public Snowflake? Before { get; }
public int? PartitionLimit { get; }
public PartitionLimit PartitionLimit { get; }
public bool ShouldDownloadMedia { get; }
public bool ShouldReuseMedia { get; }
public string DateFormat { get; }
public ExportRequest(
@@ -37,10 +42,11 @@ namespace DiscordChatExporter.Domain.Exporting
Channel channel,
string outputPath,
ExportFormat format,
DateTimeOffset? after,
DateTimeOffset? before,
int? partitionLimit,
Snowflake? after,
Snowflake? before,
PartitionLimit partitionLimit,
bool shouldDownloadMedia,
bool shouldReuseMedia,
string dateFormat)
{
Guild = guild;
@@ -51,6 +57,7 @@ namespace DiscordChatExporter.Domain.Exporting
Before = before;
PartitionLimit = partitionLimit;
ShouldDownloadMedia = shouldDownloadMedia;
ShouldReuseMedia = shouldReuseMedia;
DateFormat = dateFormat;
OutputBaseFilePath = GetOutputBaseFilePath(
@@ -74,9 +81,29 @@ namespace DiscordChatExporter.Domain.Exporting
Channel channel,
string outputPath,
ExportFormat format,
DateTimeOffset? after = null,
DateTimeOffset? before = null)
Snowflake? after = null,
Snowflake? before = null)
{
// Formats path
outputPath = Regex.Replace(outputPath, "%.", m =>
PathEx.EscapePath(m.Value switch
{
"%g" => guild.Id.ToString(),
"%G" => guild.Name,
"%t" => channel.Category.Id.ToString(),
"%T" => channel.Category.Name,
"%c" => channel.Id.ToString(),
"%C" => channel.Name,
"%p" => channel.Position?.ToString() ?? "0",
"%P" => channel.Category.Position?.ToString() ?? "0",
"%a" => (after ?? Snowflake.Zero).ToDate().ToString("yyyy-MM-dd"),
"%b" => (before?.ToDate() ?? DateTime.Now).ToString("yyyy-MM-dd"),
"%%" => "%",
_ => m.Value
})
);
// Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
@@ -92,33 +119,33 @@ namespace DiscordChatExporter.Domain.Exporting
Guild guild,
Channel channel,
ExportFormat format,
DateTimeOffset? after = null,
DateTimeOffset? before = null)
Snowflake? after = null,
Snowflake? before = null)
{
var buffer = new StringBuilder();
// Guild and channel names
buffer.Append($"{guild.Name} - {channel.Category} - {channel.Name} [{channel.Id}]");
buffer.Append($"{guild.Name} - {channel.Category.Name} - {channel.Name} [{channel.Id}]");
// Date range
if (after != null || before != null)
if (after is not null || before is not null)
{
buffer.Append(" (");
// Both 'after' and 'before' are set
if (after != null && before != null)
if (after is not null && before is not null)
{
buffer.Append($"{after:yyyy-MM-dd} to {before:yyyy-MM-dd}");
buffer.Append($"{after?.ToDate():yyyy-MM-dd} to {before?.ToDate():yyyy-MM-dd}");
}
// Only 'after' is set
else if (after != null)
else if (after is not null)
{
buffer.Append($"after {after:yyyy-MM-dd}");
buffer.Append($"after {after?.ToDate():yyyy-MM-dd}");
}
// Only 'before' is set
else
{
buffer.Append($"before {before:yyyy-MM-dd}");
buffer.Append($"before {before?.ToDate():yyyy-MM-dd}");
}
buffer.Append(")");
@@ -0,0 +1,115 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Core.Exporting
{
internal partial class MediaDownloader
{
private readonly HttpClient _httpClient;
private readonly string _workingDirPath;
private readonly bool _reuseMedia;
// URL -> Local file path
private readonly Dictionary<string, string> _pathCache = new(StringComparer.Ordinal);
public MediaDownloader(HttpClient httpClient, string workingDirPath, bool reuseMedia)
{
_httpClient = httpClient;
_workingDirPath = workingDirPath;
_reuseMedia = reuseMedia;
}
public MediaDownloader(string workingDirPath, bool reuseMedia)
: this(Http.Client, workingDirPath, reuseMedia) {}
public async ValueTask<string> DownloadAsync(string url)
{
if (_pathCache.TryGetValue(url, out var cachedFilePath))
return cachedFilePath;
var fileName = GetFileNameFromUrl(url);
var filePath = Path.Combine(_workingDirPath, fileName);
// Reuse existing files if we're allowed to
if (_reuseMedia && File.Exists(filePath))
return _pathCache[url] = filePath;
Directory.CreateDirectory(_workingDirPath);
// This retries on IOExceptions which is dangerous as we're also working with files
await Http.ExceptionPolicy.ExecuteAsync(async () =>
{
// Download the file
using var response = await _httpClient.GetAsync(url);
await using (var output = File.Create(filePath))
{
await response.Content.CopyToAsync(output);
}
// Try to set the file date according to the last-modified header
try
{
var lastModified = response.Content.Headers.TryGetValue("Last-Modified")?.Pipe(s =>
DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)
? date
: (DateTimeOffset?) null
);
if (lastModified is not null)
{
File.SetCreationTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastWriteTimeUtc(filePath, lastModified.Value.UtcDateTime);
File.SetLastAccessTimeUtc(filePath, lastModified.Value.UtcDateTime);
}
}
catch
{
// This can apparently fail for some reason.
// https://github.com/Tyrrrz/DiscordChatExporter/issues/585
// Updating file dates is not a critical task, so we'll just
// ignore exceptions thrown here.
}
});
return _pathCache[url] = filePath;
}
}
internal partial class MediaDownloader
{
private static string GetUrlHash(string url)
{
using var hash = SHA256.Create();
var data = hash.ComputeHash(Encoding.UTF8.GetBytes(url));
return data.ToHex().Truncate(5); // 5 chars ought to be enough for anybody
}
private static string GetFileNameFromUrl(string url)
{
var urlHash = GetUrlHash(url);
// Try to extract file name from URL
var fileName = Regex.Match(url, @".+/([^?]*)").Groups[1].Value;
// If it's not there, just use the URL hash as the file name
if (string.IsNullOrWhiteSpace(fileName))
return urlHash;
// Otherwise, use the original file name but inject the hash in the middle
var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
var fileExtension = Path.GetExtension(fileName);
return PathEx.EscapePath(fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension);
}
}
}
@@ -1,16 +1,15 @@
using System;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers;
namespace DiscordChatExporter.Domain.Exporting
namespace DiscordChatExporter.Core.Exporting
{
internal partial class MessageExporter : IAsyncDisposable
{
private readonly ExportContext _context;
private long _messageCount;
private int _partitionIndex;
private MessageWriter? _writer;
@@ -19,15 +18,9 @@ namespace DiscordChatExporter.Domain.Exporting
_context = context;
}
private bool IsPartitionLimitReached() =>
_messageCount > 0 &&
_context.Request.PartitionLimit != null &&
_context.Request.PartitionLimit != 0 &&
_messageCount % _context.Request.PartitionLimit == 0;
private async ValueTask ResetWriterAsync()
{
if (_writer != null)
if (_writer is not null)
{
await _writer.WritePostambleAsync();
await _writer.DisposeAsync();
@@ -38,14 +31,15 @@ namespace DiscordChatExporter.Domain.Exporting
private async ValueTask<MessageWriter> GetWriterAsync()
{
// Ensure partition limit has not been exceeded
if (IsPartitionLimitReached())
if (_writer is not null &&
_context.Request.PartitionLimit.IsReached(_writer.MessagesWritten, _writer.BytesWritten))
{
await ResetWriterAsync();
_partitionIndex++;
}
// Writer is still valid - return
if (_writer != null)
if (_writer is not null)
return _writer;
var filePath = GetPartitionFilePath(_context.Request.OutputBaseFilePath, _partitionIndex);
@@ -64,7 +58,6 @@ namespace DiscordChatExporter.Domain.Exporting
{
var writer = await GetWriterAsync();
await writer.WriteMessageAsync(message);
_messageCount++;
}
public async ValueTask DisposeAsync() => await ResetWriterAsync();
@@ -82,16 +75,17 @@ namespace DiscordChatExporter.Domain.Exporting
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(baseFilePath);
var fileExt = Path.GetExtension(baseFilePath);
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
// Generate new path
var dirPath = Path.GetDirectoryName(baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
return Path.Combine(dirPath, fileName);
return fileName;
return !string.IsNullOrWhiteSpace(dirPath)
? Path.Combine(dirPath, fileName)
: fileName;
}
private static MessageWriter CreateMessageWriter(string filePath, ExportFormat format, ExportContext context)
private static MessageWriter CreateMessageWriter(
string filePath,
ExportFormat format,
ExportContext context)
{
// Stream will be disposed by the underlying writer
var stream = File.Create(filePath);
@@ -0,0 +1,12 @@
namespace DiscordChatExporter.Core.Exporting.Partitioning
{
public class FileSizePartitionLimit : PartitionLimit
{
private readonly long _limit;
public FileSizePartitionLimit(long limit) => _limit = limit;
public override bool IsReached(long messagesWritten, long bytesWritten) =>
bytesWritten >= _limit;
}
}
@@ -0,0 +1,12 @@
namespace DiscordChatExporter.Core.Exporting.Partitioning
{
public class MessageCountPartitionLimit : PartitionLimit
{
private readonly long _limit;
public MessageCountPartitionLimit(long limit) => _limit = limit;
public override bool IsReached(long messagesWritten, long bytesWritten) =>
messagesWritten >= _limit;
}
}
@@ -0,0 +1,9 @@
namespace DiscordChatExporter.Core.Exporting.Partitioning
{
public class NullPartitionLimit : PartitionLimit
{
public static NullPartitionLimit Instance { get; } = new();
public override bool IsReached(long messagesWritten, long bytesWritten) => false;
}
}
@@ -0,0 +1,61 @@
using System;
using System.Globalization;
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Core.Exporting.Partitioning
{
public abstract partial class PartitionLimit
{
public abstract bool IsReached(long messagesWritten, long bytesWritten);
}
public partial class PartitionLimit
{
private static long? TryParseFileSizeBytes(string value, IFormatProvider? formatProvider = null)
{
var match = Regex.Match(value, @"^\s*(\d+[\.,]?\d*)\s*(\w)?b\s*$", RegexOptions.IgnoreCase);
// Number part
if (!double.TryParse(
match.Groups[1].Value,
NumberStyles.Float,
formatProvider,
out var number))
{
return null;
}
// Magnitude part
var magnitude = match.Groups[2].Value.ToUpperInvariant() switch
{
"G" => 1_000_000_000,
"M" => 1_000_000,
"K" => 1_000,
"" => 1,
_ => -1
};
if (magnitude < 0)
{
return null;
}
return (long) (number * magnitude);
}
public static PartitionLimit? TryParse(string value, IFormatProvider? formatProvider = null)
{
var fileSizeLimit = TryParseFileSizeBytes(value, formatProvider);
if (fileSizeLimit is not null)
return new FileSizePartitionLimit(fileSizeLimit.Value);
if (int.TryParse(value, NumberStyles.Integer, formatProvider, out var messageCountLimit))
return new MessageCountPartitionLimit(messageCountLimit);
return null;
}
public static PartitionLimit Parse(string value, IFormatProvider? formatProvider = null) =>
TryParse(value, formatProvider) ?? throw new FormatException($"Invalid partition limit '{value}'.");
}
}
@@ -2,11 +2,11 @@
using System.IO;
using System.Text;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal.Extensions;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers
namespace DiscordChatExporter.Core.Exporting.Writers
{
internal partial class CsvMessageWriter : MessageWriter
{
@@ -58,8 +58,10 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public override async ValueTask WriteMessageAsync(Message message)
{
await base.WriteMessageAsync(message);
// Author ID
await _writer.WriteAsync(CsvEncode(message.Author.Id));
await _writer.WriteAsync(CsvEncode(message.Author.Id.ToString()));
await _writer.WriteAsync(',');
// Author name
@@ -49,13 +49,18 @@ img {
.markdown {
max-width: 100%;
white-space: pre-wrap;
line-height: 1.3;
overflow-wrap: break-word;
}
.preserve-whitespace {
white-space: pre-wrap;
}
.spoiler {
width: fit-content;
/* width: fit-content; */
display: inline-block;
/* This is more consistent across browsers, the old attribute worked well under Chrome but not FireFox. */
}
.spoiler--hidden {
@@ -84,7 +89,7 @@ img {
box-shadow: 0 0 1px 1px rgba(0, 0, 0, 0.1);
}
.spoiler--hidden .spoiler-image img {
.spoiler--hidden .spoiler-image * {
filter: blur(44px);
}
@@ -97,7 +102,7 @@ img {
top: 50%;
transform: translate(-50%, -50%);
font-weight: 600;
padding: 0.5em 0.7em;
padding: 100%;
border-radius: 20px;
letter-spacing: 0.05em;
font-size: 0.9em;
@@ -141,8 +146,8 @@ img {
}
.emoji {
width: 1.25em;
height: 1.25em;
width: 1.325em;
height: 1.325em;
margin: 0 0.06em;
vertical-align: -0.4em;
}
@@ -202,6 +207,65 @@ img {
grid-template-columns: auto 1fr;
}
.chatlog__reference-symbol {
grid-column: 1;
border-style: solid;
border-width: 2px 0 0 2px;
border-radius: 8px 0 0 0;
margin-left: 16px;
margin-top: 8px;
}
.chatlog__reference {
display: flex;
grid-column: 2;
margin-left: 1.2em;
margin-bottom: 0.25em;
font-size: 0.875em;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
align-items: center;
}
.chatlog__reference-avatar {
border-radius: 50%;
height: 16px;
width: 16px;
margin-right: 0.25em;
}
.chatlog__reference-name {
margin-right: 0.25em;
font-weight: 600;
}
.chatlog__reference-link {
flex-grow: 1;
overflow: hidden;
text-overflow: ellipsis;
cursor: pointer;
}
.chatlog__reference-content > * {
display: inline;
pointer-events: none;
}
.chatlog__reference-content > a:hover {
text-decoration: none;
}
.chatlog__reference-content > .emoji {
width: 1.325em;
height: 1.325em;
}
.chatlog__reference-edited-timestamp {
margin-left: 0.25em;
font-size: 0.8em;
}
.chatlog__author-avatar-container {
grid-column: 1;
width: 40px;
@@ -257,6 +321,53 @@ img {
border-radius: 3px;
}
.chatlog__attachment-container {
height: 40px;
width: 100%;
max-width: 520px;
padding: 10px;
border: 1px solid;
border-radius: 3px;
overflow: hidden;
}
.chatlog__attachment-icon {
float: left;
height: 100%;
margin-right: 10px;
}
.chatlog__attachment-icon > .a {
fill: #f4f5fb;
d: path("M50,935a25,25,0,0,1-25-25V50A25,25,0,0,1,50,25H519.6L695,201.32V910a25,25,0,0,1-25,25Z");
}
.chatlog__attachment-icon > .b {
fill: #7789c4;
d: path("M509.21,50,670,211.63V910H50V50H509.21M530,0H50A50,50,0,0,0,0,50V910a50,50,0,0,0,50,50H670a50,50,0,0,0,50-50h0V191Z");
}
.chatlog__attachment-icon > .c {
fill: #f4f5fb;
d: path("M530,215a25,25,0,0,1-25-25V50a25,25,0,0,1,16.23-23.41L693.41,198.77A25,25,0,0,1,670,215Z");
}
.chatlog__attachment-icon > .d {
fill: #7789c4;
d: path("M530,70.71,649.29,190H530V70.71M530,0a50,50,0,0,0-50,50V190a50,50,0,0,0,50,50H670a50,50,0,0,0,50-50Z");
}
.chatlog__attachment-filesize {
color: #72767d;
font-size: 12px;
}
.chatlog__attachment-filename {
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
}
.chatlog__embed {
display: flex;
margin-top: 0.3em;
@@ -376,10 +487,20 @@ img {
}
.chatlog__embed-footer-text {
vertical-align: middle;
font-size: 0.75em;
font-weight: 500;
}
.chatlog__embed-youtube-container {
margin-top: 0.6em;
}
.chatlog__embed-youtube {
border: 0;
border-radius: 3px;
}
.chatlog__reactions {
display: flex;
}
@@ -406,7 +527,7 @@ img {
border-radius: 3px;
vertical-align: middle;
line-height: 1.3;
background: #7289da;
background: #5865F2;
color: #ffffff;
font-size: 0.625em;
font-weight: 500;
@@ -6,7 +6,7 @@ body {
}
a {
color: #0096cf;
color: #00aff4;
}
.spoiler-text {
@@ -34,7 +34,7 @@ a {
color: #b9bbbe !important;
}
/* === Preamble === */
/* Preamble */
.preamble__entry {
color: #ffffff;
@@ -46,6 +46,26 @@ a {
border-color: rgba(255, 255, 255, 0.1);
}
.chatlog__reference-symbol {
border-color: #4f545c;
}
.chatlog__reference {
color: #b5b6b8;
}
.chatlog__reference-link {
color: #b5b6b8;
}
.chatlog__reference-link:hover {
color: #ffffff;
}
.chatlog__reference-edited-timestamp {
color: rgba(255, 255, 255, 0.2);
}
.chatlog__author-name {
color: #ffffff;
}
@@ -62,12 +82,17 @@ a {
background-color: rgba(249, 168, 37, 0.05);
}
.chatlog__attachment-container {
background-color: #2f3136;
border-color: #292b2f;
}
.chatlog__edited-timestamp {
color: rgba(255, 255, 255, 0.2);
}
.chatlog__embed-color-pill--default {
background-color: rgba(79, 84, 92, 1);
background-color: #202225;
}
.chatlog__embed-content-container {
@@ -88,7 +113,7 @@ a {
}
.chatlog__embed-description {
color: rgba(255, 255, 255, 0.6);
color: #dcddde;
}
.chatlog__embed-field-name {
@@ -96,11 +121,11 @@ a {
}
.chatlog__embed-field-value {
color: rgba(255, 255, 255, 0.6);
color: #dcddde;
}
.chatlog__embed-footer {
color: rgba(255, 255, 255, 0.6);
color: #dcddde;
}
.chatlog__reaction {
@@ -7,7 +7,7 @@ body {
}
a {
color: #00b0f4;
color: #0068e0;
}
.spoiler-text {
@@ -47,6 +47,26 @@ a {
border-color: #eceeef;
}
.chatlog__reference-symbol {
border-color: #c7ccd1;
}
.chatlog__reference {
color: #5f5f60;
}
.chatlog__reference-link {
color: #5f5f60;
}
.chatlog__reference-link:hover {
color: #2f3136;
}
.chatlog__reference-edited-timestamp {
color: #747f8d;
}
.chatlog__author-name {
font-weight: 600;
color: #2f3136;
@@ -64,6 +84,11 @@ a {
background-color: rgba(249, 168, 37, 0.05);
}
.chatlog__attachment-container {
background-color: #f2f3f5;
border-color: #ebedef;
}
.chatlog__edited-timestamp {
color: #747f8d;
}
@@ -90,7 +115,7 @@ a {
}
.chatlog__embed-description {
color: #737f8d;
color: #2e3338;
}
.chatlog__embed-field-name {
@@ -98,11 +123,11 @@ a {
}
.chatlog__embed-field-value {
color: #737f8d;
color: #2e3338;
}
.chatlog__embed-footer {
color: rgba(79, 83, 91, 0.6);
color: #2e3338;
}
.chatlog__reaction {
@@ -1,9 +1,9 @@
using System;
using System.Collections.Generic;
using System.Linq;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Domain.Exporting.Writers.Html
namespace DiscordChatExporter.Core.Exporting.Writers.Html
{
// Used for grouping contiguous messages in HTML export
internal partial class MessageGroup
@@ -14,10 +14,21 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.Html
public IReadOnlyList<Message> Messages { get; }
public MessageGroup(User author, DateTimeOffset timestamp, IReadOnlyList<Message> messages)
public MessageReference? Reference { get; }
public Message? ReferencedMessage {get; }
public MessageGroup(
User author,
DateTimeOffset timestamp,
MessageReference? reference,
Message? referencedMessage,
IReadOnlyList<Message> messages)
{
Author = author;
Timestamp = timestamp;
Reference = reference;
ReferencedMessage = referencedMessage;
Messages = messages;
}
}
@@ -25,9 +36,14 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.Html
internal partial class MessageGroup
{
public static bool CanJoin(Message message1, Message message2) =>
string.Equals(message1.Author.Id, message2.Author.Id, StringComparison.Ordinal) &&
// Must be from the same author
message1.Author.Id == message2.Author.Id &&
// Author's name must not have changed between messages
string.Equals(message1.Author.FullName, message2.Author.FullName, StringComparison.Ordinal) &&
(message2.Timestamp - message1.Timestamp).Duration().TotalMinutes <= 7;
// Duration between messages must be 7 minutes or less
(message2.Timestamp - message1.Timestamp).Duration().TotalMinutes <= 7 &&
// Other message must not be a reply
message2.Reference is null;
public static MessageGroup Join(IReadOnlyList<Message> messages)
{
@@ -36,6 +52,8 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.Html
return new MessageGroup(
first.Author,
first.Timestamp,
first.Reference,
first.ReferencedMessage,
messages
);
}
@@ -1,7 +1,8 @@
@using System
@using System.Linq
@using System.Threading.Tasks
@inherits MiniRazor.MiniRazorTemplateBase<DiscordChatExporter.Domain.Exporting.Writers.Html.MessageGroupTemplateContext>
@namespace DiscordChatExporter.Core.Exporting.Writers.Html
@inherits MiniRazor.TemplateBase<DiscordChatExporter.Core.Exporting.Writers.Html.MessageGroupTemplateContext>
@{
string FormatDate(DateTimeOffset date) => Model.ExportContext.FormatDate(date);
@@ -13,15 +14,72 @@
ValueTask<string> ResolveUrlAsync(string url) => Model.ExportContext.ResolveMediaUrlAsync(url);
var userMember = Model.ExportContext.TryGetMember(Model.MessageGroup.Author.Id);
var userColor = Model.ExportContext.TryGetUserColor(Model.MessageGroup.Author.Id);
var userNick = (Model.MessageGroup.Author.IsBot ? Model.MessageGroup.Author.Name : (userMember?.Nick ?? Model.MessageGroup.Author.Name));
var userColorStyle = userColor != null
var userColor = Model.ExportContext.TryGetUserColor(Model.MessageGroup.Author.Id);
var userColorStyle = userColor is not null
? $"color: rgb({userColor?.R},{userColor?.G},{userColor?.B})"
: null;
var userNick = Model.MessageGroup.Author.IsBot
? Model.MessageGroup.Author.Name
: userMember?.Nick ?? Model.MessageGroup.Author.Name;
var referencedUserMember = Model.MessageGroup.ReferencedMessage is not null
? Model.ExportContext.TryGetMember(Model.MessageGroup.ReferencedMessage.Author.Id)
: null;
var referencedUserColor = Model.MessageGroup.ReferencedMessage is not null
? Model.ExportContext.TryGetUserColor(Model.MessageGroup.ReferencedMessage.Author.Id)
: null;
var referencedUserColorStyle = referencedUserColor is not null
? $"color: rgb({referencedUserColor?.R},{referencedUserColor?.G},{referencedUserColor?.B})"
: null;
var referencedUserNick = Model.MessageGroup.ReferencedMessage is not null
? Model.MessageGroup.ReferencedMessage.Author.IsBot
? Model.MessageGroup.ReferencedMessage.Author.Name
: referencedUserMember?.Nick ?? Model.MessageGroup.ReferencedMessage.Author.Name
: null;
}
<div class="chatlog__message-group">
@if (Model.MessageGroup.Reference is not null)
{
<div class="chatlog__reference-symbol">
</div>
<div class="chatlog__reference">
@if (Model.MessageGroup.ReferencedMessage is not null)
{
<img class="chatlog__reference-avatar" src="@await ResolveUrlAsync(Model.MessageGroup.ReferencedMessage.Author.AvatarUrl)" alt="Avatar">
<span class="chatlog__reference-name" title="@Model.MessageGroup.ReferencedMessage.Author.FullName" style="@referencedUserColorStyle">@referencedUserNick</span>
<div class="chatlog__reference-link" onclick="scrollToMessage(event, '@Model.MessageGroup.ReferencedMessage.Id')">
<span class="chatlog__reference-content">
@if (!string.IsNullOrWhiteSpace(Model.MessageGroup.ReferencedMessage.Content))
{
@Raw(FormatMarkdown(Model.MessageGroup.ReferencedMessage.Content))
}
else
{
<em>Click to see original message</em>
}
</span>
@if (Model.MessageGroup.ReferencedMessage.EditedTimestamp is not null)
{
<span class="chatlog__reference-edited-timestamp" title="@FormatDate(Model.MessageGroup.ReferencedMessage.EditedTimestamp.Value)">(edited)</span>
}
</div>
}
else
{
<span class="chatlog__reference-unknown">
Original message was deleted.
</span>
}
</div>
}
<div class="chatlog__author-avatar-container">
<img class="chatlog__author-avatar" src="@await ResolveUrlAsync(Model.MessageGroup.Author.AvatarUrl)" alt="Avatar">
</div>
@@ -39,44 +97,74 @@
{
var isPinnedStyle = message.IsPinned ? "chatlog__message--pinned" : null;
<div class="chatlog__message @isPinnedStyle" data-message-id="@message.Id" id="message-@message.Id">
<div class="chatlog__content">
<div class="markdown">@Raw(FormatMarkdown(message.Content)) @if (message.EditedTimestamp != null) {<span class="chatlog__edited-timestamp" title="@FormatDate(message.EditedTimestamp.Value)">(edited)</span>}</div>
</div>
<div class="chatlog__message @isPinnedStyle" data-message-id="@message.Id" id="message-@message.Id" title="Message sent: @FormatDate(message.Timestamp)">
@if (!string.IsNullOrWhiteSpace(message.Content) || message.EditedTimestamp is not null)
{
<div class="chatlog__content">
<div class="markdown">
<span class="preserve-whitespace">@Raw(FormatMarkdown(message.Content))</span>
@if (message.EditedTimestamp is not null)
{
<span class="chatlog__edited-timestamp" title="@FormatDate(message.EditedTimestamp.Value)">(edited)</span>
}
</div>
</div>
}
@foreach (var attachment in message.Attachments)
{
<div class="chatlog__attachment">
@if (attachment.IsSpoiler)
{
<div class="spoiler spoiler--hidden" onclick="showSpoiler(event, this)">
<div class="spoiler-image">
<a href="@await ResolveUrlAsync(attachment.Url)">
<img class="chatlog__attachment-thumbnail" src="@await ResolveUrlAsync(attachment.Url)" alt="Attachment">
</a>
</div>
</div>
}
else
{
<a href="@await ResolveUrlAsync(attachment.Url)">
<div class="@(attachment.IsSpoiler ? "spoiler spoiler--hidden" : "")" onclick="@(attachment.IsSpoiler ? "showSpoiler(event, this)" : "")">
<div class="@(attachment.IsSpoiler ? "spoiler-image" : "")">
@if (attachment.IsImage)
{
<img class="chatlog__attachment-thumbnail" src="@await ResolveUrlAsync(attachment.Url)" alt="Attachment">
<a href="@await ResolveUrlAsync(attachment.Url)">
<img class="chatlog__attachment-thumbnail" src="@await ResolveUrlAsync(attachment.Url)" alt="Image attachment" title="@($"Image: {attachment.FileName} ({attachment.FileSize})")">
</a>
}
else if (attachment.IsVideo)
{
<video controls class="chatlog__attachment-thumbnail">
<source src="@await ResolveUrlAsync(attachment.Url)" alt="Video attachment" title="@($"Video: {attachment.FileName} ({attachment.FileSize})")">
</video>
}
else if (attachment.IsAudio)
{
<audio controls class="chatlog__attachment-thumbnail">
<source src="@await ResolveUrlAsync(attachment.Url)" alt="Audio attachment" title="@($"Audio: {attachment.FileName} ({attachment.FileSize})")">
</audio>
}
else
{
@($"Attachment: {attachment.FileName} ({attachment.FileSize})")
<div class="chatlog__attachment-container">
<svg class="chatlog__attachment-icon" viewBox="0 0 720 960">
<path class="a"/>
<path class="b"/>
<path class="c"/>
<path class="d"/>
</svg>
<div class="chatlog__attachment-filename">
<a href="@await ResolveUrlAsync(attachment.Url)">
@attachment.FileName
</a>
</div>
<div class="chatlog__attachment-filesize">
@attachment.FileSize
</div>
</div>
}
</a>
}
</div>
</div>
</div>
}
@foreach (var embed in message.Embeds)
{
var youTubeVideo = embed.TryGetYouTubeVideo();
<div class="chatlog__embed">
@if (embed.Color != null)
@if (embed.Color is not null)
{
var embedColorStyle = $"background-color: rgba({embed.Color?.R},{embed.Color?.G},{embed.Color?.B},{embed.Color?.A})";
<div class="chatlog__embed-color-pill" style="@embedColorStyle"></div>
@@ -89,12 +177,12 @@
<div class="chatlog__embed-content-container">
<div class="chatlog__embed-content">
<div class="chatlog__embed-text">
@if (embed.Author != null)
@if (embed.Author is not null)
{
<div class="chatlog__embed-author">
@if (!string.IsNullOrWhiteSpace(embed.Author.IconUrl))
{
<img class="chatlog__embed-author-icon" src="@await ResolveUrlAsync(embed.Author.IconUrl)" alt="Author icon">
<img class="chatlog__embed-author-icon" src="@await ResolveUrlAsync(embed.Author.IconProxyUrl ?? embed.Author.IconUrl)" alt="Author icon">
}
@if (!string.IsNullOrWhiteSpace(embed.Author.Name))
@@ -119,20 +207,26 @@
@if (!string.IsNullOrWhiteSpace(embed.Url))
{
<a class="chatlog__embed-title-link" href="@embed.Url">
<div class="markdown">@Raw(FormatEmbedMarkdown(embed.Title))</div>
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(embed.Title))</div>
</a>
}
else
{
<div class="markdown">@Raw(FormatEmbedMarkdown(embed.Title))</div>
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(embed.Title))</div>
}
</div>
}
@if (!string.IsNullOrWhiteSpace(embed.Description))
@if (youTubeVideo is not null)
{
<div class="chatlog__embed-youtube-container">
<iframe class="chatlog__embed-youtube" src="@youTubeVideo.Url" width="400" height="225"></iframe>
</div>
}
else if (!string.IsNullOrWhiteSpace(embed.Description))
{
<div class="chatlog__embed-description">
<div class="markdown">@Raw(FormatEmbedMarkdown(embed.Description))</div>
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(embed.Description))</div>
</div>
}
@@ -147,14 +241,14 @@
@if (!string.IsNullOrWhiteSpace(field.Name))
{
<div class="chatlog__embed-field-name">
<div class="markdown">@Raw(FormatEmbedMarkdown(field.Name))</div>
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(field.Name))</div>
</div>
}
@if (!string.IsNullOrWhiteSpace(field.Value))
{
<div class="chatlog__embed-field-value">
<div class="markdown">@Raw(FormatEmbedMarkdown(field.Value))</div>
<div class="markdown preserve-whitespace">@Raw(FormatEmbedMarkdown(field.Value))</div>
</div>
}
</div>
@@ -163,49 +257,49 @@
}
</div>
@if (embed.Thumbnail != null)
@if (embed.Thumbnail is not null && !string.IsNullOrWhiteSpace(embed.Thumbnail.Url) && youTubeVideo is null)
{
<div class="chatlog__embed-thumbnail-container">
<a class="chatlog__embed-thumbnail-link" href="@await ResolveUrlAsync(embed.Thumbnail.Url)">
<img class="chatlog__embed-thumbnail" src="@await ResolveUrlAsync(embed.Thumbnail.Url)" alt="Thumbnail">
<a class="chatlog__embed-thumbnail-link" href="@await ResolveUrlAsync(embed.Thumbnail.ProxyUrl ?? embed.Thumbnail.Url)">
<img class="chatlog__embed-thumbnail" src="@await ResolveUrlAsync(embed.Thumbnail.ProxyUrl ?? embed.Thumbnail.Url)" alt="Thumbnail">
</a>
</div>
}
</div>
@if (embed.Image != null)
@if (embed.Image is not null && !string.IsNullOrWhiteSpace(embed.Image.Url))
{
<div class="chatlog__embed-image-container">
<a class="chatlog__embed-image-link" href="@await ResolveUrlAsync(embed.Image.Url)">
<img class="chatlog__embed-image" src="@await ResolveUrlAsync(embed.Image.Url)" alt="Image">
<a class="chatlog__embed-image-link" href="@await ResolveUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url)">
<img class="chatlog__embed-image" src="@await ResolveUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url)" alt="Image">
</a>
</div>
}
@if (embed.Footer != null || embed.Timestamp != null)
@if (embed.Footer is not null || embed.Timestamp is not null)
{
<div class="chatlog__embed-footer">
@if (!string.IsNullOrWhiteSpace(embed.Footer?.IconUrl))
{
<img class="chatlog__embed-footer-icon" src="@await ResolveUrlAsync(embed.Footer.IconUrl)" alt="Footer icon">
<img class="chatlog__embed-footer-icon" src="@await ResolveUrlAsync(embed.Footer.IconProxyUrl ?? embed.Footer.IconUrl)" alt="Footer icon">
}
<span class="chatlog__embed-footer-text">
@if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
{
@embed.Footer.Text
}
<span class="chatlog__embed-footer-text">
@if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
{
@embed.Footer.Text
}
@if (!string.IsNullOrWhiteSpace(embed.Footer?.Text) && embed.Timestamp != null)
{
@(" • ")
}
@if (!string.IsNullOrWhiteSpace(embed.Footer?.Text) && embed.Timestamp is not null)
{
@(" • ")
}
@if (embed.Timestamp != null)
{
@FormatDate(embed.Timestamp.Value)
}
</span>
@if (embed.Timestamp is not null)
{
@FormatDate(embed.Timestamp.Value)
}
</span>
</div>
}
</div>
@@ -217,8 +311,8 @@
<div class="chatlog__reactions">
@foreach (var reaction in message.Reactions)
{
<div class="chatlog__reaction">
<img class="emoji emoji--small" alt="@reaction.Emoji.Name" title="@reaction.Emoji.Name" src="@await ResolveUrlAsync(reaction.Emoji.ImageUrl)">
<div class="chatlog__reaction" title="@reaction.Emoji.Code">
<img class="emoji emoji--small" alt="@reaction.Emoji.Name" src="@await ResolveUrlAsync(reaction.Emoji.ImageUrl)">
<span class="chatlog__reaction-count">@reaction.Count</span>
</div>
}
@@ -1,6 +1,6 @@
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors;
namespace DiscordChatExporter.Domain.Exporting.Writers.Html
namespace DiscordChatExporter.Core.Exporting.Writers.Html
{
internal class MessageGroupTemplateContext
{
@@ -0,0 +1,12 @@
@namespace DiscordChatExporter.Core.Exporting.Writers.Html
@inherits MiniRazor.TemplateBase<DiscordChatExporter.Core.Exporting.Writers.Html.PostambleTemplateContext>
</div>
<div class="postamble">
<div class="postamble__entry">Exported @Model.MessagesWritten.ToString("N0") message(s)</div>
</div>
</body>
</html>
@@ -0,0 +1,15 @@
namespace DiscordChatExporter.Core.Exporting.Writers.Html
{
internal class PostambleTemplateContext
{
public ExportContext ExportContext { get; }
public long MessagesWritten { get; }
public PostambleTemplateContext(ExportContext exportContext, long messagesWritten)
{
ExportContext = exportContext;
MessagesWritten = messagesWritten;
}
}
}
@@ -1,14 +1,15 @@
@using System
@using System.Threading.Tasks
@using Tyrrrz.Extensions
@inherits MiniRazor.MiniRazorTemplateBase<DiscordChatExporter.Domain.Exporting.Writers.Html.LayoutTemplateContext>
@namespace DiscordChatExporter.Core.Exporting.Writers.Html
@inherits MiniRazor.TemplateBase<DiscordChatExporter.Core.Exporting.Writers.Html.PreambleTemplateContext>
@{
string FormatDate(DateTimeOffset date) => Model.ExportContext.FormatDate(date);
ValueTask<string> ResolveUrlAsync(string url) => Model.ExportContext.ResolveMediaUrlAsync(url);
string GetStyleSheet(string name) => Model.GetType().Assembly.GetManifestResourceString($"DiscordChatExporter.Domain.Exporting.Writers.Html.{name}.css");
string GetStyleSheet(string name) => Model.GetType().Assembly.GetManifestResourceString($"DiscordChatExporter.Core.Exporting.Writers.Html.{name}.css");
}
<!DOCTYPE html>
@@ -77,20 +78,20 @@
<div class="preamble__entry preamble__entry--small">@Model.ExportContext.Request.Channel.Topic</div>
}
@if (Model.ExportContext.Request.After != null || Model.ExportContext.Request.Before != null)
@if (Model.ExportContext.Request.After is not null || Model.ExportContext.Request.Before is not null)
{
<div class="preamble__entry preamble__entry--small">
@if (Model.ExportContext.Request.After != null && Model.ExportContext.Request.Before != null)
@if (Model.ExportContext.Request.After is not null && Model.ExportContext.Request.Before is not null)
{
@($"Between {FormatDate(Model.ExportContext.Request.After.Value)} and {FormatDate(Model.ExportContext.Request.Before.Value)}")
@($"Between {FormatDate(Model.ExportContext.Request.After.Value.ToDate())} and {FormatDate(Model.ExportContext.Request.Before.Value.ToDate())}")
}
else if (Model.ExportContext.Request.After != null)
else if (Model.ExportContext.Request.After is not null)
{
@($"After {FormatDate(Model.ExportContext.Request.After.Value)}")
@($"After {FormatDate(Model.ExportContext.Request.After.Value.ToDate())}")
}
else if (Model.ExportContext.Request.Before != null)
else if (Model.ExportContext.Request.Before is not null)
{
@($"Before {FormatDate(Model.ExportContext.Request.Before.Value)}")
@($"Before {FormatDate(Model.ExportContext.Request.Before.Value.ToDate())}")
}
</div>
}
@@ -0,0 +1,15 @@
namespace DiscordChatExporter.Core.Exporting.Writers.Html
{
internal class PreambleTemplateContext
{
public ExportContext ExportContext { get; }
public string ThemeName { get; }
public PreambleTemplateContext(ExportContext exportContext, string themeName)
{
ExportContext = exportContext;
ThemeName = themeName;
}
}
}
@@ -2,19 +2,17 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.Html;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.Html;
namespace DiscordChatExporter.Domain.Exporting.Writers
namespace DiscordChatExporter.Core.Exporting.Writers
{
internal class HtmlMessageWriter : MessageWriter
{
private readonly TextWriter _writer;
private readonly string _themeName;
private readonly List<Message> _messageGroupBuffer = new List<Message>();
private long _messageCount;
private readonly List<Message> _messageGroupBuffer = new();
public HtmlMessageWriter(Stream stream, ExportContext context, string themeName)
: base(stream, context)
@@ -25,26 +23,26 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public override async ValueTask WritePreambleAsync()
{
var templateContext = new LayoutTemplateContext(Context, _themeName, _messageCount);
var templateBundle = await TemplateBundle.ResolveAsync();
var templateContext = new PreambleTemplateContext(Context, _themeName);
await _writer.WriteLineAsync(
await templateBundle.PreambleTemplate.RenderAsync(templateContext)
await PreambleTemplate.RenderAsync(templateContext)
);
}
private async ValueTask WriteMessageGroupAsync(MessageGroup messageGroup)
{
var templateContext = new MessageGroupTemplateContext(Context, messageGroup);
var templateBundle = await TemplateBundle.ResolveAsync();
await _writer.WriteLineAsync(
await templateBundle.MessageGroupTemplate.RenderAsync(templateContext)
await MessageGroupTemplate.RenderAsync(templateContext)
);
}
public override async ValueTask WriteMessageAsync(Message message)
{
await base.WriteMessageAsync(message);
// If message group is empty or the given message can be grouped, buffer the given message
if (!_messageGroupBuffer.Any() || MessageGroup.CanJoin(_messageGroupBuffer.Last(), message))
{
@@ -58,9 +56,6 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_messageGroupBuffer.Clear();
_messageGroupBuffer.Add(message);
}
// Increment message count
_messageCount++;
}
public override async ValueTask WritePostambleAsync()
@@ -69,11 +64,10 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
if (_messageGroupBuffer.Any())
await WriteMessageGroupAsync(MessageGroup.Join(_messageGroupBuffer));
var templateContext = new LayoutTemplateContext(Context, _themeName, _messageCount);
var templateBundle = await TemplateBundle.ResolveAsync();
var templateContext = new PostambleTemplateContext(Context, MessagesWritten);
await _writer.WriteLineAsync(
await templateBundle.PostambleTemplate.RenderAsync(templateContext)
await PostambleTemplate.RenderAsync(templateContext)
);
}
@@ -1,24 +1,28 @@
using System.IO;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal.Extensions;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Writing;
namespace DiscordChatExporter.Domain.Exporting.Writers
namespace DiscordChatExporter.Core.Exporting.Writers
{
internal class JsonMessageWriter : MessageWriter
{
private readonly Utf8JsonWriter _writer;
private long _messageCount;
public JsonMessageWriter(Stream stream, ExportContext context)
: base(stream, context)
{
_writer = new Utf8JsonWriter(stream, new JsonWriterOptions
{
Indented = true
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = true,
// Validation errors may mask actual failures
// https://github.com/Tyrrrz/DiscordChatExporter/issues/413
SkipValidation = true
});
}
@@ -29,7 +33,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
{
_writer.WriteStartObject();
_writer.WriteString("id", attachment.Id);
_writer.WriteString("id", attachment.Id.ToString());
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(attachment.Url));
_writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
@@ -46,7 +50,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteString("url", embedAuthor.Url);
if (!string.IsNullOrWhiteSpace(embedAuthor.IconUrl))
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedAuthor.IconUrl));
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedAuthor.IconProxyUrl ?? embedAuthor.IconUrl));
_writer.WriteEndObject();
await _writer.FlushAsync();
@@ -57,7 +61,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartObject("thumbnail");
if (!string.IsNullOrWhiteSpace(embedThumbnail.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedThumbnail.Url));
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedThumbnail.ProxyUrl ?? embedThumbnail.Url));
_writer.WriteNumber("width", embedThumbnail.Width);
_writer.WriteNumber("height", embedThumbnail.Height);
@@ -71,7 +75,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartObject("image");
if (!string.IsNullOrWhiteSpace(embedImage.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedImage.Url));
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedImage.ProxyUrl ?? embedImage.Url));
_writer.WriteNumber("width", embedImage.Width);
_writer.WriteNumber("height", embedImage.Height);
@@ -87,7 +91,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteString("text", embedFooter.Text);
if (!string.IsNullOrWhiteSpace(embedFooter.IconUrl))
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedFooter.IconUrl));
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedFooter.IconProxyUrl ?? embedFooter.IconUrl));
_writer.WriteEndObject();
await _writer.FlushAsync();
@@ -114,16 +118,19 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteString("timestamp", embed.Timestamp);
_writer.WriteString("description", FormatMarkdown(embed.Description));
if (embed.Author != null)
if (embed.Color is not null)
_writer.WriteString("color", embed.Color.Value.ToHex());
if (embed.Author is not null)
await WriteEmbedAuthorAsync(embed.Author);
if (embed.Thumbnail != null)
if (embed.Thumbnail is not null)
await WriteEmbedThumbnailAsync(embed.Thumbnail);
if (embed.Image != null)
if (embed.Image is not null)
await WriteEmbedImageAsync(embed.Image);
if (embed.Footer != null)
if (embed.Footer is not null)
await WriteEmbedFooterAsync(embed.Footer);
// Fields
@@ -156,6 +163,20 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
await _writer.FlushAsync();
}
private async ValueTask WriteMentionAsync(User mentionedUser)
{
_writer.WriteStartObject();
_writer.WriteString("id", mentionedUser.Id.ToString());
_writer.WriteString("name", mentionedUser.Name);
_writer.WriteString("discriminator", mentionedUser.DiscriminatorFormatted);
_writer.WriteString("nickname", Context.TryGetMember(mentionedUser.Id)?.Nick ?? mentionedUser.Name);
_writer.WriteBoolean("isBot", mentionedUser.IsBot);
_writer.WriteEndObject();
await _writer.FlushAsync();
}
public override async ValueTask WritePreambleAsync()
{
// Root object (start)
@@ -163,24 +184,25 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Guild
_writer.WriteStartObject("guild");
_writer.WriteString("id", Context.Request.Guild.Id);
_writer.WriteString("id", Context.Request.Guild.Id.ToString());
_writer.WriteString("name", Context.Request.Guild.Name);
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(Context.Request.Guild.IconUrl));
_writer.WriteEndObject();
// Channel
_writer.WriteStartObject("channel");
_writer.WriteString("id", Context.Request.Channel.Id);
_writer.WriteString("id", Context.Request.Channel.Id.ToString());
_writer.WriteString("type", Context.Request.Channel.Type.ToString());
_writer.WriteString("category", Context.Request.Channel.Category);
_writer.WriteString("categoryId", Context.Request.Channel.Category.Id.ToString());
_writer.WriteString("category", Context.Request.Channel.Category.Name);
_writer.WriteString("name", Context.Request.Channel.Name);
_writer.WriteString("topic", Context.Request.Channel.Topic);
_writer.WriteEndObject();
// Date range
_writer.WriteStartObject("dateRange");
_writer.WriteString("after", Context.Request.After);
_writer.WriteString("before", Context.Request.Before);
_writer.WriteString("after", Context.Request.After?.ToDate());
_writer.WriteString("before", Context.Request.Before?.ToDate());
_writer.WriteEndObject();
// Message array (start)
@@ -190,10 +212,12 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public override async ValueTask WriteMessageAsync(Message message)
{
await base.WriteMessageAsync(message);
_writer.WriteStartObject();
// Metadata
_writer.WriteString("id", message.Id);
_writer.WriteString("id", message.Id.ToString());
_writer.WriteString("type", message.Type.ToString());
_writer.WriteString("timestamp", message.Timestamp);
_writer.WriteString("timestampEdited", message.EditedTimestamp);
@@ -205,9 +229,11 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Author
_writer.WriteStartObject("author");
_writer.WriteString("id", message.Author.Id);
_writer.WriteString("id", message.Author.Id.ToString());
_writer.WriteString("name", message.Author.Name);
_writer.WriteString("discriminator", $"{message.Author.Discriminator:0000}");
_writer.WriteString("discriminator", message.Author.DiscriminatorFormatted);
_writer.WriteString("nickname", Context.TryGetMember(message.Author.Id)?.Nick ?? message.Author.Name);
_writer.WriteString("color", Context.TryGetUserColor(message.Author.Id)?.ToHex());
_writer.WriteBoolean("isBot", message.Author.IsBot);
_writer.WriteString("avatarUrl", await Context.ResolveMediaUrlAsync(message.Author.AvatarUrl));
_writer.WriteEndObject();
@@ -236,10 +262,26 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteEndArray();
// Mentions
_writer.WriteStartArray("mentions");
foreach (var mention in message.MentionedUsers)
await WriteMentionAsync(mention);
_writer.WriteEndArray();
// Message reference
if (message.Reference is not null)
{
_writer.WriteStartObject("reference");
_writer.WriteString("messageId", message.Reference.MessageId?.ToString());
_writer.WriteString("channelId", message.Reference.ChannelId?.ToString());
_writer.WriteString("guildId", message.Reference.GuildId?.ToString());
_writer.WriteEndObject();
}
_writer.WriteEndObject();
await _writer.FlushAsync();
_messageCount++;
}
public override async ValueTask WritePostambleAsync()
@@ -247,7 +289,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Message array (end)
_writer.WriteEndArray();
_writer.WriteNumber("messageCount", _messageCount);
_writer.WriteNumber("messageCount", MessagesWritten);
// Root object (end)
_writer.WriteEndObject();
@@ -3,11 +3,13 @@ using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Markdown;
using DiscordChatExporter.Domain.Markdown.Ast;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
{
internal partial class HtmlMarkdownVisitor : MarkdownVisitor
{
@@ -75,6 +77,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
protected override MarkdownNode VisitMention(MentionNode mention)
{
var mentionId = Snowflake.TryParse(mention.Id);
if (mention.Type == MentionType.Meta)
{
_buffer
@@ -84,7 +87,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
}
else if (mention.Type == MentionType.User)
{
var member = _context.TryGetMember(mention.Id);
var member = mentionId?.Pipe(_context.TryGetMember);
var fullName = member?.User.FullName ?? "Unknown";
var nick = member?.Nick ?? "Unknown";
@@ -95,7 +98,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
}
else if (mention.Type == MentionType.Channel)
{
var channel = _context.TryGetChannel(mention.Id);
var channel = mentionId?.Pipe(_context.TryGetChannel);
var name = channel?.Name ?? "deleted-channel";
_buffer
@@ -105,11 +108,11 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
}
else if (mention.Type == MentionType.Role)
{
var role = _context.TryGetRole(mention.Id);
var role = mentionId?.Pipe(_context.TryGetRole);
var name = role?.Name ?? "deleted-role";
var color = role?.Color;
var style = color != null
var style = color is not null
? $"color: rgb({color?.R}, {color?.G}, {color?.B}); background-color: rgba({color?.R}, {color?.G}, {color?.B}, 0.1);"
: "";
@@ -128,7 +131,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
var jumboClass = _isJumbo ? "emoji--large" : "";
_buffer
.Append($"<img class=\"emoji {jumboClass}\" alt=\"{emoji.Name}\" title=\"{emoji.Name}\" src=\"{emojiImageUrl}\">");
.Append($"<img class=\"emoji {jumboClass}\" alt=\"{emoji.Name}\" title=\"{emoji.Code}\" src=\"{emojiImageUrl}\">");
return base.VisitEmoji(emoji);
}
@@ -1,8 +1,10 @@
using System.Text;
using DiscordChatExporter.Domain.Markdown;
using DiscordChatExporter.Domain.Markdown.Ast;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
{
internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
{
@@ -23,27 +25,28 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
protected override MarkdownNode VisitMention(MentionNode mention)
{
var mentionId = Snowflake.TryParse(mention.Id);
if (mention.Type == MentionType.Meta)
{
_buffer.Append($"@{mention.Id}");
}
else if (mention.Type == MentionType.User)
{
var member = _context.TryGetMember(mention.Id);
var member = mentionId?.Pipe(_context.TryGetMember);
var name = member?.User.Name ?? "Unknown";
_buffer.Append($"@{name}");
}
else if (mention.Type == MentionType.Channel)
{
var channel = _context.TryGetChannel(mention.Id);
var channel = mentionId?.Pipe(_context.TryGetChannel);
var name = channel?.Name ?? "deleted-channel";
_buffer.Append($"#{name}");
}
else if (mention.Type == MentionType.Role)
{
var role = _context.TryGetRole(mention.Id);
var role = mentionId?.Pipe(_context.TryGetRole);
var name = role?.Name ?? "deleted-role";
_buffer.Append($"@{name}");
@@ -1,9 +1,9 @@
using System;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Domain.Exporting.Writers
namespace DiscordChatExporter.Core.Exporting.Writers
{
internal abstract class MessageWriter : IAsyncDisposable
{
@@ -11,6 +11,10 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
protected ExportContext Context { get; }
public long MessagesWritten { get; private set; }
public long BytesWritten => Stream.Length;
protected MessageWriter(Stream stream, ExportContext context)
{
Stream = stream;
@@ -19,7 +23,11 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public virtual ValueTask WritePreambleAsync() => default;
public abstract ValueTask WriteMessageAsync(Message message);
public virtual ValueTask WriteMessageAsync(Message message)
{
MessagesWritten++;
return default;
}
public virtual ValueTask WritePostambleAsync() => default;
@@ -2,18 +2,16 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers
namespace DiscordChatExporter.Core.Exporting.Writers
{
internal class PlainTextMessageWriter : MessageWriter
{
private readonly TextWriter _writer;
private long _messageCount;
public PlainTextMessageWriter(Stream stream, ExportContext context)
: base(stream, context)
{
@@ -77,10 +75,10 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
}
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Thumbnail.Url));
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Thumbnail.ProxyUrl ?? embed.Thumbnail.Url));
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Image.Url));
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url));
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
await _writer.WriteLineAsync(embed.Footer.Text);
@@ -113,16 +111,16 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
{
await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync($"Guild: {Context.Request.Guild.Name}");
await _writer.WriteLineAsync($"Channel: {Context.Request.Channel.Category} / {Context.Request.Channel.Name}");
await _writer.WriteLineAsync($"Channel: {Context.Request.Channel.Category.Name} / {Context.Request.Channel.Name}");
if (!string.IsNullOrWhiteSpace(Context.Request.Channel.Topic))
await _writer.WriteLineAsync($"Topic: {Context.Request.Channel.Topic}");
if (Context.Request.After != null)
await _writer.WriteLineAsync($"After: {Context.FormatDate(Context.Request.After.Value)}");
if (Context.Request.After is not null)
await _writer.WriteLineAsync($"After: {Context.FormatDate(Context.Request.After.Value.ToDate())}");
if (Context.Request.Before != null)
await _writer.WriteLineAsync($"Before: {Context.FormatDate(Context.Request.Before.Value)}");
if (Context.Request.Before is not null)
await _writer.WriteLineAsync($"Before: {Context.FormatDate(Context.Request.Before.Value.ToDate())}");
await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync();
@@ -130,26 +128,29 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public override async ValueTask WriteMessageAsync(Message message)
{
await base.WriteMessageAsync(message);
// Header
await WriteMessageHeaderAsync(message);
// Content
if (!string.IsNullOrWhiteSpace(message.Content))
await _writer.WriteLineAsync(FormatMarkdown(message.Content));
await _writer.WriteLineAsync();
// Attachments, embeds, reactions
await WriteAttachmentsAsync(message.Attachments);
await WriteEmbedsAsync(message.Embeds);
await WriteReactionsAsync(message.Reactions);
await _writer.WriteLineAsync();
_messageCount++;
}
public override async ValueTask WritePostambleAsync()
{
await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync($"Exported {_messageCount:N0} message(s)");
await _writer.WriteLineAsync($"Exported {MessagesWritten:N0} message(s)");
await _writer.WriteLineAsync('='.Repeat(62));
}
@@ -1,11 +1,20 @@
namespace DiscordChatExporter.Domain.Markdown.Ast
using DiscordChatExporter.Core.Utils;
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal class EmojiNode : MarkdownNode
{
// Only present on custom emoji
public string? Id { get; }
// Name of custom emoji (e.g. LUL) or actual representation of standard emoji (e.g. 🙂)
public string Name { get; }
// Name of custom emoji (e.g. LUL) or name of standard emoji (e.g. slight_smile)
public string Code => !string.IsNullOrWhiteSpace(Id)
? Name
: EmojiIndex.TryGetCode(Name) ?? Name;
public bool IsAnimated { get; }
public bool IsCustomEmoji => !string.IsNullOrWhiteSpace(Id);
@@ -1,17 +1,7 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Domain.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal enum TextFormatting
{
Bold,
Italic,
Underline,
Strikethrough,
Spoiler,
Quote
}
internal class FormattedNode : MarkdownNode
{
public TextFormatting Formatting { get; }
@@ -1,4 +1,4 @@
namespace DiscordChatExporter.Domain.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal class InlineCodeBlockNode : MarkdownNode
{
@@ -1,4 +1,4 @@
namespace DiscordChatExporter.Domain.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal class LinkNode : MarkdownNode
{
@@ -1,4 +1,4 @@
namespace DiscordChatExporter.Domain.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal abstract class MarkdownNode
{
@@ -1,13 +1,5 @@
namespace DiscordChatExporter.Domain.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal enum MentionType
{
Meta,
User,
Channel,
Role
}
internal class MentionNode : MarkdownNode
{
public string Id { get; }
@@ -0,0 +1,10 @@
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal enum MentionType
{
Meta,
User,
Channel,
Role
}
}
@@ -1,4 +1,4 @@
namespace DiscordChatExporter.Domain.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal class MultiLineCodeBlockNode : MarkdownNode
{
@@ -0,0 +1,12 @@
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal enum TextFormatting
{
Bold,
Italic,
Underline,
Strikethrough,
Spoiler,
Quote
}
}
@@ -1,4 +1,4 @@
namespace DiscordChatExporter.Domain.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal class TextNode : MarkdownNode
{
@@ -1,82 +1,101 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using DiscordChatExporter.Domain.Markdown.Ast;
using DiscordChatExporter.Domain.Markdown.Matching;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Markdown.Matching;
using DiscordChatExporter.Core.Utils;
namespace DiscordChatExporter.Domain.Markdown
namespace DiscordChatExporter.Core.Markdown
{
// The following parsing logic is meant to replicate Discord's markdown grammar as close as possible
internal static partial class MarkdownParser
{
private const RegexOptions DefaultRegexOptions = RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.Multiline;
private const RegexOptions DefaultRegexOptions =
RegexOptions.Compiled |
RegexOptions.CultureInvariant |
RegexOptions.Multiline;
/* Formatting */
// Capture any character until the earliest double asterisk not followed by an asterisk
private static readonly IMatcher<MarkdownNode> BoldFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\*\\*(.+?)\\*\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Bold, Parse(p.Slice(m.Groups[1]))));
(p, m) => new FormattedNode(TextFormatting.Bold, Parse(p.Slice(m.Groups[1])))
);
// Capture any character until the earliest single asterisk not preceded or followed by an asterisk
// Opening asterisk must not be followed by whitespace
// Closing asterisk must not be preceded by whitespace
private static readonly IMatcher<MarkdownNode> ItalicFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\*(?!\\s)(.+?)(?<!\\s|\\*)\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1]))));
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1])))
);
// Capture any character until the earliest triple asterisk not followed by an asterisk
private static readonly IMatcher<MarkdownNode> ItalicBoldFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\*(\\*\\*.+?\\*\\*)\\*(?!\\*)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1]), BoldFormattedNodeMatcher)));
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1]), BoldFormattedNodeMatcher))
);
// Capture any character except underscore until an underscore
// Closing underscore must not be followed by a word character
private static readonly IMatcher<MarkdownNode> ItalicAltFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("_([^_]+)_(?!\\w)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1]))));
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1])))
);
// Capture any character until the earliest double underscore not followed by an underscore
private static readonly IMatcher<MarkdownNode> UnderlineFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("__(.+?)__(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Underline, Parse(p.Slice(m.Groups[1]))));
(p, m) => new FormattedNode(TextFormatting.Underline, Parse(p.Slice(m.Groups[1])))
);
// Capture any character until the earliest triple underscore not followed by an underscore
private static readonly IMatcher<MarkdownNode> ItalicUnderlineFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("_(__.+?__)_(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Italic, Parse(p.Slice(m.Groups[1]), UnderlineFormattedNodeMatcher)));
private static readonly IMatcher<MarkdownNode> ItalicUnderlineFormattedNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex("_(__.+?__)_(?!_)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Italic,
Parse(p.Slice(m.Groups[1]), UnderlineFormattedNodeMatcher))
);
// Capture any character until the earliest double tilde
private static readonly IMatcher<MarkdownNode> StrikethroughFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("~~(.+?)~~", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Strikethrough, Parse(p.Slice(m.Groups[1]))));
private static readonly IMatcher<MarkdownNode> StrikethroughFormattedNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex("~~(.+?)~~", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Strikethrough, Parse(p.Slice(m.Groups[1])))
);
// Capture any character until the earliest double pipe
private static readonly IMatcher<MarkdownNode> SpoilerFormattedNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\|\\|(.+?)\\|\\|", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Spoiler, Parse(p.Slice(m.Groups[1]))));
(p, m) => new FormattedNode(TextFormatting.Spoiler, Parse(p.Slice(m.Groups[1])))
);
// Capture any character until the end of the line
// Opening 'greater than' character must be followed by whitespace
private static readonly IMatcher<MarkdownNode> SingleLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("^>\\s(.+\n?)", DefaultRegexOptions),
(p, m) => new FormattedNode(TextFormatting.Quote, Parse(p.Slice(m.Groups[1]))));
(p, m) => new FormattedNode(TextFormatting.Quote, Parse(p.Slice(m.Groups[1])))
);
// Repeatedly capture any character until the end of the line
// This one is tricky as it ends up producing multiple separate captures which need to be joined
private static readonly IMatcher<MarkdownNode> RepeatedSingleLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("(?:^>\\s(.+\n?)){2,}", DefaultRegexOptions),
(p, m) =>
{
var content = string.Concat(m.Groups[1].Captures.Select(c => c.Value));
return new FormattedNode(TextFormatting.Quote, Parse(content));
});
private static readonly IMatcher<MarkdownNode> RepeatedSingleLineQuoteNodeMatcher =
new RegexMatcher<MarkdownNode>(
new Regex("(?:^>\\s(.+\n?)){2,}", DefaultRegexOptions),
(_, m) =>
{
var content = string.Concat(m.Groups[1].Captures.Select(c => c.Value));
return new FormattedNode(TextFormatting.Quote, Parse(content));
}
);
// Capture any character until the end of the input
// Opening 'greater than' characters must be followed by whitespace
private static readonly IMatcher<MarkdownNode> MultiLineQuoteNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("^>>>\\s(.+)", DefaultRegexOptions | RegexOptions.Singleline),
(p, m) => new FormattedNode(TextFormatting.Quote, Parse(p.Slice(m.Groups[1]))));
(p, m) => new FormattedNode(TextFormatting.Quote, Parse(p.Slice(m.Groups[1])))
);
/* Code blocks */
@@ -85,41 +104,48 @@ namespace DiscordChatExporter.Domain.Markdown
// There can be either one or two backticks, but equal number on both sides
private static readonly IMatcher<MarkdownNode> InlineCodeBlockNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("(`{1,2})([^`]+)\\1", DefaultRegexOptions | RegexOptions.Singleline),
m => new InlineCodeBlockNode(m.Groups[2].Value.Trim('\r', '\n')));
(_, m) => new InlineCodeBlockNode(m.Groups[2].Value.Trim('\r', '\n'))
);
// Capture language identifier and then any character until the earliest triple backtick
// Language identifier is one word immediately after opening backticks, followed immediately by newline
// Blank lines at the beginning and end of content are trimmed
private static readonly IMatcher<MarkdownNode> MultiLineCodeBlockNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("```(?:(\\w*)\\n)?(.+?)```", DefaultRegexOptions | RegexOptions.Singleline),
m => new MultiLineCodeBlockNode(m.Groups[1].Value, m.Groups[2].Value.Trim('\r', '\n')));
(_, m) => new MultiLineCodeBlockNode(m.Groups[1].Value, m.Groups[2].Value.Trim('\r', '\n'))
);
/* Mentions */
// Capture @everyone
private static readonly IMatcher<MarkdownNode> EveryoneMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@everyone",
p => new MentionNode("everyone", MentionType.Meta));
_ => new MentionNode("everyone", MentionType.Meta)
);
// Capture @here
private static readonly IMatcher<MarkdownNode> HereMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@here",
p => new MentionNode("here", MentionType.Meta));
_ => new MentionNode("here", MentionType.Meta)
);
// Capture <@123456> or <@!123456>
private static readonly IMatcher<MarkdownNode> UserMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<@!?(\\d+)>", DefaultRegexOptions),
m => new MentionNode(m.Groups[1].Value, MentionType.User));
(_, m) => new MentionNode(m.Groups[1].Value, MentionType.User)
);
// Capture <#123456>
private static readonly IMatcher<MarkdownNode> ChannelMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<#(\\d+)>", DefaultRegexOptions),
m => new MentionNode(m.Groups[1].Value, MentionType.Channel));
(_, m) => new MentionNode(m.Groups[1].Value, MentionType.Channel)
);
// Capture <@&123456>
private static readonly IMatcher<MarkdownNode> RoleMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<@&(\\d+)>", DefaultRegexOptions),
m => new MentionNode(m.Groups[1].Value, MentionType.Role));
(_, m) => new MentionNode(m.Groups[1].Value, MentionType.Role)
);
/* Emojis */
@@ -130,29 +156,46 @@ namespace DiscordChatExporter.Domain.Markdown
// (this does not match all emojis in Discord but it's reasonably accurate enough)
private static readonly IMatcher<MarkdownNode> StandardEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("((?:[\\uD83C][\\uDDE6-\\uDDFF]){2}|[\\u2600-\\u26FF]|\\p{Cs}{2}|\\d\\p{Me})", DefaultRegexOptions),
m => new EmojiNode(m.Groups[1].Value));
(_, m) => new EmojiNode(m.Groups[1].Value)
);
// Capture :thinking: (but only for known emoji codes)
private static readonly IMatcher<MarkdownNode> CodedStandardEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex(":([\\w_]+):", DefaultRegexOptions),
(_, m) =>
{
var name = EmojiIndex.TryGetName(m.Groups[1].Value);
return name is not null
? new EmojiNode(name)
: null;
}
);
// Capture <:lul:123456> or <a:lul:123456>
private static readonly IMatcher<MarkdownNode> CustomEmojiNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<(a)?:(.+?):(\\d+?)>", DefaultRegexOptions),
m => new EmojiNode(m.Groups[3].Value, m.Groups[2].Value, !string.IsNullOrWhiteSpace(m.Groups[1].Value)));
(_, m) => new EmojiNode(m.Groups[3].Value, m.Groups[2].Value, !string.IsNullOrWhiteSpace(m.Groups[1].Value))
);
/* Links */
// Capture [title](link)
private static readonly IMatcher<MarkdownNode> TitledLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\[(.+?)\\]\\((.+?)\\)", DefaultRegexOptions),
m => new LinkNode(m.Groups[2].Value, m.Groups[1].Value));
(_, m) => new LinkNode(m.Groups[2].Value, m.Groups[1].Value)
);
// Capture any non-whitespace character after http:// or https:// until the last punctuation character or whitespace
private static readonly IMatcher<MarkdownNode> AutoLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("(https?://\\S*[^\\.,:;\"\'\\s])", DefaultRegexOptions),
m => new LinkNode(m.Groups[1].Value));
(_, m) => new LinkNode(m.Groups[1].Value)
);
// Same as auto link but also surrounded by angular brackets
private static readonly IMatcher<MarkdownNode> HiddenLinkNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<(https?://\\S*[^\\.,:;\"\'\\s])>", DefaultRegexOptions),
m => new LinkNode(m.Groups[1].Value));
(_, m) => new LinkNode(m.Groups[1].Value)
);
/* Text */
@@ -160,25 +203,29 @@ namespace DiscordChatExporter.Domain.Markdown
// This escapes it from matching for formatting
private static readonly IMatcher<MarkdownNode> ShrugTextNodeMatcher = new StringMatcher<MarkdownNode>(
@"¯\_(ツ)_/¯",
p => new TextNode(p.ToString()));
p => new TextNode(p.ToString())
);
// Capture some specific emojis that don't get rendered
// This escapes it from matching for emoji
private static readonly IMatcher<MarkdownNode> IgnoredEmojiTextNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("(\\u26A7|\\u2640|\\u2642|\\u2695|\\u267E|\\u00A9|\\u00AE|\\u2122)", DefaultRegexOptions),
m => new TextNode(m.Groups[1].Value));
(_, m) => new TextNode(m.Groups[1].Value)
);
// Capture any "symbol/other" character or surrogate pair preceded by a backslash
// This escapes it from matching for emoji
private static readonly IMatcher<MarkdownNode> EscapedSymbolTextNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\\\(\\p{So}|\\p{Cs}{2})", DefaultRegexOptions),
m => new TextNode(m.Groups[1].Value));
(_, m) => new TextNode(m.Groups[1].Value)
);
// Capture any non-whitespace, non latin alphanumeric character preceded by a backslash
// This escapes it from matching for formatting or other tokens
private static readonly IMatcher<MarkdownNode> EscapedCharacterTextNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("\\\\([^a-zA-Z0-9\\s])", DefaultRegexOptions),
m => new TextNode(m.Groups[1].Value));
(_, m) => new TextNode(m.Groups[1].Value)
);
// Combine all matchers into one
// Matchers that have similar patterns are ordered from most specific to least specific
@@ -220,7 +267,8 @@ namespace DiscordChatExporter.Domain.Markdown
// Emoji
StandardEmojiNodeMatcher,
CustomEmojiNodeMatcher
CustomEmojiNodeMatcher,
CodedStandardEmojiNodeMatcher
);
// Minimal set of matchers for non-multimedia formats (e.g. plain text)
@@ -1,8 +1,8 @@
using System;
using System.Collections.Generic;
using DiscordChatExporter.Domain.Markdown.Ast;
using DiscordChatExporter.Core.Markdown.Ast;
namespace DiscordChatExporter.Domain.Markdown
namespace DiscordChatExporter.Core.Markdown
{
internal abstract class MarkdownVisitor
{
@@ -1,6 +1,6 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Domain.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Matching
{
internal class AggregateMatcher<T> : IMatcher<T>
{
@@ -27,11 +27,11 @@ namespace DiscordChatExporter.Domain.Markdown.Matching
var match = matcher.TryMatch(stringPart);
// If there's no match - continue
if (match == null)
if (match is null)
continue;
// If this match is earlier than previous earliest - replace
if (earliestMatch == null || match.StringPart.StartIndex < earliestMatch.StringPart.StartIndex)
if (earliestMatch is null || match.StringPart.StartIndex < earliestMatch.StringPart.StartIndex)
earliestMatch = match;
// If the earliest match starts at the very beginning - break,
@@ -1,7 +1,7 @@
using System;
using System.Collections.Generic;
namespace DiscordChatExporter.Domain.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Matching
{
internal interface IMatcher<T>
{
@@ -21,7 +21,7 @@ namespace DiscordChatExporter.Domain.Markdown.Matching
var match = matcher.TryMatch(stringPart.Slice(currentIndex, stringPart.EndIndex - currentIndex));
// If there's no match - break
if (match == null)
if (match is null)
break;
// If this match doesn't start immediately at current index - transform and yield fallback first
@@ -1,4 +1,4 @@
namespace DiscordChatExporter.Domain.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Matching
{
internal class ParsedMatch<T>
{
@@ -1,24 +1,19 @@
using System;
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Domain.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Matching
{
internal class RegexMatcher<T> : IMatcher<T>
{
private readonly Regex _regex;
private readonly Func<StringPart, Match, T> _transform;
private readonly Func<StringPart, Match, T?> _transform;
public RegexMatcher(Regex regex, Func<StringPart, Match, T> transform)
public RegexMatcher(Regex regex, Func<StringPart, Match, T?> transform)
{
_regex = regex;
_transform = transform;
}
public RegexMatcher(Regex regex, Func<Match, T> transform)
: this(regex, (p, m) => transform(m))
{
}
public ParsedMatch<T>? TryMatch(StringPart stringPart)
{
var match = _regex.Match(stringPart.Target, stringPart.StartIndex, stringPart.Length);
@@ -30,11 +25,15 @@ namespace DiscordChatExporter.Domain.Markdown.Matching
// Which is super weird because regex.Match(string, int) takes the whole input in context.
// So in order to properly account for ^/$ regex tokens, we need to make sure that
// the expression also matches on the bigger part of the input.
if (!_regex.IsMatch(stringPart.Target.Substring(0, stringPart.EndIndex), stringPart.StartIndex))
if (!_regex.IsMatch(stringPart.Target[..stringPart.EndIndex], stringPart.StartIndex))
return null;
var stringPartMatch = stringPart.Slice(match.Index, match.Length);
return new ParsedMatch<T>(stringPartMatch, _transform(stringPartMatch, match));
var value = _transform(stringPartMatch, match);
return value is not null
? new ParsedMatch<T>(stringPartMatch, value)
: null;
}
}
}
@@ -1,14 +1,14 @@
using System;
namespace DiscordChatExporter.Domain.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Matching
{
internal class StringMatcher<T> : IMatcher<T>
{
private readonly string _needle;
private readonly StringComparison _comparison;
private readonly Func<StringPart, T> _transform;
private readonly Func<StringPart, T?> _transform;
public StringMatcher(string needle, StringComparison comparison, Func<StringPart, T> transform)
public StringMatcher(string needle, StringComparison comparison, Func<StringPart, T?> transform)
{
_needle = needle;
_comparison = comparison;
@@ -23,14 +23,15 @@ namespace DiscordChatExporter.Domain.Markdown.Matching
public ParsedMatch<T>? TryMatch(StringPart stringPart)
{
var index = stringPart.Target.IndexOf(_needle, stringPart.StartIndex, stringPart.Length, _comparison);
if (index < 0)
return null;
if (index >= 0)
{
var stringPartMatch = stringPart.Slice(index, _needle.Length);
return new ParsedMatch<T>(stringPartMatch, _transform(stringPartMatch));
}
var stringPartMatch = stringPart.Slice(index, _needle.Length);
var value = _transform(stringPartMatch);
return null;
return value is not null
? new ParsedMatch<T>(stringPartMatch, value)
: null;
}
}
}
@@ -1,6 +1,6 @@
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Domain.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Matching
{
internal readonly struct StringPart
{
@@ -10,14 +10,13 @@ namespace DiscordChatExporter.Domain.Markdown.Matching
public int Length { get; }
public int EndIndex { get; }
public int EndIndex => StartIndex + Length;
public StringPart(string target, int startIndex, int length)
{
Target = target;
StartIndex = startIndex;
Length = length;
EndIndex = startIndex + length;
}
public StringPart(string target)
@@ -25,7 +24,7 @@ namespace DiscordChatExporter.Domain.Markdown.Matching
{
}
public StringPart Slice(int newStartIndex, int newLength) => new StringPart(Target, newStartIndex, newLength);
public StringPart Slice(int newStartIndex, int newLength) => new(Target, newStartIndex, newLength);
public StringPart Slice(int newStartIndex) => Slice(newStartIndex, EndIndex - newStartIndex);
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More