calvin.erfmann/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2026-03-16 20:02:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

More refactoring

Browse Source
This commit is contained in:
Alexey Golub
2020-04-23 00:32:48 +03:00
parent b2a48d338a
commit 9d0d7cd5dd
8 changed files with 197 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
DiscordChatExporter.Domain/Discord/AuthToken.cs
View File

@@ -16,7 +16,7 @@ namespace DiscordChatExporter.Domain.Discord
Value = value;
}
public AuthenticationHeaderValue GetAuthenticationHeader() => Type == AuthTokenType.User
public AuthenticationHeaderValue GetAuthorizationHeader() => Type == AuthTokenType.User
? new AuthenticationHeaderValue(Value)
: new AuthenticationHeaderValue("Bot", Value);

73
DiscordChatExporter.Domain/Discord/DiscordClient.Parsing.cs
View File

@@ -29,10 +29,12 @@ namespace DiscordChatExporter.Domain.Discord
{
var userId = json.GetProperty("user").Pipe(ParseId);
var nick = json.GetPropertyOrNull("nick")?.GetString();
var roles = json.GetPropertyOrNull("roles")?.EnumerateArray().Select(j => j.GetString()).ToArray() ??
Array.Empty<string>();
return new Member(userId, nick, roles);
var roleIds =
json.GetPropertyOrNull("roles")?.EnumerateArray().Select(j => j.GetString()).ToArray() ??
Array.Empty<string>();
return new Member(userId, nick, roleIds);
}
private Guild ParseGuild(JsonElement json)
@@ -40,8 +42,10 @@ namespace DiscordChatExporter.Domain.Discord
var id = ParseId(json);
var name = json.GetProperty("name").GetString();
var iconHash = json.GetProperty("icon").GetString();
var roles = json.GetPropertyOrNull("roles")?.EnumerateArray().Select(ParseRole).ToArray() ??
Array.Empty<Role>();
var roles =
json.GetPropertyOrNull("roles")?.EnumerateArray().Select(ParseRole).ToArray() ??
Array.Empty<Role>();
return new Guild(id, name, iconHash, roles);
}
@@ -53,8 +57,9 @@ namespace DiscordChatExporter.Domain.Discord
var type = (ChannelType) json.GetProperty("type").GetInt32();
var topic = json.GetPropertyOrNull("topic")?.GetString();
var guildId = json.GetPropertyOrNull("guild_id")?.GetString() ??
Guild.DirectMessages.Id;
var guildId =
json.GetPropertyOrNull("guild_id")?.GetString() ??
Guild.DirectMessages.Id;
var name =
json.GetPropertyOrNull("name")?.GetString() ??
@@ -134,10 +139,22 @@ namespace DiscordChatExporter.Domain.Discord
var image = json.GetPropertyOrNull("image")?.Pipe(ParseEmbedImage);
var footer = json.GetPropertyOrNull("footer")?.Pipe(ParseEmbedFooter);
var fields = json.GetPropertyOrNull("fields")?.EnumerateArray().Select(ParseEmbedField).ToArray() ??
Array.Empty<EmbedField>();
var fields =
json.GetPropertyOrNull("fields")?.EnumerateArray().Select(ParseEmbedField).ToArray() ??
Array.Empty<EmbedField>();
return new Embed(title, url, timestamp, color, author, description, fields, thumbnail, image, footer);
return new Embed(
title,
url,
timestamp,
color,
author,
description,
fields,
thumbnail,
image,
footer
);
}
private Emoji ParseEmoji(JsonElement json)
@@ -180,20 +197,36 @@ namespace DiscordChatExporter.Domain.Discord
var author = json.GetProperty("author").Pipe(ParseUser);
var attachments = json.GetPropertyOrNull("attachments")?.EnumerateArray().Select(ParseAttachment).ToArray() ??
Array.Empty<Attachment>();
var attachments =
json.GetPropertyOrNull("attachments")?.EnumerateArray().Select(ParseAttachment).ToArray() ??
Array.Empty<Attachment>();
var embeds = json.GetPropertyOrNull("embeds")?.EnumerateArray().Select(ParseEmbed).ToArray() ??
Array.Empty<Embed>();
var embeds =
json.GetPropertyOrNull("embeds")?.EnumerateArray().Select(ParseEmbed).ToArray() ??
Array.Empty<Embed>();
var reactions = json.GetPropertyOrNull("reactions")?.EnumerateArray().Select(ParseReaction).ToArray() ??
Array.Empty<Reaction>();
var reactions =
json.GetPropertyOrNull("reactions")?.EnumerateArray().Select(ParseReaction).ToArray() ??
Array.Empty<Reaction>();
var mentionedUsers = json.GetPropertyOrNull("mentions")?.EnumerateArray().Select(ParseUser).ToArray() ??
Array.Empty<User>();
var mentionedUsers =
json.GetPropertyOrNull("mentions")?.EnumerateArray().Select(ParseUser).ToArray() ??
Array.Empty<User>();
return new Message(id, channelId, type, author, timestamp, editedTimestamp, isPinned, content, attachments, embeds,
reactions, mentionedUsers);
return new Message(
id,
channelId,
type,
author,
timestamp,
editedTimestamp,
isPinned,
content,
attachments,
embeds,
reactions,
mentionedUsers
);
}
}
}

87
DiscordChatExporter.Domain/Discord/DiscordClient.cs
View File

@@ -18,6 +18,8 @@ namespace DiscordChatExporter.Domain.Discord
private readonly HttpClient _httpClient;
private readonly IAsyncPolicy<HttpResponseMessage> _httpRequestPolicy;
private readonly Uri _baseUri = new Uri("https://discordapp.com/api/v6/", UriKind.Absolute);
public DiscordClient(AuthToken token, HttpClient httpClient)
{
_token = token;
@@ -51,10 +53,8 @@ namespace DiscordChatExporter.Domain.Discord
{
using var response = await _httpRequestPolicy.ExecuteAsync(async () =>
{
var uri = new Uri(new Uri("https://discordapp.com/api/v6"), url);
using var request = new HttpRequestMessage(HttpMethod.Get, uri);
request.Headers.Authorization = _token.GetAuthenticationHeader();
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
request.Headers.Authorization = _token.GetAuthorizationHeader();
return await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
});
@@ -113,11 +113,13 @@ namespace DiscordChatExporter.Domain.Discord
while (true)
{
var route = "users/@me/guilds?limit=100";
if (!string.IsNullOrWhiteSpace(afterId))
route += $"&after={afterId}";
var url = new UrlBuilder()
.SetPath("users/@me/guilds")
.SetQueryParameter("limit", "100")
.SetQueryParameterIfNotNullOrWhiteSpace("after", afterId)
.Build();
var response = await GetApiResponseAsync(route);
var response = await GetApiResponseAsync(url);
var isEmpty = true;
@@ -147,7 +149,7 @@ namespace DiscordChatExporter.Domain.Discord
public async Task<IReadOnlyList<Channel>> GetGuildChannelsAsync(string guildId)
{
// Special case for direct messages pseudo-guild
// Direct messages pseudo-guild
if (guildId == Guild.DirectMessages.Id)
return Array.Empty<Channel>();
@@ -159,38 +161,42 @@ namespace DiscordChatExporter.Domain.Discord
private async Task<Message> GetLastMessageAsync(string channelId, DateTimeOffset? before = null)
{
var route = $"channels/{channelId}/messages?limit=1";
if (before != null)
route += $"&before={before.Value.ToSnowflake()}";
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
.SetQueryParameter("limit", "1")
.SetQueryParameterIfNotNullOrWhiteSpace("before", before?.ToSnowflake())
.Build();
var response = await GetApiResponseAsync(route);
var response = await GetApiResponseAsync(url);
return response.EnumerateArray().Select(ParseMessage).FirstOrDefault();
}
public async IAsyncEnumerable<Message> GetMessagesAsync(string channelId,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
public async IAsyncEnumerable<Message> GetMessagesAsync(
string channelId,
DateTimeOffset? after = null,
DateTimeOffset? before = null,
IProgress<double>? progress = null)
{
// Get the last message
var lastMessage = await GetLastMessageAsync(channelId, before);
// If the last message doesn't exist or it's outside of range - return
if (lastMessage == null || lastMessage.Timestamp < after)
{
progress?.Report(1);
yield break;
}
// Get other messages
var firstMessage = default(Message);
var afterId = after?.ToSnowflake() ?? "0";
while (true)
{
// Get message batch
var route = $"channels/{channelId}/messages?limit=100&after={afterId}";
var response = await GetApiResponseAsync(route);
var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages")
.SetQueryParameter("limit", "100")
.SetQueryParameter("after", afterId)
.Build();
var response = await GetApiResponseAsync(url);
// Parse
var messages = response
.EnumerateArray()
.Select(ParseMessage)
@@ -201,33 +207,28 @@ namespace DiscordChatExporter.Domain.Discord
if (!messages.Any())
break;
// Trim messages to range (until last message)
var messagesInRange = messages
.TakeWhile(m => m.Id != lastMessage.Id && m.Timestamp < lastMessage.Timestamp)
.ToArray();
// Yield messages
foreach (var message in messagesInRange)
foreach (var message in messages)
{
// Set first message if it's not set
firstMessage ??= message;
// Report progress (based on the time range of parsed messages compared to total)
progress?.Report((message.Timestamp - firstMessage.Timestamp).TotalSeconds /
(lastMessage.Timestamp - firstMessage.Timestamp).TotalSeconds);
// Ensure messages are in range (take into account that last message could have been deleted)
if (message.Timestamp > lastMessage.Timestamp)
yield break;
// Report progress based on the duration of parsed messages divided by total
progress?.Report(
(message.Timestamp - firstMessage.Timestamp) /
(lastMessage.Timestamp - firstMessage.Timestamp)
);
yield return message;
afterId = message.Id;
// Yielded last message - break loop
if (message.Id == lastMessage.Id)
yield break;
}
// Break if messages were trimmed (which means the last message was encountered)
if (messagesInRange.Length != messages.Length)
break;
}
// Yield last message
yield return lastMessage;
progress?.Report(1);
}
}

50
DiscordChatExporter.Domain/Discord/UrlBuilder.cs Normal file
View File

@@ -0,0 +1,50 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
namespace DiscordChatExporter.Domain.Discord
{
internal class UrlBuilder
{
private string _path = "";
private readonly Dictionary<string, string?> _queryParameters =
new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
public UrlBuilder SetPath(string path)
{
_path = path;
return this;
}
public UrlBuilder SetQueryParameter(string key, string? value)
{
var keyEncoded = WebUtility.UrlEncode(key);
var valueEncoded = WebUtility.UrlEncode(value);
_queryParameters[keyEncoded] = valueEncoded;
return this;
}
public UrlBuilder SetQueryParameterIfNotNullOrWhiteSpace(string key, string? value) =>
!string.IsNullOrWhiteSpace(value)
? SetQueryParameter(key, value)
: this;
public string Build()
{
var buffer = new StringBuilder();
buffer.Append(_path);
if (_queryParameters.Any())
buffer.Append('?');
buffer.AppendJoin('&', _queryParameters.Select(kvp => $"{kvp.Key}={kvp.Value}"));
return buffer.ToString();
}
}
}

21
DiscordChatExporter.Domain/Exporting/ChannelExporter.cs
View File

@@ -17,6 +17,8 @@ namespace DiscordChatExporter.Domain.Exporting
public ChannelExporter(DiscordClient discord) => _discord = discord;
public ChannelExporter(AuthToken token) : this(new DiscordClient(token)) {}
public async Task ExportAsync(
Guild guild,
Channel channel,
@@ -28,13 +30,12 @@ namespace DiscordChatExporter.Domain.Exporting
DateTimeOffset? before = null,
IProgress<double>? progress = null)
{
// Get base file path from output path
var baseFilePath = GetFilePathFromOutputPath(guild, channel, outputPath, format, after, before);
// Create options
// Options
var options = new ExportOptions(baseFilePath, format, partitionLimit);
// Create context
// Context
var mentionableUsers = new HashSet<User>(IdBasedEqualityComparer.Instance);
var mentionableChannels = await _discord.GetGuildChannelsAsync(guild.Id);
var mentionableRoles = guild.Roles;
@@ -44,11 +45,9 @@ namespace DiscordChatExporter.Domain.Exporting
mentionableUsers, mentionableChannels, mentionableRoles
);
// Create renderer
await using var renderer = new MessageExporter(options, context);
await using var messageExporter = new MessageExporter(options, context);
// Render messages
var renderedAnything = false;
var exportedAnything = false;
await foreach (var message in _discord.GetMessagesAsync(channel.Id, after, before, progress))
{
// Add encountered users to the list of mentionable users
@@ -68,12 +67,12 @@ namespace DiscordChatExporter.Domain.Exporting
}
// Render message
await renderer.RenderMessageAsync(message);
renderedAnything = true;
await messageExporter.ExportMessageAsync(message);
exportedAnything = true;
}
// Throw if no messages were rendered
if (!renderedAnything)
// Throw if no messages were exported
if (!exportedAnything)
throw DiscordChatExporterException.ChannelEmpty(channel);
}
}

2
DiscordChatExporter.Domain/Exporting/MessageExporter.cs
View File

@@ -62,7 +62,7 @@ namespace DiscordChatExporter.Domain.Exporting
return _writer = writer;
}
public async Task RenderMessageAsync(Message message)
public async Task ExportMessageAsync(Message message)
{
var writer = await GetWriterAsync();
await writer.WriteMessageAsync(message);