Filter which messages are exported (#622)

DiscordChatExporter.Core/DiscordChatExporter.Core.csproj

@@ -8,6 +8,7 @@
     <PackageReference Include="JsonExtensions" Version="1.1.0" />
     <PackageReference Include="MiniRazor.CodeGen" Version="2.1.2" />
     <PackageReference Include="Polly" Version="7.2.2" />
+    <PackageReference Include="Superpower" Version="2.3.0" />
     <PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
   </ItemGroup>
 

DiscordChatExporter.Core/Exporting/ChannelExporter.cs

@@ -39,6 +39,10 @@ namespace DiscordChatExporter.Core.Exporting
             var encounteredUsers = new HashSet<User>(IdBasedEqualityComparer.Instance);
             await foreach (var message in _discord.GetMessagesAsync(request.Channel.Id, request.After, request.Before, progress))
             {
+                // Skips any messages that fail to pass the supplied filter
+                if (!request.MessageFilter.Filter(message))
+                    continue;
+
                 // Resolve members for referenced users
                 foreach (var referencedUser in message.MentionedUsers.Prepend(message.Author))
                 {

DiscordChatExporter.Core/Exporting/ExportRequest.cs

@@ -4,6 +4,7 @@ using System.Text;
 using System.Text.RegularExpressions;
 using DiscordChatExporter.Core.Discord;
 using DiscordChatExporter.Core.Discord.Data;
+using DiscordChatExporter.Core.Exporting.Filtering;
 using DiscordChatExporter.Core.Exporting.Partitioning;
 using DiscordChatExporter.Core.Utils;
 
@@ -31,6 +32,8 @@ namespace DiscordChatExporter.Core.Exporting
 
         public PartitionLimit PartitionLimit { get; }
 
+        public MessageFilter MessageFilter { get; }
+
         public bool ShouldDownloadMedia { get; }
 
         public bool ShouldReuseMedia { get; }
@@ -45,6 +48,7 @@ namespace DiscordChatExporter.Core.Exporting
             Snowflake? after,
             Snowflake? before,
             PartitionLimit partitionLimit,
+            MessageFilter messageFilter,
             bool shouldDownloadMedia,
             bool shouldReuseMedia,
             string dateFormat)
@@ -56,6 +60,7 @@ namespace DiscordChatExporter.Core.Exporting
             After = after;
             Before = before;
             PartitionLimit = partitionLimit;
+            MessageFilter = messageFilter;
             ShouldDownloadMedia = shouldDownloadMedia;
             ShouldReuseMedia = shouldReuseMedia;
             DateFormat = dateFormat;

DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionKind.cs

@@ -0,0 +1,8 @@
+namespace DiscordChatExporter.Core.Exporting.Filtering
+{
+    public enum BinaryExpressionKind
+    {
+        Or,
+        And
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionMessageFilter.cs

@@ -0,0 +1,26 @@
+using DiscordChatExporter.Core.Discord.Data;
+using System;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering
+{
+    public class BinaryExpressionMessageFilter : MessageFilter
+    {
+        private readonly MessageFilter _first;
+        private readonly MessageFilter _second;
+        private readonly BinaryExpressionKind _kind;
+
+        public BinaryExpressionMessageFilter(MessageFilter first, MessageFilter second, BinaryExpressionKind kind)
+        {
+            _first = first;
+            _second = second;
+            _kind = kind;
+        }
+
+        public override bool Filter(Message message) => _kind switch
+        {
+            BinaryExpressionKind.Or => _first.Filter(message) || _second.Filter(message),
+            BinaryExpressionKind.And => _first.Filter(message) && _second.Filter(message), 
+            _ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.")
+        };
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/ContainsMessageFilter.cs

@@ -0,0 +1,15 @@
+using DiscordChatExporter.Core.Discord.Data;
+using System.Text.RegularExpressions;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering
+{
+    public class ContainsMessageFilter : MessageFilter
+    {
+        private readonly string _value;
+
+        public ContainsMessageFilter(string value) => _value = value;
+
+        public override bool Filter(Message message) =>
+            Regex.IsMatch(message.Content, $@"\b{Regex.Escape(_value)}\b", RegexOptions.IgnoreCase | DefaultRegexOptions);
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/FromMessageFilter.cs

@@ -0,0 +1,17 @@
+using DiscordChatExporter.Core.Discord.Data;
+using System;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering
+{
+    public class FromMessageFilter : MessageFilter
+    {
+        private readonly string _value;
+
+        public FromMessageFilter(string value) => _value = value;
+
+        public override bool Filter(Message message) =>
+            string.Equals(_value, message.Author.Name, StringComparison.OrdinalIgnoreCase) ||
+            string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase) ||
+            string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase);
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/HasMessageFilter.cs

@@ -0,0 +1,26 @@
+using DiscordChatExporter.Core.Discord.Data;
+using System;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering
+{
+    public class HasMessageFilter : MessageFilter
+    {
+        private readonly string _value;
+
+        public HasMessageFilter(string value) => _value = value;
+
+        public override bool Filter(Message message) =>
+            _value switch
+            {
+                "link" => Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]", DefaultRegexOptions),
+                "embed" => message.Embeds.Any(),
+                "file" => message.Attachments.Any(),
+                "video" => message.Attachments.Any(file => file.IsVideo),
+                "image" => message.Attachments.Any(file => file.IsImage),
+                "sound" => message.Attachments.Any(file => file.IsAudio),
+                _ => throw new InvalidOperationException($"Invalid value provided for the 'has' message filter: '{_value}'")
+            };
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/MentionsMessageFilter.cs

@@ -0,0 +1,19 @@
+using DiscordChatExporter.Core.Discord.Data;
+using System;
+using System.Linq;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering
+{
+    public class MentionsMessageFilter : MessageFilter
+    {
+        private readonly string _value;
+
+        public MentionsMessageFilter(string value) => _value = value;
+
+        public override bool Filter(Message message) =>
+            message.MentionedUsers.Any(user =>
+                string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase) ||
+                string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase) ||
+                string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase));
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/MessageFilter.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Text.RegularExpressions;
+using DiscordChatExporter.Core.Discord.Data;
+using DiscordChatExporter.Core.Exporting.Filtering.Parsing;
+using Superpower;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering
+{
+    public abstract partial class MessageFilter
+    {
+        public abstract bool Filter(Message message);
+    }
+
+    public partial class MessageFilter
+    {
+        protected const RegexOptions DefaultRegexOptions = RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.Multiline;
+
+        internal static MessageFilter CreateFilter(string text) => new ContainsMessageFilter(text);
+
+        internal static MessageFilter CreateFilter(string key, string value)
+        {
+            return key.ToLowerInvariant() switch
+            {
+                "from" => new FromMessageFilter(value),
+                "has" => new HasMessageFilter(value),
+                "mentions" => new MentionsMessageFilter(value),
+                _ => throw new ArgumentException($"Invalid filter type '{key}'.", nameof(key))
+            };
+        }
+
+        public static MessageFilter Parse(string value, IFormatProvider? formatProvider = null)
+        {
+            var tokens = FilterTokenizer.Instance.Tokenize(value);
+            var parsed = FilterParser.Instance.Parse(tokens);
+            return parsed;
+        }
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/NegatedMessageFilter.cs

@@ -0,0 +1,13 @@
+using DiscordChatExporter.Core.Discord.Data;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering
+{
+    public class NegatedMessageFilter : MessageFilter
+    {
+        private readonly MessageFilter _filter;
+
+        public NegatedMessageFilter(MessageFilter filter) => _filter = filter;
+
+        public override bool Filter(Message message) => !_filter.Filter(message);
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/NullMessageFilter.cs

@@ -0,0 +1,11 @@
+using DiscordChatExporter.Core.Discord.Data;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering
+{
+    public class NullMessageFilter : MessageFilter
+    {
+        public static NullMessageFilter Instance { get; } = new();
+
+        public override bool Filter(Message message) => true;
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterParser.cs

@@ -0,0 +1,68 @@
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Text;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
+{
+    public static class FilterParser
+    {
+        public static TextParser<string> QuotedString { get; } =
+           from open in Character.EqualTo('"')
+           from content in Character.EqualTo('\\').IgnoreThen(Character.AnyChar).Try()
+               .Or(Character.Except('"'))
+               .Many()
+           from close in Character.EqualTo('"')
+           select new string(content);
+
+        public static TextParser<string> UnquotedString { get; } =
+            from content in Character.EqualTo('\\').IgnoreThen(Character.In('"', '/')).Try()
+                .Or(Character.Except(c => char.IsWhiteSpace(c) || "():-|\"".Contains(c), "non-whitespace character except for (, ), :, -, |, and \""))
+                .AtLeastOnce()
+            select new string(content);
+
+        public static TokenListParser<FilterToken, string> AnyString { get; } =
+            Token.EqualTo(FilterToken.QuotedString).Apply(QuotedString)
+            .Or(Token.EqualTo(FilterToken.UnquotedString).Apply(UnquotedString));
+
+        public static TokenListParser<FilterToken, MessageFilter> AnyFilter { get; } =
+            from minus in Token.EqualTo(FilterToken.Minus).Optional()
+            from content in KeyValueFilter.Or(TextFilter).Or(GroupedFilter)
+            select minus.HasValue ? new NegatedMessageFilter(content) : content;
+
+        public static TokenListParser<FilterToken, MessageFilter> TextFilter { get; } =
+           from value in AnyString
+           select MessageFilter.CreateFilter(value);
+
+        public static TokenListParser<FilterToken, MessageFilter> KeyValueFilter { get; } =
+            from key in AnyString.Try()
+            from colon in Token.EqualTo(FilterToken.Colon).Try()
+            from value in AnyString
+            select MessageFilter.CreateFilter(key, value);
+
+        public static TokenListParser<FilterToken, MessageFilter> GroupedFilter { get; } =
+            from open in Token.EqualTo(FilterToken.LParen)
+            from content in BinaryExpression
+            from close in Token.EqualTo(FilterToken.RParen)
+            select content;
+
+        public static TokenListParser<FilterToken, MessageFilter> OrBinaryExpression { get; } =
+            from first in AnyFilter
+            from vbar in Token.EqualTo(FilterToken.VBar)
+            from rest in BinaryExpression
+            select (MessageFilter)new BinaryExpressionMessageFilter(first, rest, BinaryExpressionKind.Or);
+
+        public static TokenListParser<FilterToken, MessageFilter> AndBinaryExpression { get; } =
+            from first in AnyFilter
+            from rest in BinaryExpression
+            select (MessageFilter)new BinaryExpressionMessageFilter(first, rest, BinaryExpressionKind.And);
+
+        public static TokenListParser<FilterToken, MessageFilter> BinaryExpression { get; } = OrBinaryExpression.Try().Or(AndBinaryExpression.Try()).Or(AnyFilter);
+        
+        public static TokenListParser<FilterToken, MessageFilter> Instance { get; } = BinaryExpression.AtEnd();
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterToken.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
+{
+    public enum FilterToken
+    {
+        None,
+        LParen,
+        RParen,
+        Colon,
+        Minus,
+        VBar,
+        UnquotedString,
+        QuotedString
+    }
+}

DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterTokenizer.cs

@@ -0,0 +1,23 @@
+using Superpower;
+using Superpower.Parsers;
+using Superpower.Tokenizers;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
+{
+    public static class FilterTokenizer
+    {
+        public static Tokenizer<FilterToken> Instance { get; } = new TokenizerBuilder<FilterToken>()
+            .Ignore(Span.WhiteSpace)
+            .Match(Character.EqualTo('('), FilterToken.LParen)
+            .Match(Character.EqualTo(')'), FilterToken.RParen)
+            .Match(Character.EqualTo(':'), FilterToken.Colon)
+            .Match(Character.EqualTo('-'), FilterToken.Minus)
+            .Match(Character.EqualTo('|'), FilterToken.VBar)
+            .Match(FilterParser.QuotedString, FilterToken.QuotedString)
+            .Match(FilterParser.UnquotedString, FilterToken.UnquotedString)
+            .Build();
+    }
+}

DiscordChatExporter.Core/Utils/Polyfills.cs

@@ -0,0 +1,9 @@
+// ReSharper disable CheckNamespace
+// TODO: remove after moving to .NET 5
+
+namespace System.Runtime.CompilerServices
+{
+    internal static class IsExternalInit
+    {
+    }
+}