Merge pull request #93 from mmmeeedddsss/main

Add support for separating media files per chat

LICENSE.django

@@ -0,0 +1,36 @@
+The Whatsapp Chat Exporter is licensed under the MIT license. For more information,
+refer to the file LICENSE.
+
+Whatsapp Chat Exporter incorporates code from Django, governed by the three-clause
+BSD license—a permissive open-source license. The copyright and license details are
+provided below to adhere to Django's terms.
+
+------
+
+Copyright (c) Django Software Foundation and individual contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of Django nor the names of its contributors may be used
+       to endorse or promote products derived from this software without
+       specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -118,7 +118,7 @@ usage: wtsexporter [-h] [-a] [-i] [-e EXPORTED] [-w WA] [-m MEDIA] [-b BACKUP] [
                    [-k KEY] [-t TEMPLATE] [-s] [-c] [--offline OFFLINE] [--size [SIZE]] [--no-html] [--check-update]
                    [--assume-first-as-me] [--no-avatar] [--import] [--business] [--preserve-timestamp] [--wab WAB]
                    [--time-offset {-12 to 14}] [--date DATE] [--date-format FORMAT] [--include [phone number ...]]
-                   [--exclude [phone number ...]]
+                   [--exclude [phone number ...]] [--create-separated-media]
 
 A customizable Android and iPhone WhatsApp database parser that will give you the history of your WhatsApp
 conversations in HTML and JSON. Android Backup Crypt12, Crypt14 and Crypt15 supported.
@@ -164,8 +164,11 @@ options:
                         Include chats that match the supplied phone number
   --exclude [phone number ...]
                         Exclude chats that match the supplied phone number
+  --create-separated-media
+                        Create a copy of the media seperated per chat in <MEDIA>/separated/ directory 
+                        (Android only)
 
-WhatsApp Chat Exporter: 0.9.7 Licensed with MIT
+WhatsApp Chat Exporter: 0.10.0 Licensed with MIT
 ```
 
 # To do

Whatsapp_Chat_Exporter/__init__.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python3
 
-__version__ = "0.9.7"
+__version__ = "0.10.0"

Whatsapp_Chat_Exporter/__main__.py

@@ -26,7 +26,8 @@ def main():
         description = 'A customizable Android and iOS/iPadOS WhatsApp database parser that '
                       'will give you the history of your WhatsApp conversations in HTML '
                       'and JSON. Android Backup Crypt12, Crypt14 and Crypt15 supported.',
-        epilog = f'WhatsApp Chat Exporter: {__version__} Licensed with MIT'
+        epilog = f'WhatsApp Chat Exporter: {__version__} Licensed with MIT. See'
+                  'https://wts.knugi.dev/docs?dest=osl for all open source licenses.'
     )
     parser.add_argument(
         '-a',
@@ -245,6 +246,13 @@ def main():
         action='store_true',
         help="Output the JSON file per chat"
     )
+    parser.add_argument(
+        "--create-separated-media",
+        dest="separate_media",
+        default=False,
+        action='store_true',
+        help="Create a copy of the media seperated per chat in <MEDIA>/separated/ directory"
+    )
     args = parser.parse_args()
 
     # Check for updates
@@ -310,7 +318,6 @@ def main():
                 parser.error("Enter a phone number in the chat filter. See https://wts.knugi.dev/docs?dest=chat")
     filter_chat = (args.filter_chat_include, args.filter_chat_exclude)
 
-
     data = {}
 
     if args.android:
@@ -417,7 +424,7 @@ def main():
             with sqlite3.connect(msg_db) as db:
                 db.row_factory = sqlite3.Row
                 messages(db, data, args.media, args.timezone_offset, args.filter_date, filter_chat)
-                media(db, data, args.media, args.filter_date, filter_chat)
+                media(db, data, args.media, args.filter_date, filter_chat, args.separate_media)
                 vcard(db, data, args.media, args.filter_date, filter_chat)
                 if args.android:
                     android_handler.calls(db, data, args.timezone_offset, filter_chat)

Whatsapp_Chat_Exporter/android_handler.py

@@ -4,6 +4,7 @@ import sqlite3
 import os
 import io
 import hmac
+import shutil
 from pathlib import Path
 from mimetypes import MimeTypes
 from hashlib import sha256
@@ -12,7 +13,7 @@ from Whatsapp_Chat_Exporter.data_model import ChatStore, Message
 from Whatsapp_Chat_Exporter.utility import MAX_SIZE, ROW_SIZE, DbType, determine_metadata, JidType
 from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template
 from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS, get_status_location
-from Whatsapp_Chat_Exporter.utility import get_chat_condition
+from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify
 
 try:
     import zlib
@@ -477,7 +478,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat):
     print(f"Processing messages...({total_row_number}/{total_row_number})", end="\r")
 
 
-def media(db, data, media_folder, filter_date, filter_chat):
+def media(db, data, media_folder, filter_date, filter_chat, separate_media=True):
     # Get media
     c = db.cursor()
     try:
@@ -569,6 +570,15 @@ def media(db, data, media_folder, filter_date, filter_chat):
                     message.mime = "application/octet-stream"
             else:
                 message.mime = content["mime_type"]
+            if separate_media:
+                chat_display_name = slugify(data[content["key_remote_jid"]].name or message.sender \
+                                            or content["key_remote_jid"].split('@')[0], True)
+                current_filename = file_path.split("/")[-1]
+                new_folder = os.path.join(media_folder, "separated", chat_display_name)
+                Path(new_folder).mkdir(parents=True, exist_ok=True)
+                new_path = os.path.join(new_folder, current_filename)
+                shutil.copy2(file_path, new_path)
+                message.data = new_path
         else:
             if False: # Block execution
                 try:

Whatsapp_Chat_Exporter/ios_handler.py

@@ -1,11 +1,12 @@
 #!/usr/bin/python3
 
 import os
+import shutil
 from glob import glob
 from pathlib import Path
 from mimetypes import MimeTypes
 from Whatsapp_Chat_Exporter.data_model import ChatStore, Message
-from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Device, get_chat_condition
+from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Device, get_chat_condition, slugify
 
 
 def contacts(db, data):
@@ -205,7 +206,7 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat):
         f"Processing messages...({total_row_number}/{total_row_number})", end="\r")
 
 
-def media(db, data, media_folder, filter_date, filter_chat):
+def media(db, data, media_folder, filter_date, filter_chat, separate_media=False):
     c = db.cursor()
     # Get media
     c.execute(f"""SELECT count()
@@ -256,6 +257,15 @@ def media(db, data, media_folder, filter_date, filter_chat):
                     message.mime = "application/octet-stream"
             else:
                 message.mime = content["ZVCARDSTRING"]
+            if separate_media:
+                chat_display_name = slugify(data[content["ZCONTACTJID"]].name or message.sender \
+                                            or content["ZCONTACTJID"].split('@')[0], True)
+                current_filename = file_path.split("/")[-1]
+                new_folder = os.path.join(media_folder, "separated", chat_display_name)
+                Path(new_folder).mkdir(parents=True, exist_ok=True)
+                new_path = os.path.join(new_folder, current_filename)
+                shutil.copy2(file_path, new_path)
+                message.data = new_path
         else:
             if False: # Block execution
                 try:

Whatsapp_Chat_Exporter/utility.py

@@ -3,6 +3,8 @@ import json
 import os
 from bleach import clean as sanitize
 from markupsafe import Markup
+import unicodedata
+import re
 from datetime import datetime
 from enum import IntEnum
 from Whatsapp_Chat_Exporter.data_model import ChatStore
@@ -309,6 +311,23 @@ def setup_template(template, no_avatar):
 APPLE_TIME = datetime.timestamp(datetime(2001, 1, 1))
 
 
+def slugify(value, allow_unicode=False):
+    """
+    Taken from https://github.com/django/django/blob/master/django/utils/text.py
+    Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
+    dashes to single dashes. Remove characters that aren't alphanumerics,
+    underscores, or hyphens. Convert to lowercase. Also strip leading and
+    trailing whitespace, dashes, and underscores.
+    """
+    value = str(value)
+    if allow_unicode:
+        value = unicodedata.normalize('NFKC', value)
+    else:
+        value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
+    value = re.sub(r'[^\w\s-]', '', value.lower())
+    return re.sub(r'[-\s]+', '-', value).strip('-_')
+
+
 class WhatsAppIdentifier(StrEnum):
     MESSAGE = "7c7fba66680ef796b916b067077cc246adacf01d"
     CONTACT = "b8548dc30aa1030df0ce18ef08b882cf7ab5212f"

docs.html

@@ -7,6 +7,7 @@
                 "filter": "Filter",
                 "date": "Filters#date-filters",
                 "chat": "Filters#chat-filter",
+                "osl": "Open-Source-Licenses"
                 null: ""
             };
             const dest = new URLSearchParams(window.location.search).get('dest');