Merge branch 'dev' into bnaya-iteration

Whatsapp_Chat_Exporter/__main__.py

@@ -209,13 +209,6 @@ def main():
         action='store_true',
         help="Use Whatsapp Business default files (iOS only)"
     )
-    parser.add_argument(
-        "--preserve-timestamp",
-        dest="preserve_timestamp",
-        default=False,
-        action='store_true',
-        help="Preserve the modification timestamp of the extracted files (iOS only)"
-    )
     parser.add_argument(
         "--wab",
         "--wa-backup",
@@ -281,6 +274,13 @@ def main():
         action='store_true',
         help="Create a copy of the media seperated per chat in <MEDIA>/separated/ directory"
     )
+    parser.add_argument(
+        "--decrypt-chunk-size",
+        dest="decrypt_chunk_size",
+        default=1 * 1024 * 1024,
+        type=int,
+        help="Specify the chunk size for decrypting iOS backup, which may affect the decryption speed."
+    )
     parser.add_argument(
         "--enrich-from-vcards",
         dest="enrich_from_vcards",
@@ -293,7 +293,7 @@ def main():
         default=None,
         help="Use with --enrich-from-vcards. When numbers in the vcf file does not have a country code, this will be used. 1 is for US, 66 for Thailand etc. Most likely use the number of your own country"
     )
-    
+
     args = parser.parse_args()
 
     # Check for updates
@@ -457,7 +457,7 @@ def main():
             args.media = identifiers.DOMAIN
         if args.backup is not None:
             if not os.path.isdir(args.media):
-                ios_media_handler.extract_media(args.backup, identifiers, args.preserve_timestamp)
+                ios_media_handler.extract_media(args.backup, identifiers, args.decrypt_chunk_size)
             else:
                 print("WhatsApp directory already exists, skipping WhatsApp file extraction.")
         if args.db is None:

Whatsapp_Chat_Exporter/data_model.py

@@ -34,6 +34,7 @@ class ChatStore():
         self.their_avatar = None
         self.their_avatar_thumb = None
         self.status = None
+        self.media_base = ""
     
     def add_message(self, id, message):
         if not isinstance(message, Message):

Whatsapp_Chat_Exporter/ios_handler.py

@@ -246,10 +246,13 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=False
     while content is not None:
         file_path = f"{media_folder}/Message/{content['ZMEDIALOCALPATH']}"
         ZMESSAGE = content["ZMESSAGE"]
-        message = data[content["ZCONTACTJID"]].messages[ZMESSAGE]
+        contact = data[content["ZCONTACTJID"]]
+        message = contact.messages[ZMESSAGE]
         message.media = True
+        if contact.media_base == "":
+            contact.media_base = media_folder + "/"
         if os.path.isfile(file_path):
-            message.data = file_path
+            message.data = '/'.join(file_path.split("/")[1:])
             if content["ZVCARDSTRING"] is None:
                 guess = mime.guess_type(file_path)[0]
                 if guess is not None:
@@ -259,7 +262,7 @@ def media(db, data, media_folder, filter_date, filter_chat, separate_media=False
             else:
                 message.mime = content["ZVCARDSTRING"]
             if separate_media:
-                chat_display_name = slugify(data[content["ZCONTACTJID"]].name or message.sender \
+                chat_display_name = slugify(contact.name or message.sender \
                                             or content["ZCONTACTJID"].split('@')[0], True)
                 current_filename = file_path.split("/")[-1]
                 new_folder = os.path.join(media_folder, "separated", chat_display_name)

Whatsapp_Chat_Exporter/ios_media_handler.py

@@ -3,55 +3,59 @@
 import shutil
 import sqlite3
 import os
-import time
 import getpass
-import threading
 from Whatsapp_Chat_Exporter.utility import WhatsAppIdentifier
+from Whatsapp_Chat_Exporter.bplist import BPListReader
 try:
     from iphone_backup_decrypt import EncryptedBackup, RelativePath
-    from iphone_backup_decrypt import FailedToDecryptError
 except ModuleNotFoundError:
     support_encrypted = False
 else:
     support_encrypted = True
 
 
-def extract_encrypted(base_dir, password, identifiers, bplist_reader=None):
-    backup = EncryptedBackup(backup_directory=base_dir, passphrase=password, cleanup=False, check_same_thread=False)
-    print("Decrypting WhatsApp database...", end="")
+def extract_encrypted(base_dir, password, identifiers, decrypt_chunk_size):
+    print("Trying to decrypt the iOS backup...", end="")
+    backup = EncryptedBackup(
+        backup_directory=base_dir,
+        passphrase=password,
+        cleanup=False,
+        check_same_thread=False,
+        decrypt_chunk_size=decrypt_chunk_size
+    )
+    print("Done\nDecrypting WhatsApp database...", end="")
     try:
         backup.extract_file(
             relative_path=RelativePath.WHATSAPP_MESSAGES,
-            domain=identifiers.DOMAIN,
+            domain_like=identifiers.DOMAIN,
             output_filename=identifiers.MESSAGE
         )
         backup.extract_file(
             relative_path=RelativePath.WHATSAPP_CONTACTS,
-            domain=identifiers.DOMAIN,
+            domain_like=identifiers.DOMAIN,
             output_filename=identifiers.CONTACT
         )
-    except FailedToDecryptError:
+    except ValueError:
         print("Failed to decrypt backup: incorrect password?")
-        exit()
+        exit(7)
+    except FileNotFoundError:
+        print("Essential WhatsApp files are missing from the iOS backup.")
+        exit(6)
     else:
         print("Done")
-    extract_thread = threading.Thread(
-        target=backup.extract_files_by_domain,
-        args=(identifiers.DOMAIN, identifiers.DOMAIN, bplist_reader)
+
+    def extract_progress_handler(file_id, domain, relative_path, n, total_files):
+        if n % 100 == 0:
+            print(f"Decrypting and extracting files...({n}/{total_files})", end="\r")   
+        return True
+
+    backup.extract_files(
+        domain_like=identifiers.DOMAIN,
+        output_folder=identifiers.DOMAIN,
+        preserve_folders=True,
+        filter_callback=extract_progress_handler
     )
-    extract_thread.daemon = True
-    extract_thread.start()
-    dot = 0
-    while extract_thread.is_alive():
-        print(f"Decrypting and extracting files{'.' * dot}{' ' * (3 - dot)}", end="\r")
-        if dot < 3:
-            dot += 1
-            time.sleep(0.5)
-        else:
-            dot = 0
-            time.sleep(0.4)
-    print(f"All required files decrypted and extracted.", end="\n")
-    extract_thread.handled = True
+    print(f"All required files are decrypted and extracted.          ", end="\n")
     return backup
 
 
@@ -70,10 +74,7 @@ def is_encrypted(base_dir):
             return False
 
 
-def extract_media(base_dir, identifiers, preserve_timestamp=False):
-    if preserve_timestamp:
-        from Whatsapp_Chat_Exporter.bplist import BPListReader 
-        preserve_timestamp = BPListReader
+def extract_media(base_dir, identifiers, decrypt_chunk_size):
     if is_encrypted(base_dir):
         if not support_encrypted:
             print("You don't have the dependencies to handle encrypted backup.")
@@ -82,7 +83,7 @@ def extract_media(base_dir, identifiers, preserve_timestamp=False):
             return False
         print("Encryption detected on the backup!")
         password = getpass.getpass("Enter the password for the backup:")
-        extract_encrypted(base_dir, password, identifiers, preserve_timestamp)
+        extract_encrypted(base_dir, password, identifiers, decrypt_chunk_size)
     else:
         wts_db = os.path.join(base_dir, identifiers.MESSAGE[:2], identifiers.MESSAGE)
         contact_db = os.path.join(base_dir, identifiers.CONTACT[:2], identifiers.CONTACT)
@@ -136,11 +137,10 @@ def extract_media(base_dir, identifiers, preserve_timestamp=False):
                         pass
                 elif flags == 1:
                     shutil.copyfile(os.path.join(base_dir, folder, hashes), destination)
-                    if preserve_timestamp:
-                        metadata = BPListReader(row["metadata"]).parse()
-                        creation = metadata["$objects"][1]["Birth"]
-                        modification = metadata["$objects"][1]["LastModified"]
-                        os.utime(destination, (modification, modification))
+                    metadata = BPListReader(row["metadata"]).parse()
+                    creation = metadata["$objects"][1]["Birth"]
+                    modification = metadata["$objects"][1]["LastModified"]
+                    os.utime(destination, (modification, modification))
                 if row["_index"] % 100 == 0:
                     print(f"Extracting WhatsApp files...({row['_index']}/{total_row_number})", end="\r")
                 row = c.fetchone()

Whatsapp_Chat_Exporter/utility.py

@@ -93,6 +93,7 @@ def rendering(
                 w3css=w3css,
                 next=next,
                 status=chat.status,
+                media_base=chat.media_base
             )
         )
 

Whatsapp_Chat_Exporter/whatsapp.html

@@ -87,6 +87,7 @@
 			max-height: 100px !important;
 		}
 		</style>
+	<base href="{{ media_base }}" target="_blank">
 	</head>
 	<body>
 		<header class="w3-center w3-top">
@@ -143,7 +144,7 @@
 								{% else %}
 									{% if "image/" in msg.mime %}
 									<a href="{{ msg.data }}">
-										<img src="{{ msg.thumb if msg.thumb is not none else msg.data }}" {{ 'class="sticker"' | safe if msg.sticker }} />
+										<img src="{{ msg.thumb if msg.thumb is not none else msg.data }}" {{ 'class="sticker"' | safe if msg.sticker }} loading="lazy"/>
 									</a>
 									{% elif "audio/" in msg.mime %}
 									<audio controls="controls" autobuffer="autobuffer">
@@ -171,7 +172,7 @@
 						{% if not no_avatar and my_avatar is not none %}
 						<div class="w3-col m2 l2 pad-left-10">
 							<a href="{{ my_avatar }}">
-								<img src="{{ my_avatar }}" onerror="this.style.display='none'" class="avatar">
+								<img src="{{ my_avatar }}" onerror="this.style.display='none'" class="avatar" loading="lazy">
 							</a>
 						</div>
 						{% endif %}
@@ -191,9 +192,9 @@
 						{% if not no_avatar %}
 						<div class="w3-col m2 l2">
 								{% if their_avatar is not none %}
-								<a href="{{ their_avatar }}"><img src="{{ their_avatar_thumb or '' }}" onerror="this.style.display='none'" class="avatar"></a>
+								<a href="{{ their_avatar }}"><img src="{{ their_avatar_thumb or '' }}" onerror="this.style.display='none'" class="avatar" loading="lazy"></a>
 								{% else %}
-								<img src="{{ their_avatar_thumb or '' }}" onerror="this.style.display='none'" class="avatar">
+								<img src="{{ their_avatar_thumb or '' }}" onerror="this.style.display='none'" class="avatar" loading="lazy">
 								{% endif %}
 						</div>
 						<div class="w3-col m10 l10">
@@ -227,7 +228,7 @@
 								{% else %}
 									{% if "image/" in msg.mime %}
 									<a href="{{ msg.data }}">
-										<img src="{{ msg.thumb if msg.thumb is not none else msg.data }}" {{ 'class="sticker"' | safe if msg.sticker }} />
+										<img src="{{ msg.thumb if msg.thumb is not none else msg.data }}" {{ 'class="sticker"' | safe if msg.sticker }} loading="lazy"/>
 									</a>
 									{% elif "audio/" in msg.mime %}
 									<audio controls="controls" autobuffer="autobuffer">