From 33763b5f412de428cc09f699d50e48d74983a439 Mon Sep 17 00:00:00 2001 From: Knugi <24708955+KnugiHK@users.noreply.github.com> Date: Sun, 9 Jun 2024 07:36:58 +0000 Subject: [PATCH 01/19] Update compile-binary.yml --- .github/workflows/compile-binary.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/compile-binary.yml b/.github/workflows/compile-binary.yml index 4eca4a0..be1b723 100644 --- a/.github/workflows/compile-binary.yml +++ b/.github/workflows/compile-binary.yml @@ -16,11 +16,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pycryptodome javaobj-py3 ordered-set zstandard nuitka + pip install pycryptodome javaobj-py3 ordered-set zstandard nuitka==2.3 pip install . - name: Build binary with Nuitka run: | @@ -40,11 +40,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pycryptodome javaobj-py3 ordered-set zstandard nuitka + pip install pycryptodome javaobj-py3 ordered-set zstandard nuitka==2.3 pip install . - name: Build binary with Nuitka run: | @@ -64,11 +64,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pycryptodome javaobj-py3 ordered-set zstandard nuitka + pip install pycryptodome javaobj-py3 ordered-set zstandard nuitka==2.3 pip install . - name: Build binary with Nuitka run: | From be469aed935e48cc84d44e8cffa78daf0a29c023 Mon Sep 17 00:00:00 2001 From: Bnaya Peretz Date: Mon, 8 Jul 2024 23:06:41 +0300 Subject: [PATCH 02/19] Bnaya's assorted features --- .gitignore | 11 +++ Whatsapp_Chat_Exporter/__main__.py | 70 ++++++++++++++- Whatsapp_Chat_Exporter/android_handler.py | 2 + .../contacts_names_from_vcards.py | 88 +++++++++++++++++++ .../contacts_names_from_vcards_test.py | 22 +++++ Whatsapp_Chat_Exporter/utility.py | 7 ++ setup.py | 7 +- 7 files changed, 201 insertions(+), 6 deletions(-) create mode 100644 Whatsapp_Chat_Exporter/contacts_names_from_vcards.py create mode 100644 Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py diff --git a/.gitignore b/.gitignore index dd82d78..5831f34 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,14 @@ dmypy.json *.onefile-build/ *.exe __main__ + + +# Dev time intermidiates & temp files +result/ +WhatsApp/ +/*.db +/*.db-* +/myout +/msgstore.db +/myout-json +.vscode/ \ No newline at end of file diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index a332181..236801a 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -7,10 +7,17 @@ import shutil import json import string import glob +try: + import vobject +except ModuleNotFoundError: + vcards_deps_installed = False +else: + vcards_deps_installed = True from Whatsapp_Chat_Exporter import exported_handler, android_handler from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler +from Whatsapp_Chat_Exporter.contacts_names_from_vcards import ContactsNamesFromVCards, readVCardsFile from Whatsapp_Chat_Exporter.data_model import ChatStore -from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType +from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, is_chat_empty from Whatsapp_Chat_Exporter.utility import check_update, import_from_json from argparse import ArgumentParser, SUPPRESS from datetime import datetime @@ -85,6 +92,18 @@ def main(): type=str, const="result.json", help="Save the result to a single JSON file (default if present: result.json)") + parser.add_argument( + '--avoidJSONEnsureAscii', + dest='avoid_json_ensure_ascii', + default=False, + action='store_true', + help="Don't encode non-ascii chars in the output json files") + parser.add_argument( + '--prettyPrintJson', + dest='pretty_print_json', + default=False, + action='store_true', + help="Pretty print the output json") parser.add_argument( '-d', '--db', @@ -239,6 +258,13 @@ def main(): metavar="phone number", help="Exclude chats that match the supplied phone number" ) + parser.add_argument( + "--filter-empty", + dest="filter_empty", + default=False, + action='store_true', + help="Exclude empty chats or with zero messages with content" + ) parser.add_argument( "--per-chat", dest="json_per_chat", @@ -253,6 +279,20 @@ def main(): action='store_true', help="Create a copy of the media seperated per chat in /separated/ directory" ) + parser.add_argument( + "--enrich-names-from-vcards", + dest="enrich_names_from_vcards", + default=None, + help="Path to an exported vcf file from google contacts export, add names missing from wab database" + ) + + parser.add_argument( + "--default-country-code-for-enrich-names-from-vcards", + dest="default_country_code_for_enrich_names_from_vcards", + default=None, + help="When numbers in enrich-names-from-vcards does not have country code, this will be used. 1 is for US, 66 for Thailand etc. most likely use the number of your own country" + ) + args = parser.parse_args() # Check for updates @@ -317,9 +357,19 @@ def main(): if not chat.isnumeric(): parser.error("Enter a phone number in the chat filter. See https://wts.knugi.dev/docs?dest=chat") filter_chat = (args.filter_chat_include, args.filter_chat_exclude) + if args.enrich_names_from_vcards is not None and args.default_country_code_for_enrich_names_from_vcards is None: + parser.error("When --enrich-names-from-vcards is provided, you must also set --default-country-code-for-enrich-names-from-vcards") data = {} + contacts_names_from_vcards_enricher = ContactsNamesFromVCards() + + if args.enrich_names_from_vcards is not None: + if not vcards_deps_installed: + parser.error("To use --enrich-names-from-vcards, you must install whatsapp-chat-exporter[vcards]") + + contacts_names_from_vcards_enricher.load_vcf_file(args.enrich_names_from_vcards, args.default_country_code_for_enrich_names_from_vcards) + if args.android: contacts = android_handler.contacts messages = android_handler.messages @@ -429,6 +479,12 @@ def main(): if args.android: android_handler.calls(db, data, args.timezone_offset, filter_chat) if not args.no_html: + if contacts_names_from_vcards_enricher.should_enrich_names_from_vCards(): + contacts_names_from_vcards_enricher.enrich_names_from_vCards(data) + + if (args.filter_empty): + data = {k: v for k, v in data.items() if not is_chat_empty(v)} + create_html( data, args.output, @@ -487,11 +543,18 @@ def main(): ) if args.json and not args.import_json: + if (args.filter_empty): + data = {k: v for k, v in data.items() if not is_chat_empty(v)} + + if contacts_names_from_vcards_enricher.should_enrich_names_from_vCards(): + contacts_names_from_vcards_enricher.enrich_names_from_vCards(data) + if isinstance(data[next(iter(data))], ChatStore): data = {jik: chat.to_json() for jik, chat in data.items()} + if not args.json_per_chat: with open(args.json, "w") as f: - data = json.dumps(data) + data = json.dumps(data, ensure_ascii=not args.avoid_json_ensure_ascii, indent=2 if args.pretty_print_json else None) print(f"\nWriting JSON file...({int(len(data)/1024/1024)}MB)") f.write(data) else: @@ -506,7 +569,8 @@ def main(): else: contact = jik.replace('+', '') with open(f"{args.json}/{contact}.json", "w") as f: - f.write(json.dumps(data[jik])) + file_content_to_write = json.dumps(data[jik], ensure_ascii=not args.avoid_json_ensure_ascii, indent=2 if args.pretty_print_json else None) + f.write(file_content_to_write) print(f"Writing JSON file...({index + 1}/{total})", end="\r") print() else: diff --git a/Whatsapp_Chat_Exporter/android_handler.py b/Whatsapp_Chat_Exporter/android_handler.py index 2024112..193d5df 100644 --- a/Whatsapp_Chat_Exporter/android_handler.py +++ b/Whatsapp_Chat_Exporter/android_handler.py @@ -158,6 +158,8 @@ def contacts(db, data): c.execute("""SELECT count() FROM wa_contacts""") total_row_number = c.fetchone()[0] print(f"Processing contacts...({total_row_number})") + if total_row_number == 0: + print("No contacts profiles found in database, consider using --enrich-names-from-vcards when exported contacts from google") c.execute("""SELECT jid, COALESCE(display_name, wa_name) as display_name, status FROM wa_contacts; """) row = c.fetchone() diff --git a/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py b/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py new file mode 100644 index 0000000..b1146b0 --- /dev/null +++ b/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py @@ -0,0 +1,88 @@ +import itertools +from typing import List, TypedDict + +try: + import vobject +except ModuleNotFoundError: + vcards_deps_installed = False +else: + vcards_deps_installed = True + +class ContactsNamesFromVCards: + def __init__(self) -> None: + self.l = [] + + def should_enrich_names_from_vCards(self): + return len(self.l) > 0 + + def load_vcf_file(self, vcfFilePath: str, default_country_calling_code: str): + if not vcards_deps_installed: + raise Exception('Invariant: vobject is missing') + self.l = readVCardsFile(vcfFilePath, default_country_calling_code) + + def enrich_names_from_vCards(self, chats): + for counter, (number, name) in enumerate(self.l): + # short number must be a bad contact, lets skip it + if len(number) <= 5: + continue + + for counter, chat in enumerate(filter_dict_by_prefix(chats, number).values()): + if not hasattr(chat, 'name') or (hasattr(chat, 'name') and chat.name is None): + setattr(chat, 'name', name) + + +def readVCardsFile(vcfFilePath, default_country_calling_code: str): + contacts = [] + with open(vcfFilePath, mode="r") as f: + reader = vobject.readComponents(f) + for row in reader: + if not hasattr(row, 'fn'): + continue + + if not hasattr(row, 'tel'): + continue + + contact: ExportedGoogleContactVCARDRawNumbers = { + "full_name": row.fn.value, + "numbers": list(map(lambda tel:tel.value, row.tel_list)), + } + + contacts.append(contact) + + step2 = createNumberToNameDicts(contacts, default_country_calling_code) + + return step2 + + +def filter_dict_by_prefix(d, prefix: str): + return {k: v for k, v in d.items() if k.startswith(prefix)} + +def createNumberToNameDicts(inContacts, default_country_calling_code: str): + outContacts = list(itertools.chain.from_iterable( + [[normalize_number(num, default_country_calling_code), f"{contact['full_name']} ({i+1})" if len(contact['numbers']) > 1 else contact['full_name']] + for i, num in enumerate(contact['numbers'])] + for contact in inContacts + )) + + return outContacts + +class ExportedGoogleContactVCARDRawNumbers(TypedDict): + full_name: str + numbers: List[str] + +def normalize_number(number: str, default_country_calling_code: str): + afterSomeCleaning = number.replace('(', '').replace(')', '').replace(' ', '').replace('-', '') + + # A number that starts with a + or 00 means it already have country_calling_code + if afterSomeCleaning.startswith('+'): + afterSomeCleaning = afterSomeCleaning.replace('+', '') + elif afterSomeCleaning.startswith('00'): + afterSomeCleaning = afterSomeCleaning[2:] + else: + # Remove leading zero + if afterSomeCleaning.startswith('0'): + afterSomeCleaning = afterSomeCleaning[1:] + + afterSomeCleaning = default_country_calling_code + afterSomeCleaning + + return afterSomeCleaning \ No newline at end of file diff --git a/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py b/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py new file mode 100644 index 0000000..74205fc --- /dev/null +++ b/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py @@ -0,0 +1,22 @@ +# from contacts_names_from_vcards import readVCardsFile + +from Whatsapp_Chat_Exporter.contacts_names_from_vcards import normalize_number, readVCardsFile + + +def test_readVCardsFile(): + l = readVCardsFile("contacts.vcf", "973") + + assert len(l) > 0 + +def test_createNumberToNameDicts(): + pass + +def test_fuzzy_match_numbers(): + pass + +def test_normalize_number(): + assert normalize_number('0531234567', '1') == '1531234567' + assert normalize_number('001531234567', '2') == '1531234567' + assert normalize_number('+1531234567', '34') == '1531234567' + assert normalize_number('053(123)4567', '34') == '34531234567' + assert normalize_number('0531-234-567', '58') == '58531234567' \ No newline at end of file diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index fc75ae8..ad8e1e1 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -344,3 +344,10 @@ class JidType(IntEnum): GROUP = 1 SYSTEM_BROADCAST = 5 STATUS = 11 + +def _is_message_empty(message): + return (message.data is None or message.data == "") and not message.media + +def is_chat_empty(chat: ChatStore): + is_empty = len(chat.messages) == 0 or all(_is_message_empty(f) for f in chat.messages.values()) + return is_empty diff --git a/setup.py b/setup.py index 98429ac..5fa5484 100644 --- a/setup.py +++ b/setup.py @@ -55,9 +55,10 @@ setuptools.setup( 'crypt12': ["pycryptodome"], 'crypt14': ["pycryptodome"], 'crypt15': ["pycryptodome", "javaobj-py3"], - 'all': ["pycryptodome", "javaobj-py3"], - 'everything': ["pycryptodome", "javaobj-py3"], - 'backup': ["pycryptodome", "javaobj-py3"] + 'all': ["pycryptodome", "javaobj-py3", "vobject"], + 'everything': ["pycryptodome", "javaobj-py3", "vobject"], + 'backup': ["pycryptodome", "javaobj-py3"], + 'vcards': ["vobject", "pycryptodome", "javaobj-py3"], }, entry_points={ "console_scripts": [ From 823ed663e7d29bc7e8d1e4186f37f78bc52fa11b Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 12:12:32 +0800 Subject: [PATCH 03/19] The check happened in __main__ already --- Whatsapp_Chat_Exporter/__main__.py | 2 +- Whatsapp_Chat_Exporter/contacts_names_from_vcards.py | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 236801a..228a644 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -546,8 +546,8 @@ def main(): if (args.filter_empty): data = {k: v for k, v in data.items() if not is_chat_empty(v)} - if contacts_names_from_vcards_enricher.should_enrich_names_from_vCards(): contacts_names_from_vcards_enricher.enrich_names_from_vCards(data) + if contact_store.should_enrich_from_vcards(): if isinstance(data[next(iter(data))], ChatStore): data = {jik: chat.to_json() for jik, chat in data.items()} diff --git a/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py b/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py index b1146b0..9784ebf 100644 --- a/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py +++ b/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py @@ -1,23 +1,15 @@ import itertools from typing import List, TypedDict - -try: - import vobject -except ModuleNotFoundError: - vcards_deps_installed = False -else: - vcards_deps_installed = True +import vobject class ContactsNamesFromVCards: def __init__(self) -> None: self.l = [] - def should_enrich_names_from_vCards(self): + def should_enrich_from_vcards(self): return len(self.l) > 0 def load_vcf_file(self, vcfFilePath: str, default_country_calling_code: str): - if not vcards_deps_installed: - raise Exception('Invariant: vobject is missing') self.l = readVCardsFile(vcfFilePath, default_country_calling_code) def enrich_names_from_vCards(self, chats): From 0423fdabdadf6609ed6a0e9a8ae74eff23224b55 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 12:12:55 +0800 Subject: [PATCH 04/19] Update contacts_names_from_vcards_test.py --- Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py b/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py index 74205fc..dbe8d9f 100644 --- a/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py +++ b/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py @@ -19,4 +19,4 @@ def test_normalize_number(): assert normalize_number('001531234567', '2') == '1531234567' assert normalize_number('+1531234567', '34') == '1531234567' assert normalize_number('053(123)4567', '34') == '34531234567' - assert normalize_number('0531-234-567', '58') == '58531234567' \ No newline at end of file + assert normalize_number('0531-234-567', '58') == '58531234567' From 4886587065753c8daafb4322dc3e566e17b8fd42 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 12:15:45 +0800 Subject: [PATCH 05/19] Make the message clear --- Whatsapp_Chat_Exporter/android_handler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Whatsapp_Chat_Exporter/android_handler.py b/Whatsapp_Chat_Exporter/android_handler.py index 193d5df..094d3e1 100644 --- a/Whatsapp_Chat_Exporter/android_handler.py +++ b/Whatsapp_Chat_Exporter/android_handler.py @@ -157,9 +157,11 @@ def contacts(db, data): c = db.cursor() c.execute("""SELECT count() FROM wa_contacts""") total_row_number = c.fetchone()[0] - print(f"Processing contacts...({total_row_number})") if total_row_number == 0: - print("No contacts profiles found in database, consider using --enrich-names-from-vcards when exported contacts from google") + print("No contacts profiles found in the default database, consider using --enrich-from-vcards for adopting names from exported contacts from Google") + return False + else: + print(f"Processing contacts...({total_row_number})") c.execute("""SELECT jid, COALESCE(display_name, wa_name) as display_name, status FROM wa_contacts; """) row = c.fetchone() From 83fefe585b8e4c03b2e0dad10f504a657df9a5a6 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 12:18:18 +0800 Subject: [PATCH 06/19] Refactoring The added code should be placed above any platform specific code --- Whatsapp_Chat_Exporter/utility.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index ad8e1e1..09e6671 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -167,6 +167,12 @@ def get_chat_condition(filter, include, column): else: return "" +def _is_message_empty(message): + return (message.data is None or message.data == "") and not message.media + +def is_chat_empty(chat: ChatStore): + return len(chat.messages) == 0 or all(_is_message_empty(message) for message in chat.messages.values()) + # Android Specific CRYPT14_OFFSETS = ( @@ -344,10 +350,3 @@ class JidType(IntEnum): GROUP = 1 SYSTEM_BROADCAST = 5 STATUS = 11 - -def _is_message_empty(message): - return (message.data is None or message.data == "") and not message.media - -def is_chat_empty(chat: ChatStore): - is_empty = len(chat.messages) == 0 or all(_is_message_empty(f) for f in chat.messages.values()) - return is_empty From 7bb2fb242090b93b74c3b92ac183ba94026a31a4 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 12:26:54 +0800 Subject: [PATCH 07/19] Refactoring --pretty-print-json and --avoid_encoding_json options --- Whatsapp_Chat_Exporter/__main__.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 228a644..bc61eb8 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -93,17 +93,19 @@ def main(): const="result.json", help="Save the result to a single JSON file (default if present: result.json)") parser.add_argument( - '--avoidJSONEnsureAscii', - dest='avoid_json_ensure_ascii', + '--avoid_encoding_json', + dest='avoid_encoding_json', default=False, action='store_true', - help="Don't encode non-ascii chars in the output json files") + help="Don't encode non-ascii characters in the output JSON files") parser.add_argument( - '--prettyPrintJson', + '--pretty-print-json', dest='pretty_print_json', - default=False, - action='store_true', - help="Pretty print the output json") + default=None, + nargs='?', + const=2, + type=int, + help="Pretty print the output JSON.") parser.add_argument( '-d', '--db', @@ -554,7 +556,11 @@ def main(): if not args.json_per_chat: with open(args.json, "w") as f: - data = json.dumps(data, ensure_ascii=not args.avoid_json_ensure_ascii, indent=2 if args.pretty_print_json else None) + data = json.dumps( + data, + ensure_ascii=not args.avoid_encoding_json, + indent=args.pretty_print_json + ) print(f"\nWriting JSON file...({int(len(data)/1024/1024)}MB)") f.write(data) else: @@ -569,7 +575,7 @@ def main(): else: contact = jik.replace('+', '') with open(f"{args.json}/{contact}.json", "w") as f: - file_content_to_write = json.dumps(data[jik], ensure_ascii=not args.avoid_json_ensure_ascii, indent=2 if args.pretty_print_json else None) + file_content_to_write = json.dumps(data[jik], ensure_ascii=not args.avoid_encoding_json, indent=2 if args.pretty_print_json else None) f.write(file_content_to_write) print(f"Writing JSON file...({index + 1}/{total})", end="\r") print() From 8f304f1c48c6570f984bc92a722a7614e294ba52 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 12:43:03 +0800 Subject: [PATCH 08/19] There should be no need for enumerate() --- Whatsapp_Chat_Exporter/contacts_names_from_vcards.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py b/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py index 9784ebf..e3cb30f 100644 --- a/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py +++ b/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py @@ -13,12 +13,12 @@ class ContactsNamesFromVCards: self.l = readVCardsFile(vcfFilePath, default_country_calling_code) def enrich_names_from_vCards(self, chats): - for counter, (number, name) in enumerate(self.l): + for number, name in self.l: # short number must be a bad contact, lets skip it if len(number) <= 5: continue - for counter, chat in enumerate(filter_dict_by_prefix(chats, number).values()): + for chat in filter_dict_by_prefix(chats, number).values(): if not hasattr(chat, 'name') or (hasattr(chat, 'name') and chat.name is None): setattr(chat, 'name', name) From 5b97d6013aaed9b6975a594786ce0bbbe0a8b258 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 13:33:47 +0800 Subject: [PATCH 09/19] Refactor core code of mapping vcards --- Whatsapp_Chat_Exporter/__main__.py | 8 +- .../contacts_names_from_vcards.py | 104 +++++++++--------- .../contacts_names_from_vcards_test.py | 8 +- 3 files changed, 58 insertions(+), 62 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index bc61eb8..3cb0256 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -15,7 +15,7 @@ else: vcards_deps_installed = True from Whatsapp_Chat_Exporter import exported_handler, android_handler from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler -from Whatsapp_Chat_Exporter.contacts_names_from_vcards import ContactsNamesFromVCards, readVCardsFile +from Whatsapp_Chat_Exporter.contacts_names_from_vcards import ContactsFromVCards, read_vcards_file from Whatsapp_Chat_Exporter.data_model import ChatStore from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, is_chat_empty from Whatsapp_Chat_Exporter.utility import check_update, import_from_json @@ -364,7 +364,7 @@ def main(): data = {} - contacts_names_from_vcards_enricher = ContactsNamesFromVCards() + contacts_names_from_vcards_enricher = ContactsFromVCards() if args.enrich_names_from_vcards is not None: if not vcards_deps_installed: @@ -482,7 +482,7 @@ def main(): android_handler.calls(db, data, args.timezone_offset, filter_chat) if not args.no_html: if contacts_names_from_vcards_enricher.should_enrich_names_from_vCards(): - contacts_names_from_vcards_enricher.enrich_names_from_vCards(data) + contacts_names_from_vcards_enricher.enrich_from_vcards(data) if (args.filter_empty): data = {k: v for k, v in data.items() if not is_chat_empty(v)} @@ -548,7 +548,7 @@ def main(): if (args.filter_empty): data = {k: v for k, v in data.items() if not is_chat_empty(v)} - contacts_names_from_vcards_enricher.enrich_names_from_vCards(data) + contacts_names_from_vcards_enricher.enrich_from_vcards(data) if contact_store.should_enrich_from_vcards(): if isinstance(data[next(iter(data))], ChatStore): diff --git a/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py b/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py index e3cb30f..58d5eaa 100644 --- a/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py +++ b/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py @@ -2,79 +2,77 @@ import itertools from typing import List, TypedDict import vobject -class ContactsNamesFromVCards: + +class ExportedContactNumbers(TypedDict): + full_name: str + numbers: List[str] + + +class ContactsFromVCards: def __init__(self) -> None: - self.l = [] - + self.contact_mapping = [] + def should_enrich_from_vcards(self): - return len(self.l) > 0 - - def load_vcf_file(self, vcfFilePath: str, default_country_calling_code: str): - self.l = readVCardsFile(vcfFilePath, default_country_calling_code) - - def enrich_names_from_vCards(self, chats): - for number, name in self.l: + return len(self.contact_mapping) > 0 + + def load_vcf_file(self, vcf_file_path: str, default_country_code: str): + self.contact_mapping = read_vcards_file(vcf_file_path, default_country_code) + + def enrich_from_vcards(self, chats): + for number, name in self.contact_mapping: # short number must be a bad contact, lets skip it if len(number) <= 5: continue - for chat in filter_dict_by_prefix(chats, number).values(): + for chat in filter_chats_by_prefix(chats, number).values(): if not hasattr(chat, 'name') or (hasattr(chat, 'name') and chat.name is None): setattr(chat, 'name', name) -def readVCardsFile(vcfFilePath, default_country_calling_code: str): +def read_vcards_file(vcf_file_path, default_country_code: str): contacts = [] - with open(vcfFilePath, mode="r") as f: + with open(vcf_file_path, mode="r", encoding="utf-8") as f: reader = vobject.readComponents(f) for row in reader: - if not hasattr(row, 'fn'): + if not hasattr(row, 'fn') or not hasattr(row, 'tel'): continue - - if not hasattr(row, 'tel'): - continue - - contact: ExportedGoogleContactVCARDRawNumbers = { - "full_name": row.fn.value, - "numbers": list(map(lambda tel:tel.value, row.tel_list)), - } + contact: ExportedContactNumbers = { + "full_name": row.fn.value, + "numbers": list(map(lambda tel: tel.value, row.tel_list)), + } contacts.append(contact) - step2 = createNumberToNameDicts(contacts, default_country_calling_code) + return map_number_to_name(contacts, default_country_code) - return step2 - -def filter_dict_by_prefix(d, prefix: str): - return {k: v for k, v in d.items() if k.startswith(prefix)} +def filter_chats_by_prefix(chats, prefix: str): + return {k: v for k, v in chats.items() if k.startswith(prefix)} -def createNumberToNameDicts(inContacts, default_country_calling_code: str): - outContacts = list(itertools.chain.from_iterable( - [[normalize_number(num, default_country_calling_code), f"{contact['full_name']} ({i+1})" if len(contact['numbers']) > 1 else contact['full_name']] - for i, num in enumerate(contact['numbers'])] - for contact in inContacts - )) - return outContacts - -class ExportedGoogleContactVCARDRawNumbers(TypedDict): - full_name: str - numbers: List[str] - -def normalize_number(number: str, default_country_calling_code: str): - afterSomeCleaning = number.replace('(', '').replace(')', '').replace(' ', '').replace('-', '') +def map_number_to_name(contacts, default_country_code: str): + mapping = [] + for contact in contacts: + for index, num in enumerate(contact['numbers']): + normalized = normalize_number(num, default_country_code) + if len(contact['numbers']) > 1: + name = f"{contact['full_name']} ({index+1})" + else: + name = contact['full_name'] + mapping.append((normalized, name)) + return mapping - # A number that starts with a + or 00 means it already have country_calling_code - if afterSomeCleaning.startswith('+'): - afterSomeCleaning = afterSomeCleaning.replace('+', '') - elif afterSomeCleaning.startswith('00'): - afterSomeCleaning = afterSomeCleaning[2:] - else: - # Remove leading zero - if afterSomeCleaning.startswith('0'): - afterSomeCleaning = afterSomeCleaning[1:] - afterSomeCleaning = default_country_calling_code + afterSomeCleaning - - return afterSomeCleaning \ No newline at end of file +def normalize_number(number: str, country_code: str): + # Clean the number + number = ''.join(c for c in number if c.isdigit() or c == "+") + + # A number that starts with a + or 00 means it already have a country code + for starting_char in ('+', "00"): + if number.startswith(starting_char): + return number[len(starting_char):] + + # leading zero should be removed + if starting_char == '0': + number = number[1:] + return country_code + number # fall back diff --git a/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py b/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py index dbe8d9f..6f18507 100644 --- a/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py +++ b/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py @@ -1,14 +1,12 @@ # from contacts_names_from_vcards import readVCardsFile -from Whatsapp_Chat_Exporter.contacts_names_from_vcards import normalize_number, readVCardsFile +from Whatsapp_Chat_Exporter.contacts_names_from_vcards import normalize_number, read_vcards_file def test_readVCardsFile(): - l = readVCardsFile("contacts.vcf", "973") - - assert len(l) > 0 + assert len(read_vcards_file("contacts.vcf", "973")) > 0 -def test_createNumberToNameDicts(): +def test_create_number_to_name_dicts(): pass def test_fuzzy_match_numbers(): From b301dd22d08282f552e431ac0039866ea2e88850 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 13:47:07 +0800 Subject: [PATCH 10/19] Refactoring --- Whatsapp_Chat_Exporter/__main__.py | 38 ++++++++++++++++-------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 3cb0256..28254dd 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -15,7 +15,7 @@ else: vcards_deps_installed = True from Whatsapp_Chat_Exporter import exported_handler, android_handler from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler -from Whatsapp_Chat_Exporter.contacts_names_from_vcards import ContactsFromVCards, read_vcards_file +from Whatsapp_Chat_Exporter.contacts_names_from_vcards import ContactsFromVCards from Whatsapp_Chat_Exporter.data_model import ChatStore from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, is_chat_empty from Whatsapp_Chat_Exporter.utility import check_update, import_from_json @@ -282,17 +282,16 @@ def main(): help="Create a copy of the media seperated per chat in /separated/ directory" ) parser.add_argument( - "--enrich-names-from-vcards", - dest="enrich_names_from_vcards", + "--enrich-from-vcards", + dest="enrich_from_vcards", default=None, - help="Path to an exported vcf file from google contacts export, add names missing from wab database" + help="Path to an exported vcf file from Google contacts export. Add names missing from WhatsApp's default database" ) - parser.add_argument( - "--default-country-code-for-enrich-names-from-vcards", - dest="default_country_code_for_enrich_names_from_vcards", + "--default-country-code", + dest="default_contry_code", default=None, - help="When numbers in enrich-names-from-vcards does not have country code, this will be used. 1 is for US, 66 for Thailand etc. most likely use the number of your own country" + help="Use with --enrich-from-vcards. When numbers in the vcf file does not have a country code, this will be used. 1 is for US, 66 for Thailand etc. Most likely use the number of your own country" ) args = parser.parse_args() @@ -319,6 +318,8 @@ def main(): (args.json[-5:] == ".json" and os.path.isfile(args.json[:-5])) ): parser.error("When --per-chat is enabled, the destination of --json must be a directory.") + if args.enrich_from_vcards is not None and args.default_contry_code is None: + parser.error("When --enrich-from-vcards is provided, you must also set --default-country-code") if args.filter_date is not None: if " - " in args.filter_date: start, end = args.filter_date.split(" - ") @@ -359,18 +360,19 @@ def main(): if not chat.isnumeric(): parser.error("Enter a phone number in the chat filter. See https://wts.knugi.dev/docs?dest=chat") filter_chat = (args.filter_chat_include, args.filter_chat_exclude) - if args.enrich_names_from_vcards is not None and args.default_country_code_for_enrich_names_from_vcards is None: - parser.error("When --enrich-names-from-vcards is provided, you must also set --default-country-code-for-enrich-names-from-vcards") data = {} - contacts_names_from_vcards_enricher = ContactsFromVCards() + contact_store = ContactsFromVCards() - if args.enrich_names_from_vcards is not None: + if args.enrich_from_vcards is not None: if not vcards_deps_installed: - parser.error("To use --enrich-names-from-vcards, you must install whatsapp-chat-exporter[vcards]") - - contacts_names_from_vcards_enricher.load_vcf_file(args.enrich_names_from_vcards, args.default_country_code_for_enrich_names_from_vcards) + parser.error( + "You don't have the dependency to enrich contacts with vCard.\n" + "Read more on how to deal with enriching contacts:\n" + "https://github.com/KnugiHK/Whatsapp-Chat-Exporter/blob/main/README.md#usage" + ) + contact_store.load_vcf_file(args.enrich_from_vcards, args.default_contry_code) if args.android: contacts = android_handler.contacts @@ -481,8 +483,8 @@ def main(): if args.android: android_handler.calls(db, data, args.timezone_offset, filter_chat) if not args.no_html: - if contacts_names_from_vcards_enricher.should_enrich_names_from_vCards(): - contacts_names_from_vcards_enricher.enrich_from_vcards(data) + if contact_store.should_enrich_from_vcards(): + contact_store.enrich_from_vcards(data) if (args.filter_empty): data = {k: v for k, v in data.items() if not is_chat_empty(v)} @@ -548,8 +550,8 @@ def main(): if (args.filter_empty): data = {k: v for k, v in data.items() if not is_chat_empty(v)} - contacts_names_from_vcards_enricher.enrich_from_vcards(data) if contact_store.should_enrich_from_vcards(): + contact_store.enrich_from_vcards(data) if isinstance(data[next(iter(data))], ChatStore): data = {jik: chat.to_json() for jik, chat in data.items()} From 6e37061e711f5a1cfd2fc0124794424692dc0253 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 13:52:07 +0800 Subject: [PATCH 11/19] Rename module --- Whatsapp_Chat_Exporter/__main__.py | 2 +- .../{contacts_names_from_vcards.py => contacts_from_vcards.py} | 0 ...s_names_from_vcards_test.py => contacts_from_vcards_test.py} | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename Whatsapp_Chat_Exporter/{contacts_names_from_vcards.py => contacts_from_vcards.py} (100%) rename Whatsapp_Chat_Exporter/{contacts_names_from_vcards_test.py => contacts_from_vcards_test.py} (85%) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 28254dd..7f49e44 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -15,7 +15,7 @@ else: vcards_deps_installed = True from Whatsapp_Chat_Exporter import exported_handler, android_handler from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler -from Whatsapp_Chat_Exporter.contacts_names_from_vcards import ContactsFromVCards +from Whatsapp_Chat_Exporter.contacts_from_vcards import ContactsFromVCards from Whatsapp_Chat_Exporter.data_model import ChatStore from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, is_chat_empty from Whatsapp_Chat_Exporter.utility import check_update, import_from_json diff --git a/Whatsapp_Chat_Exporter/contacts_names_from_vcards.py b/Whatsapp_Chat_Exporter/contacts_from_vcards.py similarity index 100% rename from Whatsapp_Chat_Exporter/contacts_names_from_vcards.py rename to Whatsapp_Chat_Exporter/contacts_from_vcards.py diff --git a/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py b/Whatsapp_Chat_Exporter/contacts_from_vcards_test.py similarity index 85% rename from Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py rename to Whatsapp_Chat_Exporter/contacts_from_vcards_test.py index 6f18507..bd4929b 100644 --- a/Whatsapp_Chat_Exporter/contacts_names_from_vcards_test.py +++ b/Whatsapp_Chat_Exporter/contacts_from_vcards_test.py @@ -1,6 +1,6 @@ # from contacts_names_from_vcards import readVCardsFile -from Whatsapp_Chat_Exporter.contacts_names_from_vcards import normalize_number, read_vcards_file +from Whatsapp_Chat_Exporter.contacts_from_vcards import normalize_number, read_vcards_file def test_readVCardsFile(): From 09e5e1a7565fdb4cb8091d2582447ece57316189 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 13:59:47 +0800 Subject: [PATCH 12/19] Update readme for enriching contacts --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index a5c6b10..2fd2b2f 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,13 @@ Simply invoke the following command from shell. ```sh wtsexporter -a ``` +#### Enriching Contact from vCard +Usually, the default WhatsApp contact database extracted from your phone will contains the contact names and the exporter will use it to map your chats. However, some reported cases showed that the database could has never been populated. +In this case, you can export your contacts to a vCard file from your phone or a cloud provider like Google Contacts. Then, install the necessary dependency and run the following command from the shell: +```sh +pip install whatsapp-chat-exporter["vcards"] +wtsexporter -a --enrich-from-vcard contacts.vcf --default-country-code 852 +``` ### Encrypted Android WhatsApp Backup In order to support the decryption, install pycryptodome if it is not installed From 80bdc4414a33784f180bbc0c41a087e822fa7932 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 14:12:21 +0800 Subject: [PATCH 13/19] Refactoring empty filtering --- Whatsapp_Chat_Exporter/__main__.py | 24 ++++++++++++----------- Whatsapp_Chat_Exporter/android_handler.py | 7 ++++--- Whatsapp_Chat_Exporter/utility.py | 2 +- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 7f49e44..37f9b54 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -17,7 +17,7 @@ from Whatsapp_Chat_Exporter import exported_handler, android_handler from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler from Whatsapp_Chat_Exporter.contacts_from_vcards import ContactsFromVCards from Whatsapp_Chat_Exporter.data_model import ChatStore -from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, is_chat_empty +from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, chat_is_empty from Whatsapp_Chat_Exporter.utility import check_update, import_from_json from argparse import ArgumentParser, SUPPRESS from datetime import datetime @@ -261,11 +261,11 @@ def main(): help="Exclude chats that match the supplied phone number" ) parser.add_argument( - "--filter-empty", + "--dont-filter-empty", dest="filter_empty", default=False, action='store_true', - help="Exclude empty chats or with zero messages with content" + help="By default, the exporter will not render chats with no valid message. Setting this flag will cause the exporter to render those." ) parser.add_argument( "--per-chat", @@ -485,9 +485,6 @@ def main(): if not args.no_html: if contact_store.should_enrich_from_vcards(): contact_store.enrich_from_vcards(data) - - if (args.filter_empty): - data = {k: v for k, v in data.items() if not is_chat_empty(v)} create_html( data, @@ -496,7 +493,8 @@ def main(): args.embedded, args.offline, args.size, - args.no_avatar + args.no_avatar, + not args.filter_empty ) else: print( @@ -531,7 +529,9 @@ def main(): args.template, args.embedded, args.offline, - args.size + args.size, + args.no_avatar, + not args.filter_empty ) for file in glob.glob(r'*.*'): shutil.copy(file, args.output) @@ -543,12 +543,14 @@ def main(): args.template, args.embedded, args.offline, - args.size + args.size, + args.no_avatar, + not args.filter_empty ) if args.json and not args.import_json: - if (args.filter_empty): - data = {k: v for k, v in data.items() if not is_chat_empty(v)} + if not args.filter_empty: + data = {k: v for k, v in data.items() if not chat_is_empty(v)} if contact_store.should_enrich_from_vcards(): contact_store.enrich_from_vcards(data) diff --git a/Whatsapp_Chat_Exporter/android_handler.py b/Whatsapp_Chat_Exporter/android_handler.py index 094d3e1..32c2121 100644 --- a/Whatsapp_Chat_Exporter/android_handler.py +++ b/Whatsapp_Chat_Exporter/android_handler.py @@ -11,7 +11,7 @@ from markupsafe import escape as htmle from hashlib import sha256 from base64 import b64decode, b64encode from Whatsapp_Chat_Exporter.data_model import ChatStore, Message -from Whatsapp_Chat_Exporter.utility import MAX_SIZE, ROW_SIZE, DbType, determine_metadata, JidType +from Whatsapp_Chat_Exporter.utility import MAX_SIZE, ROW_SIZE, DbType, determine_metadata, JidType, chat_is_empty from Whatsapp_Chat_Exporter.utility import rendering, Crypt, Device, get_file_name, setup_template from Whatsapp_Chat_Exporter.utility import brute_force_offset, CRYPT14_OFFSETS, get_status_location from Whatsapp_Chat_Exporter.utility import get_chat_condition, slugify @@ -749,7 +749,8 @@ def create_html( embedded=False, offline_static=False, maximum_size=None, - no_avatar=False + no_avatar=False, + filter_empty=True ): template = setup_template(template, no_avatar) @@ -763,7 +764,7 @@ def create_html( for current, contact in enumerate(data): chat = data[contact] - if len(chat.messages) == 0: + if filter_empty and chat_is_empty(chat): continue safe_file_name, name = get_file_name(contact, chat) diff --git a/Whatsapp_Chat_Exporter/utility.py b/Whatsapp_Chat_Exporter/utility.py index 09e6671..1eb8b02 100644 --- a/Whatsapp_Chat_Exporter/utility.py +++ b/Whatsapp_Chat_Exporter/utility.py @@ -170,7 +170,7 @@ def get_chat_condition(filter, include, column): def _is_message_empty(message): return (message.data is None or message.data == "") and not message.media -def is_chat_empty(chat: ChatStore): +def chat_is_empty(chat: ChatStore): return len(chat.messages) == 0 or all(_is_message_empty(message) for message in chat.messages.values()) From 0cf113561adea373d436cc18a34a9de895aca241 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 14:16:57 +0800 Subject: [PATCH 14/19] Rename variable and files --- Whatsapp_Chat_Exporter/__main__.py | 6 +++--- .../{contacts_from_vcards.py => vcards_contacts.py} | 2 +- ...contacts_from_vcards_test.py => vcards_contacts_test.py} | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename Whatsapp_Chat_Exporter/{contacts_from_vcards.py => vcards_contacts.py} (98%) rename Whatsapp_Chat_Exporter/{contacts_from_vcards_test.py => vcards_contacts_test.py} (86%) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 37f9b54..93a578c 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -15,7 +15,7 @@ else: vcards_deps_installed = True from Whatsapp_Chat_Exporter import exported_handler, android_handler from Whatsapp_Chat_Exporter import ios_handler, ios_media_handler -from Whatsapp_Chat_Exporter.contacts_from_vcards import ContactsFromVCards +from Whatsapp_Chat_Exporter.vcards_contacts import ContactsFromVCards from Whatsapp_Chat_Exporter.data_model import ChatStore from Whatsapp_Chat_Exporter.utility import APPLE_TIME, Crypt, DbType, chat_is_empty from Whatsapp_Chat_Exporter.utility import check_update, import_from_json @@ -483,7 +483,7 @@ def main(): if args.android: android_handler.calls(db, data, args.timezone_offset, filter_chat) if not args.no_html: - if contact_store.should_enrich_from_vcards(): + if contact_store.is_empty(): contact_store.enrich_from_vcards(data) create_html( @@ -552,7 +552,7 @@ def main(): if not args.filter_empty: data = {k: v for k, v in data.items() if not chat_is_empty(v)} - if contact_store.should_enrich_from_vcards(): + if contact_store.is_empty(): contact_store.enrich_from_vcards(data) if isinstance(data[next(iter(data))], ChatStore): diff --git a/Whatsapp_Chat_Exporter/contacts_from_vcards.py b/Whatsapp_Chat_Exporter/vcards_contacts.py similarity index 98% rename from Whatsapp_Chat_Exporter/contacts_from_vcards.py rename to Whatsapp_Chat_Exporter/vcards_contacts.py index 58d5eaa..95fc155 100644 --- a/Whatsapp_Chat_Exporter/contacts_from_vcards.py +++ b/Whatsapp_Chat_Exporter/vcards_contacts.py @@ -12,7 +12,7 @@ class ContactsFromVCards: def __init__(self) -> None: self.contact_mapping = [] - def should_enrich_from_vcards(self): + def is_empty(self): return len(self.contact_mapping) > 0 def load_vcf_file(self, vcf_file_path: str, default_country_code: str): diff --git a/Whatsapp_Chat_Exporter/contacts_from_vcards_test.py b/Whatsapp_Chat_Exporter/vcards_contacts_test.py similarity index 86% rename from Whatsapp_Chat_Exporter/contacts_from_vcards_test.py rename to Whatsapp_Chat_Exporter/vcards_contacts_test.py index bd4929b..194b637 100644 --- a/Whatsapp_Chat_Exporter/contacts_from_vcards_test.py +++ b/Whatsapp_Chat_Exporter/vcards_contacts_test.py @@ -1,6 +1,6 @@ # from contacts_names_from_vcards import readVCardsFile -from Whatsapp_Chat_Exporter.contacts_from_vcards import normalize_number, read_vcards_file +from Whatsapp_Chat_Exporter.vcards_contacts import normalize_number, read_vcards_file def test_readVCardsFile(): From b2f679d9753a9bc36931be18fb487611c1dfb001 Mon Sep 17 00:00:00 2001 From: Knugi <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 10:25:44 +0000 Subject: [PATCH 15/19] Fix --avoid-encoding-json option Co-authored-by: Bnaya Peretz --- Whatsapp_Chat_Exporter/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 93a578c..922a210 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -93,7 +93,7 @@ def main(): const="result.json", help="Save the result to a single JSON file (default if present: result.json)") parser.add_argument( - '--avoid_encoding_json', + '--avoid-encoding-json', dest='avoid_encoding_json', default=False, action='store_true', From ea01a727cf2dc9c1bd6c3fd9c7be8f408bdb5946 Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 18:33:33 +0800 Subject: [PATCH 16/19] Fixed the wrong boolean value --- Whatsapp_Chat_Exporter/vcards_contacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Whatsapp_Chat_Exporter/vcards_contacts.py b/Whatsapp_Chat_Exporter/vcards_contacts.py index 95fc155..872f619 100644 --- a/Whatsapp_Chat_Exporter/vcards_contacts.py +++ b/Whatsapp_Chat_Exporter/vcards_contacts.py @@ -13,7 +13,7 @@ class ContactsFromVCards: self.contact_mapping = [] def is_empty(self): - return len(self.contact_mapping) > 0 + return self.contact_mapping == [] def load_vcf_file(self, vcf_file_path: str, default_country_code: str): self.contact_mapping = read_vcards_file(vcf_file_path, default_country_code) From d95b075ac0172047bc97263e53e723e8cb41e93c Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 18:57:16 +0800 Subject: [PATCH 17/19] Invert the --dont-filter-empty option to provide clarity --- Whatsapp_Chat_Exporter/__main__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 922a210..4447a46 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -262,9 +262,9 @@ def main(): ) parser.add_argument( "--dont-filter-empty", - dest="filter_empty", - default=False, - action='store_true', + dest="disable_filter_empty", + default=True, + action='store_false', help="By default, the exporter will not render chats with no valid message. Setting this flag will cause the exporter to render those." ) parser.add_argument( @@ -494,7 +494,7 @@ def main(): args.offline, args.size, args.no_avatar, - not args.filter_empty + args.disable_filter_empty ) else: print( @@ -531,7 +531,7 @@ def main(): args.offline, args.size, args.no_avatar, - not args.filter_empty + args.disable_filter_empty ) for file in glob.glob(r'*.*'): shutil.copy(file, args.output) @@ -545,11 +545,11 @@ def main(): args.offline, args.size, args.no_avatar, - not args.filter_empty + args.disable_filter_empty ) if args.json and not args.import_json: - if not args.filter_empty: + if args.disable_filter_empty: data = {k: v for k, v in data.items() if not chat_is_empty(v)} if contact_store.is_empty(): From 8069882dc51a8e9f72c72d14ef139cb393d91a4e Mon Sep 17 00:00:00 2001 From: KnugiHK <24708955+KnugiHK@users.noreply.github.com> Date: Sat, 13 Jul 2024 21:54:13 +0800 Subject: [PATCH 18/19] Update variable name for filter_empty --- Whatsapp_Chat_Exporter/__main__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 4447a46..03d40cb 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -262,7 +262,7 @@ def main(): ) parser.add_argument( "--dont-filter-empty", - dest="disable_filter_empty", + dest="filter_empty", default=True, action='store_false', help="By default, the exporter will not render chats with no valid message. Setting this flag will cause the exporter to render those." @@ -494,7 +494,7 @@ def main(): args.offline, args.size, args.no_avatar, - args.disable_filter_empty + args.filter_empty ) else: print( @@ -531,7 +531,7 @@ def main(): args.offline, args.size, args.no_avatar, - args.disable_filter_empty + args.filter_empty ) for file in glob.glob(r'*.*'): shutil.copy(file, args.output) @@ -545,11 +545,11 @@ def main(): args.offline, args.size, args.no_avatar, - args.disable_filter_empty + args.filter_empty ) if args.json and not args.import_json: - if args.disable_filter_empty: + if args.filter_empty: data = {k: v for k, v in data.items() if not chat_is_empty(v)} if contact_store.is_empty(): From 13904ea4d8aa3fb9493d9dd7535edabbe582fab0 Mon Sep 17 00:00:00 2001 From: Bnaya Peretz Date: Mon, 15 Jul 2024 08:48:15 +0300 Subject: [PATCH 19/19] Fix cond --- Whatsapp_Chat_Exporter/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Whatsapp_Chat_Exporter/__main__.py b/Whatsapp_Chat_Exporter/__main__.py index 03d40cb..828bbf3 100644 --- a/Whatsapp_Chat_Exporter/__main__.py +++ b/Whatsapp_Chat_Exporter/__main__.py @@ -483,7 +483,7 @@ def main(): if args.android: android_handler.calls(db, data, args.timezone_offset, filter_chat) if not args.no_html: - if contact_store.is_empty(): + if not contact_store.is_empty(): contact_store.enrich_from_vcards(data) create_html( @@ -552,7 +552,7 @@ def main(): if args.filter_empty: data = {k: v for k, v in data.items() if not chat_is_empty(v)} - if contact_store.is_empty(): + if not contact_store.is_empty(): contact_store.enrich_from_vcards(data) if isinstance(data[next(iter(data))], ChatStore):