mirror of
https://github.com/KnugiHK/WhatsApp-Chat-Exporter.git
synced 2026-04-22 05:54:40 +00:00
Merge branch 'dev' into feature/export-reactions
This commit is contained in:
Knugi
GitHub
@@ -4,13 +4,14 @@ import logging
|
||||
import sqlite3
|
||||
import os
|
||||
import shutil
|
||||
from tqdm import tqdm
|
||||
from pathlib import Path
|
||||
from mimetypes import MimeTypes
|
||||
from markupsafe import escape as htmle
|
||||
from base64 import b64decode, b64encode
|
||||
from datetime import datetime
|
||||
from Whatsapp_Chat_Exporter.data_model import ChatStore, Message
|
||||
from Whatsapp_Chat_Exporter.utility import CLEAR_LINE, CURRENT_TZ_OFFSET, MAX_SIZE, ROW_SIZE, JidType, Device
|
||||
from Whatsapp_Chat_Exporter.data_model import ChatStore, Message, Timing
|
||||
from Whatsapp_Chat_Exporter.utility import CLEAR_LINE, MAX_SIZE, ROW_SIZE, JidType, Device
|
||||
from Whatsapp_Chat_Exporter.utility import rendering, get_file_name, setup_template, get_cond_for_empty
|
||||
from Whatsapp_Chat_Exporter.utility import get_status_location, convert_time_unit, determine_metadata
|
||||
from Whatsapp_Chat_Exporter.utility import get_chat_condition, safe_name, bytes_to_readable
|
||||
@@ -47,12 +48,15 @@ def contacts(db, data, enrich_from_vcards):
|
||||
logger.info(f"Processed {total_row_number} contacts\n")
|
||||
|
||||
c.execute("SELECT jid, COALESCE(display_name, wa_name) as display_name, status FROM wa_contacts;")
|
||||
row = c.fetchone()
|
||||
while row is not None:
|
||||
current_chat = data.add_chat(row["jid"], ChatStore(Device.ANDROID, row["display_name"]))
|
||||
if row["status"] is not None:
|
||||
current_chat.status = row["status"]
|
||||
row = c.fetchone()
|
||||
|
||||
with tqdm(total=total_row_number, desc="Processing contacts", unit="contact", leave=False) as pbar:
|
||||
while (row := _fetch_row_safely(c)) is not None:
|
||||
current_chat = data.add_chat(row["jid"], ChatStore(Device.ANDROID, row["display_name"]))
|
||||
if row["status"] is not None:
|
||||
current_chat.status = row["status"]
|
||||
pbar.update(1)
|
||||
total_time = pbar.format_dict['elapsed']
|
||||
logger.info(f"Processed {total_row_number} contacts in {convert_time_unit(total_time)}{CLEAR_LINE}")
|
||||
|
||||
return True
|
||||
|
||||
@@ -72,7 +76,6 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat,
|
||||
"""
|
||||
c = db.cursor()
|
||||
total_row_number = _get_message_count(c, filter_empty, filter_date, filter_chat)
|
||||
logger.info(f"Processing messages...(0/{total_row_number})\r")
|
||||
|
||||
try:
|
||||
content_cursor = _get_messages_cursor_legacy(c, filter_empty, filter_date, filter_chat)
|
||||
@@ -84,23 +87,12 @@ def messages(db, data, media_folder, timezone_offset, filter_date, filter_chat,
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
i = 0
|
||||
# Fetch the first row safely
|
||||
content = _fetch_row_safely(content_cursor)
|
||||
|
||||
while content is not None:
|
||||
_process_single_message(data, content, table_message, timezone_offset)
|
||||
|
||||
i += 1
|
||||
if i % 1000 == 0:
|
||||
logger.info(f"Processing messages...({i}/{total_row_number})\r")
|
||||
|
||||
# Fetch the next row safely
|
||||
content = _fetch_row_safely(content_cursor)
|
||||
|
||||
_get_reactions(db, data)
|
||||
logger.info(f"Processed {total_row_number} messages{CLEAR_LINE}")
|
||||
|
||||
with tqdm(total=total_row_number, desc="Processing messages", unit="msg", leave=False) as pbar:
|
||||
while (content := _fetch_row_safely(content_cursor)) is not None:
|
||||
_process_single_message(data, content, table_message, timezone_offset)
|
||||
pbar.update(1)
|
||||
total_time = pbar.format_dict['elapsed']
|
||||
logger.info(f"Processed {total_row_number} messages in {convert_time_unit(total_time)}{CLEAR_LINE}")
|
||||
|
||||
# Helper functions for message processing
|
||||
|
||||
@@ -126,14 +118,16 @@ def _get_message_count(cursor, filter_empty, filter_date, filter_chat):
|
||||
{include_filter}
|
||||
{exclude_filter}""")
|
||||
except sqlite3.OperationalError:
|
||||
empty_filter = get_cond_for_empty(filter_empty, "jid.raw_string", "broadcast")
|
||||
empty_filter = get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")
|
||||
date_filter = f'AND timestamp {filter_date}' if filter_date is not None else ''
|
||||
include_filter = get_chat_condition(
|
||||
filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")
|
||||
filter_chat[0], True, ["key_remote_jid", "group_sender_jid"], "jid", "android")
|
||||
exclude_filter = get_chat_condition(
|
||||
filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")
|
||||
filter_chat[1], False, ["key_remote_jid", "group_sender_jid"], "jid", "android")
|
||||
|
||||
cursor.execute(f"""SELECT count()
|
||||
cursor.execute(f"""SELECT count(),
|
||||
COALESCE(lid_global.raw_string, jid.raw_string) as key_remote_jid,
|
||||
COALESCE(lid_group.raw_string, jid_group.raw_string) as group_sender_jid
|
||||
FROM message
|
||||
LEFT JOIN chat
|
||||
ON chat._id = message.chat_row_id
|
||||
@@ -141,6 +135,14 @@ def _get_message_count(cursor, filter_empty, filter_date, filter_chat):
|
||||
ON jid._id = chat.jid_row_id
|
||||
LEFT JOIN jid jid_group
|
||||
ON jid_group._id = message.sender_jid_row_id
|
||||
LEFT JOIN jid_map as jid_map_global
|
||||
ON chat.jid_row_id = jid_map_global.lid_row_id
|
||||
LEFT JOIN jid lid_global
|
||||
ON jid_map_global.jid_row_id = lid_global._id
|
||||
LEFT JOIN jid_map as jid_map_group
|
||||
ON message.sender_jid_row_id = jid_map_group.lid_row_id
|
||||
LEFT JOIN jid lid_group
|
||||
ON jid_map_group.jid_row_id = lid_group._id
|
||||
WHERE 1=1
|
||||
{empty_filter}
|
||||
{date_filter}
|
||||
@@ -219,11 +221,11 @@ def _get_messages_cursor_new(cursor, filter_empty, filter_date, filter_chat):
|
||||
empty_filter = get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")
|
||||
date_filter = f'AND message.timestamp {filter_date}' if filter_date is not None else ''
|
||||
include_filter = get_chat_condition(
|
||||
filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")
|
||||
filter_chat[0], True, ["key_remote_jid", "lid_group.raw_string"], "jid_global", "android")
|
||||
exclude_filter = get_chat_condition(
|
||||
filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid_global", "android")
|
||||
filter_chat[1], False, ["key_remote_jid", "lid_group.raw_string"], "jid_global", "android")
|
||||
|
||||
cursor.execute(f"""SELECT jid_global.raw_string as key_remote_jid,
|
||||
cursor.execute(f"""SELECT COALESCE(lid_global.raw_string, jid_global.raw_string) as key_remote_jid,
|
||||
message._id,
|
||||
message.from_me as key_from_me,
|
||||
message.timestamp,
|
||||
@@ -238,7 +240,7 @@ def _get_messages_cursor_new(cursor, filter_empty, filter_date, filter_chat):
|
||||
message.key_id,
|
||||
message_quoted.text_data as quoted_data,
|
||||
message.message_type as media_wa_type,
|
||||
jid_group.raw_string as group_sender_jid,
|
||||
COALESCE(lid_group.raw_string, jid_group.raw_string) as group_sender_jid,
|
||||
chat.subject as chat_subject,
|
||||
missed_call_logs.video_call,
|
||||
message.sender_jid_row_id,
|
||||
@@ -248,7 +250,8 @@ def _get_messages_cursor_new(cursor, filter_empty, filter_date, filter_chat):
|
||||
jid_new.raw_string as new_jid,
|
||||
jid_global.type as jid_type,
|
||||
COALESCE(receipt_user.receipt_timestamp, message.received_timestamp) as received_timestamp,
|
||||
COALESCE(receipt_user.read_timestamp, receipt_user.played_timestamp) as read_timestamp
|
||||
COALESCE(receipt_user.read_timestamp, receipt_user.played_timestamp) as read_timestamp,
|
||||
message_media.raw_transcription_text as transcription_text
|
||||
FROM message
|
||||
LEFT JOIN message_quoted
|
||||
ON message_quoted.message_row_id = message._id
|
||||
@@ -280,6 +283,14 @@ def _get_messages_cursor_new(cursor, filter_empty, filter_date, filter_chat):
|
||||
ON jid_new._id = message_system_number_change.new_jid_row_id
|
||||
LEFT JOIN receipt_user
|
||||
ON receipt_user.message_row_id = message._id
|
||||
LEFT JOIN jid_map as jid_map_global
|
||||
ON chat.jid_row_id = jid_map_global.lid_row_id
|
||||
LEFT JOIN jid lid_global
|
||||
ON jid_map_global.jid_row_id = lid_global._id
|
||||
LEFT JOIN jid_map as jid_map_group
|
||||
ON message.sender_jid_row_id = jid_map_group.lid_row_id
|
||||
LEFT JOIN jid lid_group
|
||||
ON jid_map_group.jid_row_id = lid_group._id
|
||||
WHERE key_remote_jid <> '-1'
|
||||
{empty_filter}
|
||||
{date_filter}
|
||||
@@ -321,7 +332,7 @@ def _process_single_message(data, content, table_message, timezone_offset):
|
||||
timestamp=content["timestamp"],
|
||||
time=content["timestamp"],
|
||||
key_id=content["key_id"],
|
||||
timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET,
|
||||
timezone_offset=timezone_offset,
|
||||
message_type=content["media_wa_type"],
|
||||
received_timestamp=content["received_timestamp"],
|
||||
read_timestamp=content["read_timestamp"]
|
||||
@@ -353,9 +364,12 @@ def _process_single_message(data, content, table_message, timezone_offset):
|
||||
if not table_message and content["media_caption"] is not None:
|
||||
# Old schema
|
||||
message.caption = content["media_caption"]
|
||||
elif table_message and content["media_wa_type"] == 1 and content["data"] is not None:
|
||||
elif table_message:
|
||||
# New schema
|
||||
message.caption = content["data"]
|
||||
if content["media_wa_type"] == 1 and content["data"] is not None:
|
||||
message.caption = content["data"]
|
||||
elif content["media_wa_type"] == 2 and content["transcription_text"] is not None:
|
||||
message.caption = f'"{content["transcription_text"]}"'
|
||||
else:
|
||||
message.caption = None
|
||||
|
||||
@@ -547,7 +561,7 @@ def _get_reactions(db, data):
|
||||
logger.info(f"Processed reactions{CLEAR_LINE}")
|
||||
|
||||
|
||||
def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separate_media=True):
|
||||
def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separate_media=True, fix_dot_files=False):
|
||||
"""
|
||||
Process WhatsApp media files from the database.
|
||||
|
||||
@@ -562,8 +576,6 @@ def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separa
|
||||
"""
|
||||
c = db.cursor()
|
||||
total_row_number = _get_media_count(c, filter_empty, filter_date, filter_chat)
|
||||
logger.info(f"Processing media...(0/{total_row_number})\r")
|
||||
|
||||
try:
|
||||
content_cursor = _get_media_cursor_legacy(c, filter_empty, filter_date, filter_chat)
|
||||
except sqlite3.OperationalError:
|
||||
@@ -575,18 +587,12 @@ def media(db, data, media_folder, filter_date, filter_chat, filter_empty, separa
|
||||
# Ensure thumbnails directory exists
|
||||
Path(f"{media_folder}/thumbnails").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
i = 0
|
||||
while content is not None:
|
||||
_process_single_media(data, content, media_folder, mime, separate_media)
|
||||
|
||||
i += 1
|
||||
if i % 100 == 0:
|
||||
logger.info(f"Processing media...({i}/{total_row_number})\r")
|
||||
|
||||
content = content_cursor.fetchone()
|
||||
|
||||
logger.info(f"Processed {total_row_number} media{CLEAR_LINE}")
|
||||
|
||||
with tqdm(total=total_row_number, desc="Processing media", unit="media", leave=False) as pbar:
|
||||
while (content := _fetch_row_safely(content_cursor)) is not None:
|
||||
_process_single_media(data, content, media_folder, mime, separate_media, fix_dot_files)
|
||||
pbar.update(1)
|
||||
total_time = pbar.format_dict['elapsed']
|
||||
logger.info(f"Processed {total_row_number} media in {convert_time_unit(total_time)}{CLEAR_LINE}")
|
||||
|
||||
# Helper functions for media processing
|
||||
|
||||
@@ -617,11 +623,13 @@ def _get_media_count(cursor, filter_empty, filter_date, filter_chat):
|
||||
empty_filter = get_cond_for_empty(filter_empty, "jid.raw_string", "broadcast")
|
||||
date_filter = f'AND message.timestamp {filter_date}' if filter_date is not None else ''
|
||||
include_filter = get_chat_condition(
|
||||
filter_chat[0], True, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")
|
||||
filter_chat[0], True, ["key_remote_jid", "group_sender_jid"], "jid", "android")
|
||||
exclude_filter = get_chat_condition(
|
||||
filter_chat[1], False, ["jid.raw_string", "jid_group.raw_string"], "jid", "android")
|
||||
filter_chat[1], False, ["key_remote_jid", "group_sender_jid"], "jid", "android")
|
||||
|
||||
cursor.execute(f"""SELECT count()
|
||||
cursor.execute(f"""SELECT count(),
|
||||
COALESCE(lid_global.raw_string, jid.raw_string) as key_remote_jid,
|
||||
COALESCE(lid_group.raw_string, jid_group.raw_string) as group_sender_jid
|
||||
FROM message_media
|
||||
INNER JOIN message
|
||||
ON message_media.message_row_id = message._id
|
||||
@@ -631,6 +639,14 @@ def _get_media_count(cursor, filter_empty, filter_date, filter_chat):
|
||||
ON jid._id = chat.jid_row_id
|
||||
LEFT JOIN jid jid_group
|
||||
ON jid_group._id = message.sender_jid_row_id
|
||||
LEFT JOIN jid_map as jid_map_global
|
||||
ON chat.jid_row_id = jid_map_global.lid_row_id
|
||||
LEFT JOIN jid lid_global
|
||||
ON jid_map_global.jid_row_id = lid_global._id
|
||||
LEFT JOIN jid_map as jid_map_group
|
||||
ON message.sender_jid_row_id = jid_map_group.lid_row_id
|
||||
LEFT JOIN jid lid_group
|
||||
ON jid_map_group.jid_row_id = lid_group._id
|
||||
WHERE 1=1
|
||||
{empty_filter}
|
||||
{date_filter}
|
||||
@@ -679,18 +695,19 @@ def _get_media_cursor_new(cursor, filter_empty, filter_date, filter_chat):
|
||||
empty_filter = get_cond_for_empty(filter_empty, "key_remote_jid", "broadcast")
|
||||
date_filter = f'AND message.timestamp {filter_date}' if filter_date is not None else ''
|
||||
include_filter = get_chat_condition(
|
||||
filter_chat[0], True, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")
|
||||
filter_chat[0], True, ["key_remote_jid", "group_sender_jid"], "jid", "android")
|
||||
exclude_filter = get_chat_condition(
|
||||
filter_chat[1], False, ["key_remote_jid", "jid_group.raw_string"], "jid", "android")
|
||||
filter_chat[1], False, ["key_remote_jid", "group_sender_jid"], "jid", "android")
|
||||
|
||||
cursor.execute(f"""SELECT jid.raw_string as key_remote_jid,
|
||||
cursor.execute(f"""SELECT COALESCE(lid_global.raw_string, jid.raw_string) as key_remote_jid,
|
||||
message_row_id,
|
||||
file_path,
|
||||
message_url,
|
||||
mime_type,
|
||||
media_key,
|
||||
file_hash,
|
||||
thumbnail
|
||||
thumbnail,
|
||||
COALESCE(lid_group.raw_string, jid_group.raw_string) as group_sender_jid
|
||||
FROM message_media
|
||||
INNER JOIN message
|
||||
ON message_media.message_row_id = message._id
|
||||
@@ -702,6 +719,14 @@ def _get_media_cursor_new(cursor, filter_empty, filter_date, filter_chat):
|
||||
ON message_media.file_hash = media_hash_thumbnail.media_hash
|
||||
LEFT JOIN jid jid_group
|
||||
ON jid_group._id = message.sender_jid_row_id
|
||||
LEFT JOIN jid_map as jid_map_global
|
||||
ON chat.jid_row_id = jid_map_global.lid_row_id
|
||||
LEFT JOIN jid lid_global
|
||||
ON jid_map_global.jid_row_id = lid_global._id
|
||||
LEFT JOIN jid_map as jid_map_group
|
||||
ON message.sender_jid_row_id = jid_map_group.lid_row_id
|
||||
LEFT JOIN jid lid_group
|
||||
ON jid_map_group.jid_row_id = lid_group._id
|
||||
WHERE jid.type <> 7
|
||||
{empty_filter}
|
||||
{date_filter}
|
||||
@@ -711,7 +736,7 @@ def _get_media_cursor_new(cursor, filter_empty, filter_date, filter_chat):
|
||||
return cursor
|
||||
|
||||
|
||||
def _process_single_media(data, content, media_folder, mime, separate_media):
|
||||
def _process_single_media(data, content, media_folder, mime, separate_media, fix_dot_files=False):
|
||||
"""Process a single media file."""
|
||||
file_path = f"{media_folder}/{content['file_path']}"
|
||||
current_chat = data.get_chat(content["key_remote_jid"])
|
||||
@@ -719,8 +744,6 @@ def _process_single_media(data, content, media_folder, mime, separate_media):
|
||||
message.media = True
|
||||
|
||||
if os.path.isfile(file_path):
|
||||
message.data = file_path
|
||||
|
||||
# Set mime type
|
||||
if content["mime_type"] is None:
|
||||
guess = mime.guess_type(file_path)[0]
|
||||
@@ -730,6 +753,16 @@ def _process_single_media(data, content, media_folder, mime, separate_media):
|
||||
message.mime = "application/octet-stream"
|
||||
else:
|
||||
message.mime = content["mime_type"]
|
||||
|
||||
if fix_dot_files and file_path.endswith("."):
|
||||
extension = mime.guess_extension(message.mime)
|
||||
if message.mime == "application/octet-stream" or not extension:
|
||||
new_file_path = file_path[:-1]
|
||||
else:
|
||||
extension = mime.guess_extension(message.mime)
|
||||
new_file_path = file_path[:-1] + extension
|
||||
os.rename(file_path, new_file_path)
|
||||
file_path = new_file_path
|
||||
|
||||
# Copy media to separate folder if needed
|
||||
if separate_media:
|
||||
@@ -741,6 +774,8 @@ def _process_single_media(data, content, media_folder, mime, separate_media):
|
||||
new_path = os.path.join(new_folder, current_filename)
|
||||
shutil.copy2(file_path, new_path)
|
||||
message.data = new_path
|
||||
else:
|
||||
message.data = file_path
|
||||
else:
|
||||
message.data = "The media is missing"
|
||||
message.mime = "media"
|
||||
@@ -764,45 +799,56 @@ def vcard(db, data, media_folder, filter_date, filter_chat, filter_empty):
|
||||
rows = _execute_vcard_query_legacy(c, filter_date, filter_chat, filter_empty)
|
||||
|
||||
total_row_number = len(rows)
|
||||
logger.info(f"Processing vCards...(0/{total_row_number})\r")
|
||||
|
||||
# Create vCards directory if it doesn't exist
|
||||
path = os.path.join(media_folder, "vCards")
|
||||
Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for index, row in enumerate(rows):
|
||||
_process_vcard_row(row, path, data)
|
||||
logger.info(f"Processing vCards...({index + 1}/{total_row_number})\r")
|
||||
logger.info(f"Processed {total_row_number} vCards{CLEAR_LINE}")
|
||||
|
||||
with tqdm(total=total_row_number, desc="Processing vCards", unit="vcard", leave=False) as pbar:
|
||||
for row in rows:
|
||||
_process_vcard_row(row, path, data)
|
||||
pbar.update(1)
|
||||
total_time = pbar.format_dict['elapsed']
|
||||
logger.info(f"Processed {total_row_number} vCards in {convert_time_unit(total_time)}{CLEAR_LINE}")
|
||||
|
||||
def _execute_vcard_query_modern(c, filter_date, filter_chat, filter_empty):
|
||||
"""Execute vCard query for modern WhatsApp database schema."""
|
||||
|
||||
# Build the filter conditions
|
||||
chat_filter_include = get_chat_condition(
|
||||
filter_chat[0], True, ["messages.key_remote_jid", "remote_resource"], "jid", "android")
|
||||
chat_filter_exclude = get_chat_condition(
|
||||
filter_chat[1], False, ["messages.key_remote_jid", "remote_resource"], "jid", "android")
|
||||
date_filter = f'AND messages.timestamp {filter_date}' if filter_date is not None else ''
|
||||
empty_filter = get_cond_for_empty(filter_empty, "key_remote_jid", "messages.needs_push")
|
||||
include_filter = get_chat_condition(
|
||||
filter_chat[0], True, ["key_remote_jid", "group_sender_jid"], "jid", "android")
|
||||
exclude_filter = get_chat_condition(
|
||||
filter_chat[1], False, ["key_remote_jid", "group_sender_jid"], "jid", "android")
|
||||
|
||||
query = f"""SELECT message_row_id,
|
||||
messages.key_remote_jid,
|
||||
vcard,
|
||||
messages.media_name
|
||||
FROM messages_vcards
|
||||
INNER JOIN messages
|
||||
ON messages_vcards.message_row_id = messages._id
|
||||
INNER JOIN jid
|
||||
ON messages.key_remote_jid = jid.raw_string
|
||||
LEFT JOIN chat
|
||||
ON chat.jid_row_id = jid._id
|
||||
COALESCE(lid_global.raw_string, jid.raw_string) as key_remote_jid,
|
||||
vcard,
|
||||
messages.media_name,
|
||||
COALESCE(lid_group.raw_string, jid_group.raw_string) as group_sender_jid
|
||||
FROM messages_vcards
|
||||
INNER JOIN messages
|
||||
ON messages_vcards.message_row_id = messages._id
|
||||
INNER JOIN jid
|
||||
ON messages.key_remote_jid = jid.raw_string
|
||||
LEFT JOIN chat
|
||||
ON chat.jid_row_id = jid._id
|
||||
LEFT JOIN jid jid_group
|
||||
ON jid_group._id = message.sender_jid_row_id
|
||||
LEFT JOIN jid_map as jid_map_global
|
||||
ON chat.jid_row_id = jid_map_global.lid_row_id
|
||||
LEFT JOIN jid lid_global
|
||||
ON jid_map_global.jid_row_id = lid_global._id
|
||||
LEFT JOIN jid_map as jid_map_group
|
||||
ON message.sender_jid_row_id = jid_map_group.lid_row_id
|
||||
LEFT JOIN jid lid_group
|
||||
ON jid_map_group.jid_row_id = lid_group._id
|
||||
WHERE 1=1
|
||||
{empty_filter}
|
||||
{date_filter}
|
||||
{chat_filter_include}
|
||||
{chat_filter_exclude}
|
||||
{include_filter}
|
||||
{exclude_filter}
|
||||
ORDER BY messages.key_remote_jid ASC;"""
|
||||
c.execute(query)
|
||||
return c.fetchall()
|
||||
@@ -879,32 +925,37 @@ def calls(db, data, timezone_offset, filter_chat):
|
||||
chat = ChatStore(Device.ANDROID, "WhatsApp Calls")
|
||||
|
||||
# Process each call
|
||||
content = calls_data.fetchone()
|
||||
while content is not None:
|
||||
_process_call_record(content, chat, data, timezone_offset)
|
||||
content = calls_data.fetchone()
|
||||
with tqdm(total=total_row_number, desc="Processing calls", unit="call", leave=False) as pbar:
|
||||
while (content := _fetch_row_safely(calls_data)) is not None:
|
||||
_process_call_record(content, chat, data, timezone_offset)
|
||||
pbar.update(1)
|
||||
total_time = pbar.format_dict['elapsed']
|
||||
|
||||
# Add the calls chat to the data
|
||||
data.add_chat("000000000000000", chat)
|
||||
logger.info(f"Processed {total_row_number} calls{CLEAR_LINE}")
|
||||
|
||||
logger.info(f"Processed {total_row_number} calls in {convert_time_unit(total_time)}{CLEAR_LINE}")
|
||||
|
||||
def _get_calls_count(c, filter_chat):
|
||||
"""Get the count of call records that match the filter."""
|
||||
|
||||
# Build the filter conditions
|
||||
chat_filter_include = get_chat_condition(filter_chat[0], True, ["jid.raw_string"])
|
||||
chat_filter_exclude = get_chat_condition(filter_chat[1], False, ["jid.raw_string"])
|
||||
include_filter = get_chat_condition(filter_chat[0], True, ["key_remote_jid"])
|
||||
exclude_filter = get_chat_condition(filter_chat[1], False, ["key_remote_jid"])
|
||||
|
||||
query = f"""SELECT count()
|
||||
query = f"""SELECT count(),
|
||||
COALESCE(lid_global.raw_string, jid.raw_string) as key_remote_jid
|
||||
FROM call_log
|
||||
INNER JOIN jid
|
||||
ON call_log.jid_row_id = jid._id
|
||||
LEFT JOIN chat
|
||||
ON call_log.jid_row_id = chat.jid_row_id
|
||||
LEFT JOIN jid_map as jid_map_global
|
||||
ON chat.jid_row_id = jid_map_global.lid_row_id
|
||||
LEFT JOIN jid lid_global
|
||||
ON jid_map_global.jid_row_id = lid_global._id
|
||||
WHERE 1=1
|
||||
{chat_filter_include}
|
||||
{chat_filter_exclude}"""
|
||||
{include_filter}
|
||||
{exclude_filter}"""
|
||||
c.execute(query)
|
||||
return c.fetchone()[0]
|
||||
|
||||
@@ -913,11 +964,11 @@ def _fetch_calls_data(c, filter_chat):
|
||||
"""Fetch call data from the database."""
|
||||
|
||||
# Build the filter conditions
|
||||
chat_filter_include = get_chat_condition(filter_chat[0], True, ["jid.raw_string"])
|
||||
chat_filter_exclude = get_chat_condition(filter_chat[1], False, ["jid.raw_string"])
|
||||
include_filter = get_chat_condition(filter_chat[0], True, ["key_remote_jid"])
|
||||
exclude_filter = get_chat_condition(filter_chat[1], False, ["key_remote_jid"])
|
||||
|
||||
query = f"""SELECT call_log._id,
|
||||
jid.raw_string,
|
||||
COALESCE(lid_global.raw_string, jid.raw_string) as key_remote_jid,
|
||||
from_me,
|
||||
call_id,
|
||||
timestamp,
|
||||
@@ -931,9 +982,13 @@ def _fetch_calls_data(c, filter_chat):
|
||||
ON call_log.jid_row_id = jid._id
|
||||
LEFT JOIN chat
|
||||
ON call_log.jid_row_id = chat.jid_row_id
|
||||
LEFT JOIN jid_map as jid_map_global
|
||||
ON chat.jid_row_id = jid_map_global.lid_row_id
|
||||
LEFT JOIN jid lid_global
|
||||
ON jid_map_global.jid_row_id = lid_global._id
|
||||
WHERE 1=1
|
||||
{chat_filter_include}
|
||||
{chat_filter_exclude}"""
|
||||
{include_filter}
|
||||
{exclude_filter}"""
|
||||
c.execute(query)
|
||||
return c
|
||||
|
||||
@@ -945,13 +1000,13 @@ def _process_call_record(content, chat, data, timezone_offset):
|
||||
timestamp=content["timestamp"],
|
||||
time=content["timestamp"],
|
||||
key_id=content["call_id"],
|
||||
timezone_offset=timezone_offset if timezone_offset else CURRENT_TZ_OFFSET,
|
||||
timezone_offset=timezone_offset,
|
||||
received_timestamp=None, # TODO: Add timestamp
|
||||
read_timestamp=None # TODO: Add timestamp
|
||||
)
|
||||
|
||||
# Get caller/callee name
|
||||
_jid = content["raw_string"]
|
||||
_jid = content["key_remote_jid"]
|
||||
name = data.get_chat(_jid).name if _jid in data else content["chat_subject"] or None
|
||||
if _jid is not None and "@" in _jid:
|
||||
fallback = _jid.split('@')[0]
|
||||
@@ -996,6 +1051,7 @@ def _construct_call_description(content, call):
|
||||
return description
|
||||
|
||||
|
||||
# TODO: Marked for enhancement on multi-threaded processing
|
||||
def create_html(
|
||||
data,
|
||||
output_folder,
|
||||
@@ -1011,7 +1067,6 @@ def create_html(
|
||||
template = setup_template(template, no_avatar, experimental)
|
||||
|
||||
total_row_number = len(data)
|
||||
logger.info(f"Generating chats...(0/{total_row_number})\r")
|
||||
|
||||
# Create output directory if it doesn't exist
|
||||
if not os.path.isdir(output_folder):
|
||||
@@ -1019,43 +1074,42 @@ def create_html(
|
||||
|
||||
w3css = get_status_location(output_folder, offline_static)
|
||||
|
||||
for current, contact in enumerate(data):
|
||||
current_chat = data.get_chat(contact)
|
||||
if len(current_chat) == 0:
|
||||
# Skip empty chats
|
||||
continue
|
||||
with tqdm(total=total_row_number, desc="Generating HTML", unit="file", leave=False) as pbar:
|
||||
for contact in data:
|
||||
current_chat = data.get_chat(contact)
|
||||
if len(current_chat) == 0:
|
||||
# Skip empty chats
|
||||
continue
|
||||
|
||||
safe_file_name, name = get_file_name(contact, current_chat)
|
||||
safe_file_name, name = get_file_name(contact, current_chat)
|
||||
|
||||
if maximum_size is not None:
|
||||
_generate_paginated_chat(
|
||||
current_chat,
|
||||
safe_file_name,
|
||||
name,
|
||||
contact,
|
||||
output_folder,
|
||||
template,
|
||||
w3css,
|
||||
maximum_size,
|
||||
headline
|
||||
)
|
||||
else:
|
||||
_generate_single_chat(
|
||||
current_chat,
|
||||
safe_file_name,
|
||||
name,
|
||||
contact,
|
||||
output_folder,
|
||||
template,
|
||||
w3css,
|
||||
headline
|
||||
)
|
||||
|
||||
if current % 10 == 0:
|
||||
logger.info(f"Generating chats...({current}/{total_row_number})\r")
|
||||
|
||||
logger.info(f"Generated {total_row_number} chats{CLEAR_LINE}")
|
||||
if maximum_size is not None:
|
||||
_generate_paginated_chat(
|
||||
current_chat,
|
||||
safe_file_name,
|
||||
name,
|
||||
contact,
|
||||
output_folder,
|
||||
template,
|
||||
w3css,
|
||||
maximum_size,
|
||||
headline
|
||||
)
|
||||
else:
|
||||
_generate_single_chat(
|
||||
current_chat,
|
||||
safe_file_name,
|
||||
name,
|
||||
contact,
|
||||
output_folder,
|
||||
template,
|
||||
w3css,
|
||||
headline
|
||||
)
|
||||
|
||||
pbar.update(1)
|
||||
total_time = pbar.format_dict['elapsed']
|
||||
logger.info(f"Generated {total_row_number} chats in {convert_time_unit(total_time)}{CLEAR_LINE}")
|
||||
|
||||
def _generate_single_chat(current_chat, safe_file_name, name, contact, output_folder, template, w3css, headline):
|
||||
"""Generate a single HTML file for a chat."""
|
||||
|
||||
Reference in New Issue
Block a user