calvin.erfmann/TagStudio
1
0
Fork 0
mirror of https://github.com/TagStudioDev/TagStudio.git synced 2026-08-12 23:30:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: almost final migrations cleanup (#1456)

Browse Source 
* fix: backup library before making any changes

* refactor: inline make_tables + minor cleanup

* refactor: remove unnecessary assurance

Bumping the auto increment value has been done since the original sql PR, so it doesn't need to be done on migrations.
See e5e7b8afc6.

* refactor: don't blindly create all tables in the beginning

The only table that has been added since DB version 6 (the earliest supported version), is the versions table in DB version 101.
This commit removes the "create all tables" statement, and instead creates the versions table in the 101 migration.
See 12e074b71d.

* refactor: don't require setting library_dir to create a backup

* refactor: move migrations to different file

* refactor: package each migration in a class

* fix: some syntax errors had slipped through

* fix: allow set_version to fail, but don't commit in that case

* refactor: condense imports

* fix: add override decorators

* refactor: move set_version to DBMigrations

* refactor: remove unnecessary assignment

* refactor: use _ instead of __

* fix: add missing field templates tables
This commit is contained in:
Jann Stute
2026-08-12 00:14:33 +02:00
committed by GitHub
parent 5852c59f1c
commit b29e612e7f
5 changed files with 645 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tagstudio/core/library/alchemy/constants.py
+17
View File
@@ -4,6 +4,11 @@
from sqlalchemy import text
from tagstudio.core.library.alchemy.fields import (
DatetimeFieldTemplate,
TextFieldTemplate,
)
SQL_FILENAME: str = "ts_library.sqlite"
JSON_FILENAME: str = "ts_library.json"
@@ -32,3 +37,15 @@ WITH RECURSIVE ChildTags AS (
)
SELECT tag_id FROM ChildTags;
""")
DEFAULT_FIELD_TEMPLATES = (
TextFieldTemplate(name="Title"),
TextFieldTemplate(name="Author"),
TextFieldTemplate(name="Artist"),
TextFieldTemplate(name="URL"),
TextFieldTemplate(name="Description", is_multiline=True),
TextFieldTemplate(name="Notes", is_multiline=True),
TextFieldTemplate(name="Comments", is_multiline=True),
DatetimeFieldTemplate(name="Date"),
)
src/tagstudio/core/library/alchemy/db.py
+1 -39
View File
@@ -6,12 +6,9 @@ from pathlib import Path
from typing import override
import structlog
from sqlalchemy import Dialect, Engine, String, TypeDecorator, create_engine, text
from sqlalchemy.exc import OperationalError
from sqlalchemy import Dialect, String, TypeDecorator
from sqlalchemy.orm import DeclarativeBase
from tagstudio.core.constants import RESERVED_TAG_END
logger = structlog.getLogger(__name__)
@@ -34,38 +31,3 @@ class PathType(TypeDecorator):
class Base(DeclarativeBase):
type_annotation_map = {Path: PathType}
def make_engine(connection_string: str) -> Engine:
return create_engine(connection_string)
def make_tables(engine: Engine) -> None:
logger.info("[Library] Creating DB tables...")
with engine.connect() as conn:
# TODO: this should instead be migrations that create the exact tables that were added in
# the respective DB versions
Base.metadata.create_all(conn)
conn.commit()
# TODO: this needs to be a migration
# tag IDs < 1000 are reserved
# create tag and delete it to bump the autoincrement sequence
# TODO - find a better way
# is this the better way?
result = conn.execute(text("SELECT SEQ FROM sqlite_sequence WHERE name='tags'"))
autoincrement_val = result.scalar()
if not autoincrement_val or autoincrement_val <= RESERVED_TAG_END:
try:
conn.execute(
text(
"INSERT INTO tags "
"(id, name, color_namespace, color_slug, is_category, is_hidden) VALUES "
f"({RESERVED_TAG_END}, 'temp', NULL, NULL, false, false)"
)
)
conn.execute(text(f"DELETE FROM tags WHERE id = {RESERVED_TAG_END}"))
conn.commit()
except OperationalError as e:
logger.error("Could not initialize built-in tags", error=e)
conn.rollback()
src/tagstudio/core/library/alchemy/library.py
+55 -452
View File
@@ -2,11 +2,6 @@
# SPDX-License-Identifier: GPL-3.0-only
# NOTE: This file contains necessary use of deprecated first-party code until that
# code is removed in a future version (prefs).
# pyright: reportDeprecated=false
import re
import shutil
import sys
@@ -21,7 +16,6 @@ from typing import TYPE_CHECKING
import sqlalchemy
import structlog
import ujson
from humanfriendly import format_timespan # pyright: ignore[reportUnknownVariableType]
from sqlalchemy import (
URL,
@@ -44,7 +38,7 @@ from sqlalchemy import (
update,
)
from sqlalchemy.dialects import sqlite
from sqlalchemy.exc import IntegrityError
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.orm import (
InstanceState,
Session,
@@ -72,16 +66,13 @@ from tagstudio.core.library.alchemy.constants import (
DB_VERSION,
DB_VERSION_CURRENT_KEY,
DB_VERSION_INITIAL_KEY,
DEFAULT_FIELD_TEMPLATES,
JSON_FILENAME,
SQL_FILENAME,
TAG_CHILDREN_QUERY,
)
from tagstudio.core.library.alchemy.db import make_tables
from tagstudio.core.library.alchemy.enums import (
MAX_SQL_VARIABLES,
BrowsingState,
SortingModeEnum,
)
from tagstudio.core.library.alchemy.db import Base as ModelBase
from tagstudio.core.library.alchemy.enums import MAX_SQL_VARIABLES, BrowsingState, SortingModeEnum
from tagstudio.core.library.alchemy.fields import (
LEGACY_FIELD_MAP,
BaseField,
@@ -92,6 +83,7 @@ from tagstudio.core.library.alchemy.fields import (
TextFieldTemplate,
)
from tagstudio.core.library.alchemy.joins import TagEntry, TagParent
from tagstudio.core.library.alchemy.migrations import DBMigrations, MigrationError
from tagstudio.core.library.alchemy.models import (
Entry,
Namespace,
@@ -104,7 +96,6 @@ from tagstudio.core.library.alchemy.visitors import SQLBoolExpressionBuilder
from tagstudio.core.library.ignore import migrate_ext_list
from tagstudio.core.library.json.library import Library as JsonLibrary
from tagstudio.core.utils.types import unwrap
from tagstudio.qt.translations import Translations
if TYPE_CHECKING:
from sqlalchemy import Select
@@ -170,20 +161,6 @@ def get_default_tags() -> tuple[Tag, ...]:
return archive_tag, favorite_tag, meta_tag
def get_default_field_templates() -> tuple[BaseFieldTemplate, ...]:
"""Return the default field templates for a new TagStudio library."""
title = TextFieldTemplate(name="Title")
author = TextFieldTemplate(name="Author")
artist = TextFieldTemplate(name="Artist")
url = TextFieldTemplate(name="URL")
description = TextFieldTemplate(name="Description", is_multiline=True)
notes = TextFieldTemplate(name="Notes", is_multiline=True)
comments = TextFieldTemplate(name="Comments", is_multiline=True)
date = DatetimeFieldTemplate(name="Date")
return title, author, artist, url, description, notes, comments, date
# The difference in the number of default JSON tags vs default tags in the current version.
DEFAULT_TAG_DIFF: int = len(get_default_tags()) - len([TAG_ARCHIVED, TAG_FAVORITE])
@@ -431,21 +408,41 @@ class Library:
self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME
) -> LibraryStatus:
self.engine = self.__get_engine(library_dir, in_memory, sql_filename)
loaded_db_version: int = 0
logger.info(
"[Library] Opening SQLite Library",
library_dir=library_dir,
)
logger.info(f"[Library] Library DB version: {loaded_db_version}")
make_tables(self.engine)
logger.info("[Library] Creating DB tables...")
with self.engine.connect() as conn:
ModelBase.metadata.create_all(conn)
conn.commit()
# TODO - find a better way
# is this the better way?
# Could we perhaps update the row we are reading from here?
result = conn.execute(text("SELECT SEQ FROM sqlite_sequence WHERE name='tags'"))
autoincrement_val = result.scalar()
if not autoincrement_val or autoincrement_val <= RESERVED_TAG_END:
try:
conn.execute(
text(
"INSERT INTO tags "
"(id, name, color_namespace, color_slug, is_category, is_hidden) "
f"VALUES ({RESERVED_TAG_END}, 'temp', NULL, NULL, false, false)"
)
)
conn.execute(text(f"DELETE FROM tags WHERE id = {RESERVED_TAG_END}"))
conn.commit()
except OperationalError as e:
logger.error("Could not initialize built-in tags", error=e)
conn.rollback()
with Session(self.engine) as session:
# Add default tag color namespaces.
namespaces = default_color_groups.namespaces()
# TODO: are all of these commits necessary?
session.add_all(namespaces)
session.flush()
@@ -465,7 +462,7 @@ class Library:
session.flush()
# Add default field templates
for template in get_default_field_templates():
for template in DEFAULT_FIELD_TEMPLATES:
session.add(template)
session.flush()
@@ -506,414 +503,25 @@ class Library:
def open_sqlite_library(
self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME
) -> LibraryStatus:
logger.info("[Library] Opening SQLite Library", library_dir=library_dir)
self.engine = self.__get_engine(library_dir, in_memory, sql_filename)
loaded_db_version: int = 0
initial_db_version: int = DB_VERSION
logger.info(
"[Library] Opening SQLite Library",
library_dir=library_dir,
)
try:
migrations = DBMigrations(library_dir, self.engine)
# Don't check DB version when creating new library
loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY)
initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY)
# save backup if patches will be applied
if migrations.required:
Library.save_library_backup_to_disk(library_dir)
# ======================== Library Database Version Checking =======================
# DB_VERSION 6 is the first supported SQLite DB version.
# If the DB_VERSION is >= 100, that means it's a compound major + minor version.
# - Dividing by 100 and flooring gives the major (breaking changes) version.
# - If a DB has major version higher than the current program, don't load it.
# - If only the minor version is higher, it's still allowed to load.
if loaded_db_version < 6 or (
loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100
):
mismatch_text = Translations["status.library_version_mismatch"]
found_text = Translations["status.library_version_found"]
expected_text = Translations["status.library_version_expected"]
return LibraryStatus(
success=False,
message=(
f"{mismatch_text}\n"
f"{found_text} v{loaded_db_version}, "
f"{expected_text} v{DB_VERSION}"
),
)
logger.info(f"[Library] Library DB version: {loaded_db_version}")
# TODO: this is very sketchy; blindly creating all tables the newest DB version should have
# without considering what version the DB is currently on and then doing all of the
# migrations after that seems like it could cause problems in some scenarios.
# instead only have this on creation and create new tables as part of migrations
# Note: this actually produces an error and fails to initialise built-in tags when opening
# a library that doesn't yet have the is_hidden property on the tags table
make_tables(self.engine)
# save backup if patches will be applied
if loaded_db_version < DB_VERSION:
self.library_dir = library_dir
self.save_library_backup_to_disk()
self.library_dir = None
# migrate DB step by step from one version to the next
# (migration_method, db_version, initial_db_version)
migrations = [
(self.__apply_db7_migration, 7, None), # changes: value_type, tags
(self.__apply_db8_migration, 8, None), # changes: tag_colors
(self.__apply_db9_migration, 9, None), # changes: entries
(self.__apply_db100_migration, 100, None), # changes: tag_parents
(self.__apply_db101_migration, 101, None), # changes: versions
(self.__apply_db102_migration, 102, None), # changes: tag_parents
(self.__apply_db103_migration, 103, None), # changes: tags
(self.__apply_db104_migration, 104, None), # changes: deletes preferences
(self.__apply_db200_migration, 200, None), # changes: field tables
(self.__apply_db201_migration, 201, 200), # changes: field tables
(self.__apply_db202_migration, 202, None), # changes: tag_parents
(self.__apply_db300_migration, 300, None), # changes: deletes folders
]
for migration, v, iv in migrations:
if loaded_db_version < v and (iv is None or initial_db_version < iv):
logger.info(f"[Library][Migration][{v}] Starting DB Migration")
with Session(self.engine) as session:
# any error causes transaction to rollback
migration(session, library_dir)
loaded_db_version = v
self.set_version(session, DB_VERSION_CURRENT_KEY, v)
session.commit()
logger.info(f"[Library][Migration][{v}] Completed DB Migration")
assert loaded_db_version >= DB_VERSION, (
"Ran all migrations, but the DB is still not on the newest version"
)
logger.info(f"[Library] Library migrated to DB version {DB_VERSION}")
migrations.run()
except MigrationError as e:
return LibraryStatus(success=False, message=e.args[0])
# everything is fine, set the library path
self.library_dir = library_dir
return LibraryStatus(success=True, library_path=library_dir)
def __apply_db7_migration(self, session: Session, _library_dir: Path):
"""Migrate DB from DB_VERSION 6 to 7."""
logger.info("[Library][Migration][7] Applying patches to DB_VERSION: 6 library...")
# Repair tags that may have a disambiguation_id pointing towards a deleted tag.
# TODO: combine into single sql statement
all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all()
disam_stmt = (
update(Tag)
.where(Tag.disambiguation_id.not_in(all_tag_ids))
.values(disambiguation_id=None)
)
session.execute(disam_stmt)
session.flush()
def __apply_db8_migration(self, session: Session, library_dir: Path):
"""Migrate DB from DB_VERSION 7 to 8."""
# Add the missing color_border column to the TagColorGroups table.
session.execute(
text("ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL")
)
session.flush()
logger.info("[Library][Migration][8] Added color_border column to tag_colors table")
# collect new default tag colors
tag_colors: list[TagColorGroup] = [
color
for color in default_color_groups.shades()
if color.slug in ["burgundy", "dark-teal", "dark_lavender"]
]
# Add any new default colors introduced in DB_VERSION 8
for color in tag_colors:
session.add(color)
session.flush()
logger.info(
"[Library][Migration][8] Migrated tag colors to DB_VERSION 8+",
color_name=tag_colors,
)
# Update Neon colors to use the the color_border property
for color in default_color_groups.neon():
neon_stmt = (
update(TagColorGroup)
.where(
and_(
TagColorGroup.namespace == color.namespace,
TagColorGroup.slug == color.slug,
)
)
.values(
slug=color.slug,
namespace=color.namespace,
name=color.name,
primary=color.primary,
secondary=color.secondary,
color_border=color.color_border,
)
)
session.execute(neon_stmt)
session.flush()
def __apply_db9_migration(self, session: Session, library_dir: Path):
"""Migrate DB from DB_VERSION 8 to 9."""
# Apply database schema changes
add_filename_column = text(
"ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''"
)
session.execute(add_filename_column)
session.flush()
logger.info("[Library][Migration][9] Added filename column to entries table")
# Populate the new filename column.
for entry in self.__all_entries(session):
entry.filename = entry.path.name
session.merge(entry)
session.flush()
logger.info("[Library][Migration][9] Populated filename column in entries table")
def __apply_db100_migration(self, session: Session, library_dir: Path):
"""Migrate DB to DB_VERSION 100."""
# Repair parent-child tag relationships that are the wrong way around.
stmt = update(TagParent).values(
parent_id=TagParent.child_id,
child_id=TagParent.parent_id,
)
session.execute(stmt)
session.flush()
logger.info("[Library][Migration][100] Refactored TagParent table")
def __apply_db101_migration(self, session: Session, library_dir: Path):
"""Migrate DB to DB_VERSION 101."""
# Ensure version rows are present
session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100))
session.flush()
def __apply_db102_migration(self, session: Session, library_dir: Path):
"""Migrate DB to DB_VERSION 102."""
# delete TagParents with a dangling parent reference
stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct()))
session.execute(stmt)
session.flush()
logger.info("[Library][Migration][102] Verified TagParent table data")
def __apply_db103_migration(self, session: Session, library_dir: Path):
"""Migrate DB from DB_VERSION 102 to 103."""
# add the new hidden column for tags
session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0"))
session.flush()
logger.info("[Library][Migration][103] Added is_hidden column to tags table")
# mark the "Archived" tag as hidden
session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True})
session.flush()
logger.info("[Library][Migration][103] Updated archived tag to be hidden")
def __apply_db104_migration(self, session: Session, library_dir: Path):
"""Migrate DB from DB_VERSION 103 to 104."""
# Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist
self.__migrate_sql_to_ts_ignore(session, library_dir)
session.execute(text("DROP TABLE preferences"))
session.flush()
def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path):
# Do not continue if existing '.ts_ignore' file is found
ts_ignore = library_dir / TS_FOLDER_NAME / IGNORE_NAME
if Path(ts_ignore).exists():
return
# Load legacy extension data
extensions: list[str] = ujson.loads(
unwrap(
session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'"))
)
)
is_exclude_list: bool = unwrap(
session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'"))
)
with open(ts_ignore, "w") as f:
f.write(migrate_ext_list(extensions, is_exclude_list))
def __apply_db200_migration(self, session: Session, library_dir: Path):
"""Migrate DB to DB_VERSION 200."""
# Drop unused 'boolean_fields' and 'value_type' tables
logger.info("[Library][Migration][200] Dropping boolean_fields and value_type tables...")
session.execute(text("DROP TABLE boolean_fields"))
session.execute(text("DROP TABLE value_type"))
# Add 'name' column to text_fields and datetime_fields tables
logger.info("[Library][Migration][200] Adding name columns to field tables...")
stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""')
session.execute(stmt)
stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""')
session.execute(stmt)
# Drop unnecessary 'position' columns
logger.info("[Library][Migration][200] Dropping position columns to field tables...")
session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position"))
session.execute(text("ALTER TABLE text_fields DROP COLUMN position"))
# Add 'is_multiline' column to text_fields table
logger.info("[Library][Migration][200] Adding is_multiline column to text_fields...")
stmt = text("ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0")
session.execute(stmt)
session.flush()
# Move values from old `type_key` columns into new `name` columns
logger.info("[Library][Migration][200] Moving values from type_key columns to name...")
session.execute(text("UPDATE text_fields SET name = type_key"))
session.execute(text("UPDATE datetime_fields SET name = type_key"))
session.flush()
# Change `name` values to title case
logger.info("[Library][Migration][200] Normalizing TextField names...")
for text_field in session.execute(select(TextField)).scalars():
# NOTE: The only exception to the "Title Case" conversion is the "URL" field.
text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ")
logger.info("[Library][Migration][200] Normalizing DatetimeField names...")
for datetime_field in session.execute(select(DatetimeField)).scalars():
datetime_field.name = datetime_field.name.title().replace("_", " ")
session.flush()
# Add correct `is_multiline` values to text_fields table
logger.info("[Library][Migration][200] Updating is_multiline for legacy TEXT_BOXes...")
text_boxes = [
x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True
]
update_stmt = (
update(TextField).where(TextField.name.in_(text_boxes)).values(is_multiline=True)
)
session.execute(update_stmt)
session.flush()
# Repair legacy "Description" fields to use is_multiline = True
logger.info("[Library][Migration][200] Repairing legacy Description fields...")
desc_stmt = (
update(TextField)
.where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712
.values(is_multiline=True)
)
session.execute(desc_stmt)
# Repair legacy "Comments" fields to use is_multiline = True
logger.info("[Library][Migration][200] Repairing legacy Comment fields...")
comm_stmt = (
update(TextField)
.where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712
.values(is_multiline=True)
)
session.execute(comm_stmt)
# Add default field templates
logger.info("[Library][Migration][200] Adding default field templates...")
for template in get_default_field_templates():
session.add(template)
session.flush()
# DB indices for improved performance
session.execute(
text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)")
)
session.execute(
text("CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)")
)
session.execute(
text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)")
)
def __apply_db201_migration(self, session: Session, library_dir: Path):
"""Migrate DB to DB_VERSION 201."""
create_text_fields_table = text("""
CREATE TABLE text_fields_new (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
name VARCHAR NOT NULL,
entry_id INTEGER NOT NULL,
value VARCHAR,
is_multiline BOOLEAN NOT NULL,
FOREIGN KEY(entry_id) REFERENCES entries (id)
)
""")
create_datetime_fields_table = text("""
CREATE TABLE datetime_fields_new (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
name VARCHAR NOT NULL,
entry_id INTEGER NOT NULL,
value VARCHAR,
FOREIGN KEY(entry_id) REFERENCES entries (id)
)
""")
logger.info("[Library][Migration][201] Dropping type_key from text_fields table...")
session.execute(create_text_fields_table)
session.flush()
session.execute(
text("""
INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline)
SELECT id, name, entry_id, value, is_multiline
FROM text_fields
""")
)
session.execute(text("DROP TABLE text_fields"))
session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields"))
logger.info("[Library][Migration][201] Dropping type_key from datetime_fields table...")
session.execute(create_datetime_fields_table)
session.flush()
session.execute(
text("""
INSERT INTO datetime_fields_new (id, name, entry_id, value)
SELECT id, name, entry_id, value
FROM datetime_fields
""")
)
session.execute(text("DROP TABLE datetime_fields"))
session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields"))
session.flush()
def __apply_db202_migration(self, session: Session, library_dir: Path):
"""Migrate DB to DB_VERSION 202."""
stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct()))
session.execute(stmt)
session.flush()
logger.info("[Library][Migration][202] Verified TagParent table data")
def __apply_db300_migration(self, session: Session, library_dir: Path):
## remove folder_id column from entries table
# create new table in the desired scheme (without folder_id column)
session.execute(
text("""
CREATE TABLE entries_new (
id INTEGER NOT NULL,
path VARCHAR NOT NULL,
suffix VARCHAR NOT NULL,
date_created DATETIME,
date_modified DATETIME,
date_added DATETIME,
filename TEXT NOT NULL DEFAULT '',
PRIMARY KEY (id),
UNIQUE (path)
)
""")
)
session.flush()
# transfer data to new table
session.execute(
text("""
INSERT INTO entries_new (id, path, suffix, date_created, date_modified, date_added,
filename)
SELECT id, path, suffix, date_created, date_modified, date_added, filename
FROM entries
""")
)
# delete old table
session.execute(text("DROP TABLE entries"))
# rename new table to old table
session.execute(text("ALTER TABLE entries_new RENAME TO entries"))
session.flush()
## drop table "folders"
session.execute(text("DROP TABLE folders"))
session.flush()
@property
def field_templates(self) -> Sequence[BaseFieldTemplate]:
with Session(self.engine) as session:
@@ -1061,7 +669,8 @@ class Library:
with Session(self.engine) as session:
return unwrap(session.scalar(select(func.count(Entry.id))))
def __all_entries(self, session: Session, with_joins: bool = False) -> Iterator[Entry]:
@staticmethod
def _all_entries(session: Session, with_joins: bool = False) -> Iterator[Entry]:
"""Load entries without joins."""
stmt = select(Entry)
if with_joins:
@@ -1090,7 +699,7 @@ class Library:
def all_entries(self, with_joins: bool = False) -> Iterator[Entry]:
"""Load entries without joins."""
with Session(self.engine) as session:
return self.__all_entries(session, with_joins)
return Library._all_entries(session, with_joins)
@property
def tags(self) -> list[Tag]:
@@ -1838,16 +1447,17 @@ class Library:
session.rollback()
return None
def save_library_backup_to_disk(self) -> Path:
assert isinstance(self.library_dir, Path)
makedirs(str(self.library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME), exist_ok=True)
@staticmethod
def save_library_backup_to_disk(library_dir: Path) -> Path:
assert isinstance(library_dir, Path)
makedirs(str(library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME), exist_ok=True)
filename = f"ts_library_backup_{datetime.now(UTC).strftime('%Y_%m_%d_%H%M%S')}.sqlite"
target_path = self.library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME / filename
target_path = library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME / filename
shutil.copy2(
self.library_dir / TS_FOLDER_NAME / SQL_FILENAME,
library_dir / TS_FOLDER_NAME / SQL_FILENAME,
target_path,
)
@@ -2139,8 +1749,12 @@ class Library:
Args:
key(str): The key for the name of the version type to set.
"""
with Session(self.engine) as session:
engine = sqlalchemy.inspect(self.engine)
return Library._get_version(self.engine, key)
@staticmethod
def _get_version(engine, key: str) -> int:
with Session(engine) as session:
engine = sqlalchemy.inspect(engine)
try:
# "Version" table added in DB_VERSION 101
if engine and engine.has_table("versions"):
@@ -2160,17 +1774,6 @@ class Library:
except Exception:
return 0
def set_version(self, session: Session, key: str, value: int) -> None:
"""Set a version value to the DB.
Args:
session(Session): The SQLAlchemy DB Session to use.
key(str): The key for the name of the version type to set.
value(int): The version value to set.
"""
# Insert if key has no value yet, otherwise update the value
session.merge(Version(key=key, value=value))
def mirror_entry_fields(self, entries: list[Entry]) -> None:
"""Mirror fields among multiple Entry items."""
all_fields: set[BaseField] = set()
src/tagstudio/core/library/alchemy/migrations.py
+571
View File
@@ -0,0 +1,571 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: MIT
from collections.abc import Callable
from pathlib import Path
from typing import override
import structlog
import ujson
from sqlalchemy import Engine, and_, delete, select, text, update
from sqlalchemy.orm import Session
from tagstudio.core.constants import IGNORE_NAME, TAG_ARCHIVED, TS_FOLDER_NAME
from tagstudio.core.library.alchemy import default_color_groups
from tagstudio.core.library.alchemy.constants import (
DB_VERSION,
DB_VERSION_CURRENT_KEY,
DB_VERSION_INITIAL_KEY,
DEFAULT_FIELD_TEMPLATES,
)
from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP, DatetimeField, TextField
from tagstudio.core.library.alchemy.joins import TagParent
from tagstudio.core.library.alchemy.models import Tag, TagColorGroup, Version
from tagstudio.core.library.ignore import migrate_ext_list
from tagstudio.core.utils.types import unwrap
from tagstudio.qt.translations import Translations
logger = structlog.get_logger(__name__)
class MigrationError(Exception):
pass
class DBMigration:
version: int
initial_version: int | None = None
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log: Callable[[str], str]):
raise NotImplementedError
class DBMigrations:
def __init__(self, library_dir: Path, engine: Engine) -> None:
from tagstudio.core.library.alchemy.library import Library
self.library_dir = library_dir
self.engine = engine
# Don't check DB version when creating new library
self.loaded_db_version = Library._get_version(engine, DB_VERSION_CURRENT_KEY)
self.initial_db_version = Library._get_version(engine, DB_VERSION_INITIAL_KEY)
# ======================== Library Database Version Checking =======================
# DB_VERSION 6 is the first supported SQLite DB version.
# If the DB_VERSION is >= 100, that means it's a compound major + minor version.
# - Dividing by 100 and flooring gives the major (breaking changes) version.
# - If a DB has major version higher than the current program, don't load it.
# - If only the minor version is higher, it's still allowed to load.
if self.loaded_db_version < 6 or (
self.loaded_db_version >= 100 and self.loaded_db_version // 100 > DB_VERSION // 100
):
mismatch_text = Translations["status.library_version_mismatch"]
found_text = Translations["status.library_version_found"]
expected_text = Translations["status.library_version_expected"]
raise MigrationError(
f"{mismatch_text}\n"
f"{found_text} v{self.loaded_db_version}, "
f"{expected_text} v{DB_VERSION}"
)
logger.info(
f"[Library][Migration] Starting with library DB version: {self.loaded_db_version}"
)
@property
def required(self) -> bool:
return self.loaded_db_version < DB_VERSION
def run(self):
# migrate DB step by step from one version to the next
# (migration_method, db_version, initial_db_version)
migrations: list[type[DBMigration]] = [
MigrationTo7, # changes: value_type, tags
MigrationTo8, # changes: tag_colors
MigrationTo9, # changes: entries
MigrationTo100, # changes: tag_parents
MigrationTo101, # changes: versions
MigrationTo102, # changes: tag_parents
MigrationTo103, # changes: tags
MigrationTo104, # changes: deletes preferences
MigrationTo200, # changes: field tables
MigrationTo201, # changes: field tables
MigrationTo202, # changes: tag_parents
MigrationTo300, # changes: deletes folders
]
with Session(self.engine) as session:
for migration in migrations:
if self.loaded_db_version < migration.version and (
migration.initial_version is None
or self.initial_db_version < migration.initial_version
):
logger.info(f"[Library][Migration][{migration.version}] Starting DB Migration")
# any error causes transaction to rollback
migration.run(
session,
self.library_dir,
lambda msg, v=migration.version: f"[Library][Migration][{v}] {msg}",
)
self.loaded_db_version = migration.version
try:
self._set_version(session, DB_VERSION_CURRENT_KEY, migration.version)
except Exception as e:
logger.info(
f"[Library][Migration][{migration.version}] "
"Couldn't update version, continuing without commit",
error=e,
)
session.flush()
else:
session.commit()
logger.info(f"[Library][Migration][{migration.version}] Completed DB Migration")
assert self.loaded_db_version >= DB_VERSION, (
"Ran all migrations, but the DB is still not on the newest version"
)
logger.info(f"[Library][Migration] Library migrated to DB version {DB_VERSION}")
def _set_version(self, session: Session, key: str, value: int) -> None:
"""Set a version value to the DB.
Args:
session(Session): The SQLAlchemy DB Session to use.
key(str): The key for the name of the version type to set.
value(int): The version value to set.
"""
# Insert if key has no value yet, otherwise update the value
session.merge(Version(key=key, value=value))
class MigrationTo7(DBMigration):
version = 7
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB from DB_VERSION 6 to 7."""
logger.info(fmt_log("Applying patches to DB_VERSION: 6 library..."))
# Repair tags that may have a disambiguation_id pointing towards a deleted tag.
# TODO: combine into single sql statement
all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all()
disam_stmt = (
update(Tag)
.where(Tag.disambiguation_id.not_in(all_tag_ids))
.values(disambiguation_id=None)
)
session.execute(disam_stmt)
session.flush()
class MigrationTo8(DBMigration):
version = 8
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB from DB_VERSION 7 to 8."""
# Add the missing color_border column to the TagColorGroups table.
session.execute(
text("ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL")
)
session.flush()
logger.info(fmt_log("Added color_border column to tag_colors table"))
# collect new default tag colors
tag_colors: list[TagColorGroup] = [
color
for color in default_color_groups.shades()
if color.slug in ["burgundy", "dark-teal", "dark_lavender"]
]
# Add any new default colors introduced in DB_VERSION 8
for color in tag_colors:
session.add(color)
session.flush()
logger.info(
fmt_log("Migrated tag colors to DB_VERSION 8+"),
color_name=tag_colors,
)
# Update Neon colors to use the the color_border property
for color in default_color_groups.neon():
neon_stmt = (
update(TagColorGroup)
.where(
and_(
TagColorGroup.namespace == color.namespace,
TagColorGroup.slug == color.slug,
)
)
.values(
slug=color.slug,
namespace=color.namespace,
name=color.name,
primary=color.primary,
secondary=color.secondary,
color_border=color.color_border,
)
)
session.execute(neon_stmt)
session.flush()
class MigrationTo9(DBMigration):
version = 9
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB from DB_VERSION 8 to 9."""
# Apply database schema changes
add_filename_column = text(
"ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''"
)
session.execute(add_filename_column)
session.flush()
logger.info(fmt_log("Added filename column to entries table"))
# Populate the new filename column.
from tagstudio.core.library.alchemy.library import Library
for entry in Library._all_entries(session):
entry.filename = entry.path.name
session.merge(entry)
session.flush()
logger.info(fmt_log("Populated filename column in entries table"))
class MigrationTo100(DBMigration):
version = 100
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB to DB_VERSION 100."""
# Repair parent-child tag relationships that are the wrong way around.
stmt = update(TagParent).values(
parent_id=TagParent.child_id,
child_id=TagParent.parent_id,
)
session.execute(stmt)
session.flush()
logger.info(fmt_log("Refactored TagParent table"))
class MigrationTo101(DBMigration):
version = 101
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB to DB_VERSION 101."""
# Create versions table
session.execute(
text("""
CREATE TABLE versions (
"key" VARCHAR NOT NULL PRIMARY KEY,
value INTEGER NOT NULL
)
""")
)
session.flush()
# Ensure version rows are present
session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100))
session.flush()
logger.info(fmt_log("Created versions table"))
class MigrationTo102(DBMigration):
version = 102
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB to DB_VERSION 102."""
# delete TagParents with a dangling parent reference
stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct()))
session.execute(stmt)
session.flush()
logger.info(fmt_log("Verified TagParent table data"))
class MigrationTo103(DBMigration):
version = 103
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB from DB_VERSION 102 to 103."""
# add the new hidden column for tags
session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0"))
session.flush()
logger.info(fmt_log("Added is_hidden column to tags table"))
# mark the "Archived" tag as hidden
session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True})
session.flush()
logger.info(fmt_log("Updated archived tag to be hidden"))
class MigrationTo104(DBMigration):
version = 104
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB from DB_VERSION 103 to 104."""
# Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist
cls.__migrate_sql_to_ts_ignore(session, library_dir)
session.execute(text("DROP TABLE preferences"))
session.flush()
@classmethod
def __migrate_sql_to_ts_ignore(cls, session: Session, library_dir: Path):
# Do not continue if existing '.ts_ignore' file is found
ts_ignore = library_dir / TS_FOLDER_NAME / IGNORE_NAME
if Path(ts_ignore).exists():
return
# Load legacy extension data
extensions: list[str] = ujson.loads(
unwrap(
session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'"))
)
)
is_exclude_list: bool = unwrap(
session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'"))
)
with open(ts_ignore, "w") as f:
f.write(migrate_ext_list(extensions, is_exclude_list))
class MigrationTo200(DBMigration):
version = 200
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB to DB_VERSION 200."""
# Drop unused 'boolean_fields' and 'value_type' tables
logger.info(fmt_log("Dropping boolean_fields and value_type tables..."))
session.execute(text("DROP TABLE boolean_fields"))
session.execute(text("DROP TABLE value_type"))
# Add 'name' column to text_fields and datetime_fields tables
logger.info(fmt_log("Adding name columns to field tables..."))
stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""')
session.execute(stmt)
stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""')
session.execute(stmt)
# Drop unnecessary 'position' columns
logger.info(fmt_log("Dropping position columns to field tables..."))
session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position"))
session.execute(text("ALTER TABLE text_fields DROP COLUMN position"))
# Add 'is_multiline' column to text_fields table
logger.info(fmt_log("Adding is_multiline column to text_fields..."))
stmt = text("ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0")
session.execute(stmt)
session.flush()
# Move values from old `type_key` columns into new `name` columns
logger.info(fmt_log("Moving values from type_key columns to name..."))
session.execute(text("UPDATE text_fields SET name = type_key"))
session.execute(text("UPDATE datetime_fields SET name = type_key"))
session.flush()
# Change `name` values to title case
logger.info(fmt_log("Normalizing TextField names..."))
for text_field in session.execute(select(TextField)).scalars():
# NOTE: The only exception to the "Title Case" conversion is the "URL" field.
text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ")
logger.info(fmt_log("Normalizing DatetimeField names..."))
for datetime_field in session.execute(select(DatetimeField)).scalars():
datetime_field.name = datetime_field.name.title().replace("_", " ")
session.flush()
# Add correct `is_multiline` values to text_fields table
logger.info(fmt_log("Updating is_multiline for legacy TEXT_BOXes..."))
text_boxes = [
x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True
]
update_stmt = (
update(TextField).where(TextField.name.in_(text_boxes)).values(is_multiline=True)
)
session.execute(update_stmt)
session.flush()
# Repair legacy "Description" fields to use is_multiline = True
logger.info(fmt_log("Repairing legacy Description fields..."))
desc_stmt = (
update(TextField)
.where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712
.values(is_multiline=True)
)
session.execute(desc_stmt)
# Repair legacy "Comments" fields to use is_multiline = True
logger.info(fmt_log("Repairing legacy Comment fields..."))
comm_stmt = (
update(TextField)
.where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712
.values(is_multiline=True)
)
session.execute(comm_stmt)
# Add field templates tables
session.execute(
text("""
CREATE TABLE text_field_templates (
id INTEGER NOT NULL PRIMARY KEY,
is_multiline BOOLEAN NOT NULL,
name VARCHAR NOT NULL
)
""")
)
session.execute(
text("""
CREATE TABLE datetime_field_templates (
id INTEGER NOT NULL PRIMARY KEY,
name VARCHAR NOT NULL
)
""")
)
session.flush()
# Add default field templates
logger.info(fmt_log("Adding default field templates..."))
for template in DEFAULT_FIELD_TEMPLATES:
session.add(template)
session.flush()
# DB indices for improved performance
session.execute(
text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)")
)
session.execute(
text("CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)")
)
session.execute(
text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)")
)
class MigrationTo201(DBMigration):
version = 201
initial_version = 200
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB to DB_VERSION 201."""
create_text_fields_table = text("""
CREATE TABLE text_fields_new (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
name VARCHAR NOT NULL,
entry_id INTEGER NOT NULL,
value VARCHAR,
is_multiline BOOLEAN NOT NULL,
FOREIGN KEY(entry_id) REFERENCES entries (id)
)
""")
create_datetime_fields_table = text("""
CREATE TABLE datetime_fields_new (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
name VARCHAR NOT NULL,
entry_id INTEGER NOT NULL,
value VARCHAR,
FOREIGN KEY(entry_id) REFERENCES entries (id)
)
""")
logger.info(fmt_log("Dropping type_key from text_fields table..."))
session.execute(create_text_fields_table)
session.flush()
session.execute(
text("""
INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline)
SELECT id, name, entry_id, value, is_multiline
FROM text_fields
""")
)
session.execute(text("DROP TABLE text_fields"))
session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields"))
logger.info(fmt_log("Dropping type_key from datetime_fields table..."))
session.execute(create_datetime_fields_table)
session.flush()
session.execute(
text("""
INSERT INTO datetime_fields_new (id, name, entry_id, value)
SELECT id, name, entry_id, value
FROM datetime_fields
""")
)
session.execute(text("DROP TABLE datetime_fields"))
session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields"))
session.flush()
class MigrationTo202(DBMigration):
version = 202
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
"""Migrate DB to DB_VERSION 202."""
stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct()))
session.execute(stmt)
session.flush()
logger.info(fmt_log("Verified TagParent table data"))
class MigrationTo300(DBMigration):
version = 300
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log):
## remove folder_id column from entries table
# create new table in the desired scheme (without folder_id column)
session.execute(
text("""
CREATE TABLE entries_new (
id INTEGER NOT NULL,
path VARCHAR NOT NULL,
suffix VARCHAR NOT NULL,
date_created DATETIME,
date_modified DATETIME,
date_added DATETIME,
filename TEXT NOT NULL DEFAULT '',
PRIMARY KEY (id),
UNIQUE (path)
)
""")
)
session.flush()
# transfer data to new table
session.execute(
text("""
INSERT INTO entries_new (id, path, suffix, date_created, date_modified, date_added,
filename)
SELECT id, path, suffix, date_created, date_modified, date_added, filename
FROM entries
""")
)
# delete old table
session.execute(text("DROP TABLE entries"))
# rename new table to old table
session.execute(text("ALTER TABLE entries_new RENAME TO entries"))
session.flush()
## drop table "folders"
session.execute(text("DROP TABLE folders"))
session.flush()
src/tagstudio/qt/ts_qt.py
+1 -1
View File
@@ -849,7 +849,7 @@ class QtDriver(DriverMixin, QObject):
logger.info("Backing Up Library...")
self.main_window.status_bar.showMessage(Translations["status.library_backup_in_progress"])
start_time = time.time()
target_path = self.lib.save_library_backup_to_disk()
target_path = Library.save_library_backup_to_disk(unwrap(self.lib.library_dir))
end_time = time.time()
self.main_window.status_bar.showMessage(
Translations.format(