Refactor history dialog to use virtualized list and SQLite

Replaces the history dialog's widget-per-entry approach with a virtualized QListView using a custom model and delegate for improved performance. Switches the HistoryManager backend from JSON file storage to SQLite, including automatic migration of legacy data. Adds support for efficient search, thumbnail caching, and context menu actions in the new UI.

src/utils/ytsage_history_manager.py

@@ -3,144 +3,259 @@ History Manager Module
 ======================
 
 This module provides **thread-safe** centralized management for download
-history in YTSage. It handles reading, writing, and managing download history
+history in YTSage using SQLite for high performance and scalability.
-stored in a JSON file.
-
-Thread safety is ensured using a reentrant lock (`RLock`), so multiple threads
-can safely access or modify history concurrently.
 
 Features
 --------
-- Thread-safe operations for getting, adding, and removing history entries.
+- Scalable: Uses SQLite instead of parsing potentially large JSON files.
-- Loads history from a JSON file (`APP_HISTORY_FILE`).
+- Thread-safe: Handles database connections safely.
-- Creates the history file if missing or corrupt.
+- Migration: Automatically migrates legacy JSON history to SQLite.
-- Manages download history with metadata including thumbnails, file paths, and download options.
+- CRUD: Create, Read, Delete, Clear operations for history entries.
-- Provides safe error handling with logging instead of raising exceptions.
-- Persists updates back to disk automatically.
 
 Usage
 -----
 from src.utils.ytsage_history_manager import HistoryManager
 
 # Add a download to history
-HistoryManager.add_entry(
+HistoryManager.add_entry(...)
-    title="Video Title",
-    url="https://youtube.com/watch?v=...",
-    thumbnail_url="https://...",
-    file_path="/path/to/file.mp4",
-    format_id="137+140",
-    is_audio_only=False,
-    resolution="1080p",
-    download_options={...}
-)
 
 # Get all history entries
 history = HistoryManager.get_all_entries()
 
+# Get recent entries (limit + offset support planned)
 # Remove an entry
 HistoryManager.remove_entry(entry_id)
 
 # Clear all history
 HistoryManager.clear_history()
-
-Design Notes
-------------
-- History entries are stored in `HistoryManager._history` (a list of dicts).
-- Each entry has a unique ID based on timestamp.
-- All modifications trigger a save (`_save`) to keep JSON in sync.
-- Logs actions and errors using the app's central logger.
-- Uses `RLock` to allow safe concurrent access from multiple threads.
-
-Exceptions
-----------
-- Any issues during file I/O (permissions, disk errors, JSON corruption)
-  are caught and logged. The application continues running with an empty
-  history when possible.
 """
 
 import json
+import sqlite3
 import threading
 import time
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
-from src.utils.ytsage_constants import APP_HISTORY_FILE
+from src.utils.ytsage_constants import APP_HISTORY_FILE, APP_DATA_DIR
 from src.utils.ytsage_logger import logger
 
 
 class HistoryManager:
     """
-    Thread-safe history manager for YTSage.
+    Thread-safe history manager for YTSage using SQLite.
-
-    Provides methods to load, save, get, add, and remove download history entries.
-    Automatically persists changes to disk.
     """
 
     _lock = threading.RLock()
-    _history_file = APP_HISTORY_FILE
+    # Define DB file next to the old JSON file
-    _history: List[Dict[str, Any]] = []
+    _db_file = APP_DATA_DIR / "ytsage_history.db"
-    _loaded = False
+    _initialized = False
 
     @classmethod
-    def _load(cls) -> None:
+    def _init_db(cls):
-        """
+        """Initialize the database: create table and migrate if needed."""
-        Loads download history from a JSON file if it exists and is valid.
+        if cls._initialized:
-        If the file is missing or corrupt, initializes with an empty history.
+            return
-        Logs actions and errors during the process.
-        """
-        with cls._lock:
-            if cls._history_file.exists():
-                try:
-                    with open(cls._history_file, "r", encoding="utf-8") as f:
-                        data = json.load(f)
-                        # Ensure it's a list
-                        if isinstance(data, list):
-                            cls._history = data
-                        else:
-                            cls._history = []
-                            logger.warning("History file format invalid, initialized empty history.")
-                    logger.info(f"History loaded from file: {len(cls._history)} entries.")
-                except json.JSONDecodeError:
-                    cls._history = []
-                    logger.warning("History file corrupt, initialized empty history.")
-                except Exception as e:
-                    cls._history = []
-                    logger.error(f"Error loading history file: {e}")
-            else:
-                cls._history = []
-                cls._save()
-                logger.info("History file not found, created empty history.")
-            cls._loaded = True
 
-    @classmethod
-    def _save(cls) -> None:
-        """
-        Save current history to JSON file.
-
-        Note:
-            May raise exceptions if the file cannot be written due to permission issues,
-            disk errors, or other I/O problems.
-        """
         with cls._lock:
+            # Check if file exists to know if we need to migrate or just create schema
+            db_exists = cls._db_file.exists()
+            legacy_json_exists = APP_HISTORY_FILE.exists()
+
             try:
-                # Ensure parent directory exists
+                # Ensure directory exists
-                cls._history_file.parent.mkdir(parents=True, exist_ok=True)
+                cls._db_file.parent.mkdir(parents=True, exist_ok=True)
                 
-                with open(cls._history_file, "w", encoding="utf-8") as f:
+                with sqlite3.connect(cls._db_file, check_same_thread=False) as conn:
-                    json.dump(cls._history, f, indent=2, ensure_ascii=False)
+                    cursor = conn.cursor()
-                logger.debug(f"History saved to file: {len(cls._history)} entries.")
+                    
-            except (OSError, PermissionError) as e:
+                    # Create table
-                logger.exception(f"Failed to save history: {e}")
+                    cursor.execute("""
-            except Exception as e:
+                        CREATE TABLE IF NOT EXISTS history (
-                logger.exception(f"Unexpected error while saving history: {e}")
+                            id TEXT PRIMARY KEY,
+                            title TEXT,
+                            url TEXT,
+                            channel TEXT,
+                            file_path TEXT,
+                            download_date TEXT,
+                            file_size INTEGER,
+                            thumbnail_url TEXT,
+                            format_id TEXT,
+                            resolution TEXT,
+                            is_audio_only INTEGER,
+                            duration TEXT,
+                            options TEXT,
+                            timestamp REAL
+                        )
+                    """)
+                    
+                    # Index for faster sorting by date
+                    cursor.execute("""
+                        CREATE INDEX IF NOT EXISTS idx_timestamp 
+                        ON history (timestamp DESC)
+                    """)
+                    
+                    conn.commit()
+            
+                # If we just created the DB and have a JSON file, migrate
+                if not db_exists and legacy_json_exists:
+                    cls._migrate_legacy_json()
+                
+                cls._initialized = True
+                
+            except sqlite3.Error as e:
+                logger.error(f"Failed to initialize history database: {e}")
 
     @classmethod
-    def _ensure_loaded(cls) -> None:
+    def _migrate_legacy_json(cls):
-        """Ensure history is loaded before any operation."""
+        """Migrate legacy JSON history to SQLite."""
-        with cls._lock:
+        logger.info("Migrating legacy history JSON to SQLite...")
-            if not cls._loaded:
+        try:
-                cls._load()
+            with open(APP_HISTORY_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                
+            if isinstance(data, list):
+                count = 0
+                with sqlite3.connect(cls._db_file, check_same_thread=False) as conn:
+                    cursor = conn.cursor()
+                    for entry in data:
+                        try:
+                            # Safely extract download_options logic if complex
+                            options_json = json.dumps(entry.get("download_options", {}))
+                            
+                            # Construct timestamp from isoformat if missing
+                            ts = entry.get("timestamp")
+                            if not ts and "download_date" in entry:
+                                try:
+                                    dt = datetime.fromisoformat(entry["download_date"])
+                                    ts = dt.timestamp()
+                                except Exception:
+                                    ts = time.time()
+                            
+                            cursor.execute("""
+                                INSERT OR IGNORE INTO history (
+                                    id, title, url, channel, file_path, download_date,
+                                    file_size, thumbnail_url, format_id, resolution,
+                                    is_audio_only, duration, options, timestamp
+                                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                            """, (
+                                entry.get("id", str(int(time.time()*1000))),
+                                entry.get("title", ""),
+                                entry.get("url", ""),
+                                entry.get("channel", "Unknown"),
+                                entry.get("file_path", ""),
+                                entry.get("download_date", ""),
+                                entry.get("file_size", 0),
+                                entry.get("thumbnail_url", ""),
+                                entry.get("format_id", ""),
+                                entry.get("resolution", ""),
+                                1 if entry.get("is_audio_only") else 0,
+                                entry.get("duration", ""),
+                                options_json,
+                                ts or time.time()
+                            ))
+                            count += 1
+                        except Exception as e:
+                            logger.error(f"Skipped invalid entry during migration: {e}")
+                    
+                    conn.commit()
+                
+                logger.info(f"Successfully migrated {count} history entries.")
+                
+                # Rename old JSON to .bak to avoid re-migration, or keep as backup
+                try:
+                    APP_HISTORY_FILE.rename(APP_HISTORY_FILE.with_suffix(".json.bak"))
+                except Exception as e:
+                    logger.warning(f"Could not rename legacy history file: {e}")
+                    
+        except Exception as e:
+            logger.error(f"Migration failed: {e}")
+
+    @classmethod
+    def _get_connection(cls):
+        """Get a database connection."""
+        cls._init_db()
+        return sqlite3.connect(cls._db_file, check_same_thread=False)
+
+    @classmethod
+    def get_all_entries(cls, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Get all history entries, sorted by most recent first.
+        
+        Args:
+            limit: Optional limit on number of entries to return (most recent first)
+
+        Returns:
+            List of dictionary entries.
+        """
+        entries = []
+        try:
+            with cls._lock:  # Lock for simple concurrency safety
+                with cls._get_connection() as conn:
+                    # Return dict-like rows
+                    conn.row_factory = sqlite3.Row
+                    cursor = conn.cursor()
+                    
+                    query = "SELECT * FROM history ORDER BY timestamp DESC"
+                    params = ()
+                    
+                    if limit is not None:
+                        query += " LIMIT ?"
+                        params = (limit,)
+                        
+                    cursor.execute(query, params)
+                    rows = cursor.fetchall()
+                    
+                    for row in rows:
+                        entry = dict(row)
+                        # Convert boolean back
+                        entry["is_audio_only"] = bool(entry["is_audio_only"])
+                        # Parse options JSON
+                        try:
+                            entry["download_options"] = json.loads(entry["options"]) if entry["options"] else {}
+                        except json.JSONDecodeError:
+                            entry["download_options"] = {}
+                        del entry["options"] # Remove internal column
+                        entries.append(entry)
+                        
+        except Exception as e:
+            logger.error(f"Error fetching history: {e}")
+            
+        return entries
+
+    @classmethod
+    def get_entry(cls, entry_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Get a specific history entry by ID.
+
+        Args:
+            entry_id: The unique entry ID
+
+        Returns:
+            Entry dictionary or None if not found
+        """
+        try:
+            with cls._lock:
+                with cls._get_connection() as conn:
+                    conn.row_factory = sqlite3.Row
+                    cursor = conn.cursor()
+                    cursor.execute("SELECT * FROM history WHERE id = ?", (entry_id,))
+                    row = cursor.fetchone()
+                    
+                    if row:
+                        entry = dict(row)
+                        entry["is_audio_only"] = bool(entry["is_audio_only"])
+                        try:
+                            entry["download_options"] = json.loads(entry["options"]) if entry["options"] else {}
+                        except json.JSONDecodeError:
+                            entry["download_options"] = {}
+                        del entry["options"]
+                        return entry
+            return None
+        except Exception as e:
+            logger.error(f"Error fetching entry {entry_id}: {e}")
+            return None
 
     @classmethod
     def add_entry(
@@ -158,140 +273,98 @@ class HistoryManager:
         download_options: Optional[Dict[str, Any]] = None,
     ) -> str:
         """
-        Add a new download entry to history.
+        Add a new entry to the download history.
-
-        Args:
-            title: Video/audio title
-            url: Original URL
-            thumbnail_url: Thumbnail URL (can be None)
-            file_path: Path to downloaded file
-            format_id: Format ID used for download
-            is_audio_only: Whether it's audio-only download
-            resolution: Resolution string (e.g., "1080p", "best audio")
-            file_size: File size in bytes (optional)
-            channel: Channel name (optional)
-            duration: Duration string (optional)
-            download_options: Dictionary of all download options used (optional)
-
-        Returns:
-            str: The unique ID of the created entry
         """
-        cls._ensure_loaded()
+        if download_options is None:
+            download_options = {}
 
-        with cls._lock:
+        timestamp = time.time()
-            # Generate unique ID based on timestamp
+        # Ensure unique ID
-            entry_id = f"{int(time.time() * 1000)}"
+        unique_id = f"{int(timestamp * 1000)}"
+        download_date = datetime.fromtimestamp(timestamp).isoformat()
         
-            # Get file size if not provided
+        # Determine file size if not provided
-            if file_size is None:
+        if file_size is None:
-                try:
+            try:
-                    file_path_obj = Path(file_path)
+                p = Path(file_path)
-                    if file_path_obj.exists():
+                if p.exists():
-                        file_size = file_path_obj.stat().st_size
+                    file_size = p.stat().st_size
-                except Exception as e:
+            except Exception:
-                    logger.debug(f"Could not get file size: {e}")
+                file_size = 0
-                    file_size = 0
 
-            entry = {
+        # Allow None for optional strings
-                "id": entry_id,
+        channel = channel or "Unknown"
-                "title": title,
+        duration = duration or ""
-                "url": url,
+        thumbnail_url = thumbnail_url or ""
-                "thumbnail_url": thumbnail_url,
-                "file_path": file_path,
-                "download_date": datetime.now().isoformat(),
-                "format_id": format_id,
-                "is_audio_only": is_audio_only,
-                "resolution": resolution,
-                "file_size": file_size or 0,
-                "channel": channel,
-                "duration": duration,
-                "download_options": download_options or {},
-            }
         
-            # Add to beginning of list (most recent first)
+        try:
-            cls._history.insert(0, entry)
+            with cls._lock:
-            cls._save()
+                with cls._get_connection() as conn:
+                    cursor = conn.cursor()
                     
-            logger.info(f"Added entry to history: {title}")
+                    cursor.execute("""
-            return entry_id
+                        INSERT INTO history (
+                            id, title, url, channel, file_path, download_date,
+                            file_size, thumbnail_url, format_id, resolution,
+                            is_audio_only, duration, options, timestamp
+                        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """, (
+                        unique_id,
+                        title,
+                        url,
+                        channel,
+                        str(file_path),
+                        download_date,
+                        file_size,
+                        thumbnail_url,
+                        format_id,
+                        resolution,
+                        1 if is_audio_only else 0,
+                        duration,
+                        json.dumps(download_options),
+                        timestamp
+                    ))
+                    conn.commit()
             
-    @classmethod
+            logger.info(f"Added history entry: {title}")
-    def get_all_entries(cls, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+            return unique_id
-        """
-        Retrieve all history entries.
             
-        Args:
+        except Exception as e:
-            limit: Optional limit on number of entries to return (most recent first)
+            logger.error(f"Error adding history entry: {e}")
-
+            return ""
-        Returns:
-            List of history entry dictionaries
-        """
-        cls._ensure_loaded()
-        
-        with cls._lock:
-            if limit is not None:
-                return cls._history[:limit]
-            return cls._history.copy()
-
-    @classmethod
-    def get_entry(cls, entry_id: str) -> Optional[Dict[str, Any]]:
-        """
-        Get a specific history entry by ID.
-
-        Args:
-            entry_id: The unique entry ID
-
-        Returns:
-            Entry dictionary or None if not found
-        """
-        cls._ensure_loaded()
-        
-        with cls._lock:
-            for entry in cls._history:
-                if entry.get("id") == entry_id:
-                    return entry.copy()
-            return None
 
     @classmethod
     def remove_entry(cls, entry_id: str) -> bool:
-        """
+        """Remove an entry from history by ID."""
-        Remove a specific entry from history.
+        try:
-
+            with cls._lock:
-        Args:
+                with cls._get_connection() as conn:
-            entry_id: The unique entry ID to remove
+                    cursor = conn.cursor()
-
+                    cursor.execute("DELETE FROM history WHERE id = ?", (entry_id,))
-        Returns:
+                    if cursor.rowcount > 0:
-            bool: True if entry was found and removed, False otherwise
+                        conn.commit()
-        """
+                        logger.info(f"Removed history entry: {entry_id}")
-        cls._ensure_loaded()
+                        return True
-        
+            return False
-        with cls._lock:
+        except Exception as e:
-            for i, entry in enumerate(cls._history):
+            logger.error(f"Error removing history entry: {e}")
-                if entry.get("id") == entry_id:
-                    removed = cls._history.pop(i)
-                    cls._save()
-                    logger.info(f"Removed entry from history: {removed.get('title', 'Unknown')}")
-                    return True
-            
-            logger.debug(f"Entry ID '{entry_id}' not found in history.")
             return False
 
     @classmethod
     def clear_history(cls) -> int:
-        """
+        """Clear all history entries."""
-        Clear all history entries.
+        try:
-
+            with cls._lock:
-        Returns:
+                with cls._get_connection() as conn:
-            int: Number of entries that were cleared
+                    cursor = conn.cursor()
-        """
+                    cursor.execute("DELETE FROM history")
-        cls._ensure_loaded()
+                    count = cursor.rowcount
-        
+                    conn.commit()
-        with cls._lock:
+            logger.info("History cleared")
-            count = len(cls._history)
-            cls._history = []
-            cls._save()
-            logger.info(f"Cleared all history: {count} entries removed.")
             return count
+        except Exception as e:
+            logger.error(f"Error clearing history: {e}")
+            return 0
             
     @classmethod
     def search_entries(cls, query: str) -> List[Dict[str, Any]]:
@@ -304,45 +377,34 @@ class HistoryManager:
         Returns:
             List of matching history entries
         """
-        cls._ensure_loaded()
-        
         if not query:
             return cls.get_all_entries()
             
-        query_lower = query.lower()
+        entries = []
+        try:
+            search_pattern = f"%{query}%"
+            with cls._lock:
+                with cls._get_connection() as conn:
+                    conn.row_factory = sqlite3.Row
+                    cursor = conn.cursor()
+                    cursor.execute("""
+                        SELECT * FROM history 
+                        WHERE title LIKE ? OR channel LIKE ? OR url LIKE ?
+                        ORDER BY timestamp DESC
+                    """, (search_pattern, search_pattern, search_pattern))
+                    rows = cursor.fetchall()
                     
-        with cls._lock:
+                    for row in rows:
-            results = []
+                        entry = dict(row)
-            for entry in cls._history:
+                        entry["is_audio_only"] = bool(entry["is_audio_only"])
-                # Search in title, channel, and URL
+                        try:
-                title = (entry.get("title") or "").lower()
+                            entry["download_options"] = json.loads(entry["options"]) if entry["options"] else {}
-                channel = (entry.get("channel") or "").lower()
+                        except json.JSONDecodeError:
-                url = (entry.get("url") or "").lower()
+                            entry["download_options"] = {}
+                        del entry["options"]
+                        entries.append(entry)
                         
-                if query_lower in title or query_lower in channel or query_lower in url:
+        except Exception as e:
-                    results.append(entry.copy())
+            logger.error(f"Error searching history: {e}")
             
-            return results
+        return entries
-
-    @classmethod
-    def get_statistics(cls) -> Dict[str, Any]:
-        """
-        Get statistics about download history.
-
-        Returns:
-            Dictionary with statistics (total_downloads, total_size, etc.)
-        """
-        cls._ensure_loaded()
-        
-        with cls._lock:
-            total_downloads = len(cls._history)
-            total_size = sum(entry.get("file_size", 0) for entry in cls._history)
-            video_count = sum(1 for entry in cls._history if not entry.get("is_audio_only", False))
-            audio_count = sum(1 for entry in cls._history if entry.get("is_audio_only", False))
-            
-            return {
-                "total_downloads": total_downloads,
-                "total_size": total_size,
-                "video_count": video_count,
-                "audio_count": audio_count,
-            }