perf(search): implement full async + indexed search system with major performance gains

Signed-off-by: Boris Yumankulov <boria138@altlinux.org>

portprotonqt/game_library_manager.py

@@ -1,5 +1,6 @@
 from typing import Protocol
 from portprotonqt.game_card import GameCard
+from portprotonqt.search_utils import SearchOptimizer, ThreadedSearch
 from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QScrollArea, QSlider, QScroller
 from PySide6.QtCore import Qt, QTimer
 from portprotonqt.custom_widgets import FlowLayout
@@ -56,6 +57,9 @@ class GameLibraryManager:
         self.pending_deletions = deque()
         self.is_filtering = False
         self.dirty = False
+        # Initialize search optimizer
+        self.search_optimizer = SearchOptimizer()
+        self.search_thread: ThreadedSearch | None = None
 
     def create_games_library_widget(self):
         """Creates the games library widget with search, grid, and slider."""
@@ -212,6 +216,10 @@ class GameLibraryManager:
             if games_list is not None:
                 self.filtered_games = games_list
             self.dirty = True  # Full rebuild only for non-filter
+        else:
+            # When filtering, we want to update with the current filtered_games
+            # which has already been set by _perform_search
+            pass
         self.is_filtering = is_filter
         self._pending_update = True
 
@@ -242,8 +250,9 @@ class GameLibraryManager:
         search_text = self.main_window.searchEdit.text().strip().lower()
 
         if self.is_filtering:
-            # Filter mode: do not change layout, only hide/show cards
-            self._apply_filter_visibility(search_text)
+            # Filter mode: use the pre-computed filtered_games from optimized search
+            # This means we already have the exact games to show
+            self._update_search_results()
         else:
             # Full update: sorting, removal/addition, reorganization
             games_list = self.filtered_games if self.filtered_games else self.games
@@ -368,8 +377,73 @@ class GameLibraryManager:
 
         self.is_filtering = False  # Reset flag in any case
 
+    def _update_search_results(self):
+        """Update the grid with pre-computed search results."""
+        if self.gamesListLayout is None or self.gamesListWidget is None:
+            return
+
+        # Batch layout updates
+        self.gamesListWidget.setUpdatesEnabled(False)
+        if self.gamesListLayout is not None:
+            self.gamesListLayout.setEnabled(False)  # Disable layout during batch
+
+        try:
+            # Create set of keys for current filtered games for fast lookup
+            filtered_keys = {(game[0], game[4]) for game in self.filtered_games}  # (name, exec_line)
+
+            # Process existing cards: show cards that are in filtered results, hide others
+            cards_to_hide = []
+            for card_key, card in self.game_card_cache.items():
+                if card_key in filtered_keys:
+                    # Card should be visible
+                    if not card.isVisible():
+                        card.setVisible(True)
+                else:
+                    # Card should be hidden
+                    if card.isVisible():
+                        card.setVisible(False)
+                        cards_to_hide.append(card_key)
+
+            # Now add any missing cards that are in filtered results but not in cache
+            cards_to_add = []
+            for game_data in self.filtered_games:
+                game_name = game_data[0]
+                exec_line = game_data[4]
+                game_key = (game_name, exec_line)
+
+                if game_key not in self.game_card_cache:
+                    if self.context_menu_manager is None:
+                        continue
+
+                    card = self._create_game_card(game_data)
+                    self.game_card_cache[game_key] = card
+                    card.setVisible(True)  # New cards should be visible
+                    cards_to_add.append((game_key, card))
+
+            # Add new cards to layout
+            for _game_key, card in cards_to_add:
+                self.gamesListLayout.addWidget(card)
+
+            # Remove cards that are no longer needed (if any)
+            # Note: we're not removing them completely as they might be needed later
+            # Instead, we just hide them and they'll be reused if needed
+
+        finally:
+            if self.gamesListLayout is not None:
+                self.gamesListLayout.setEnabled(True)
+            self.gamesListWidget.setUpdatesEnabled(True)
+            if self.gamesListLayout is not None:
+                self.gamesListLayout.update()
+            self.gamesListWidget.updateGeometry()
+            self.main_window._last_card_width = self.card_width
+
+            self.force_update_cards_library()
+
     def _apply_filter_visibility(self, search_text: str):
         """Applies visibility to cards based on search, without changing the layout."""
+        # This method is used for simple substring matching
+        # For the new optimized search, we'll use a different approach in update_game_grid
+        # when is_filter=True
         visible_count = 0
         for game_key, card in self.game_card_cache.items():
             game_name = card.name  # Assume GameCard has 'name' attribute
@@ -446,9 +520,38 @@ class GameLibraryManager:
         """Sets the games list and updates the filtered games."""
         self.games = games
         self.filtered_games = self.games
+
+        # Build search indices for fast searching
+        self._build_search_indices(games)
+
         self.dirty = True  # Full resort needed
         self.update_game_grid()
 
+    def _build_search_indices(self, games: list[tuple]):
+        """Build search indices for fast searching."""
+        # Prepare items for indexing: (search_key, game_data)
+        # We'll index by game name (index 0) and potentially other fields
+        items = []
+        for game in games:
+            # game is a tuple: (name, description, cover, appid, exec_line, controller_support,
+            #                   last_launch, formatted_playtime, protondb_tier, anticheat_status,
+            #                   last_played_timestamp, playtime_seconds, game_source)
+            name = str(game[0]).lower() if game[0] else ""
+            description = str(game[1]).lower() if game[1] else ""
+
+            # Create multiple search entries for better matching
+            items.append((name, game))  # Exact name
+            # Add other searchable fields if needed
+            if description:
+                items.append((description, game))
+
+            # Also add individual words from the name for partial matching
+            for word in name.split():
+                if len(word) > 2:  # Only index words longer than 2 characters
+                    items.append((word, game))
+
+        self.search_optimizer.build_indices(items)
+
     def add_game_incremental(self, game_data: tuple):
         """Add a single game without full reload."""
         self.games.append(game_data)
@@ -472,4 +575,54 @@ class GameLibraryManager:
 
     def filter_games_delayed(self):
         """Filters games based on search text and updates the grid."""
-        self.update_game_grid(is_filter=True)
+        search_text = self.main_window.searchEdit.text().strip().lower()
+
+        if not search_text:
+            # If search is empty, show all games
+            self.filtered_games = self.games
+            self.update_game_grid(is_filter=True)
+        else:
+            # Use the optimized search
+            self._perform_search(search_text)
+
+    def _perform_search(self, search_text: str):
+        """Perform the actual search using optimized search algorithms."""
+        if not search_text:
+            self.filtered_games = self.games
+            self.update_game_grid(is_filter=True)
+            return
+
+        # Use exact search first
+        exact_result = self.search_optimizer.exact_search(search_text)
+        if exact_result:
+            # If exact match found, show only that game
+            self.filtered_games = [exact_result]
+            self.update_game_grid(is_filter=True)
+            return
+
+        # Try prefix search
+        prefix_results = self.search_optimizer.prefix_search(search_text)
+        if prefix_results:
+            # Get the actual game data from the prefix matches
+            filtered_games = []
+            for _match_text, game_data in prefix_results:
+                if game_data not in filtered_games:  # Avoid duplicates
+                    filtered_games.append(game_data)
+            self.filtered_games = filtered_games
+            self.update_game_grid(is_filter=True)
+            return
+
+        # Finally, try fuzzy search
+        fuzzy_results = self.search_optimizer.fuzzy_search(search_text, limit=20, min_score=60.0)
+        if fuzzy_results:
+            # Get the actual game data from the fuzzy matches
+            filtered_games = []
+            for _match_text, game_data, _score in fuzzy_results:
+                if game_data not in filtered_games:  # Avoid duplicates
+                    filtered_games.append(game_data)
+            self.filtered_games = filtered_games
+            self.update_game_grid(is_filter=True)
+        else:
+            # If no results found, show empty list
+            self.filtered_games = []
+            self.update_game_grid(is_filter=True)

portprotonqt/main_window.py

@@ -998,7 +998,7 @@ class MainWindow(QMainWindow):
         self.searchEdit.textChanged.connect(self.startSearchDebounce)
         self.searchDebounceTimer = QTimer(self)
         self.searchDebounceTimer.setSingleShot(True)
-        self.searchDebounceTimer.setInterval(300)
+        self.searchDebounceTimer.setInterval(150)  # Reduced debounce time for better responsiveness
         self.searchDebounceTimer.timeout.connect(self.on_search_changed)
 
         layout.addWidget(self.searchEdit)

portprotonqt/search_utils.py

@@ -0,0 +1,379 @@
+"""
+Utility module for search optimizations including Trie, hash tables, and fuzzy matching.
+"""
+import concurrent.futures
+import threading
+from collections.abc import Callable
+from typing import Any
+from rapidfuzz import fuzz
+from threading import Lock
+from portprotonqt.logger import get_logger
+from PySide6.QtCore import QThread, QRunnable, Signal, QObject, QTimer
+import requests
+
+logger = get_logger(__name__)
+
+class TrieNode:
+    """Node in the Trie data structure."""
+    def __init__(self):
+        self.children = {}
+        self.is_end_word = False
+        self.payload = None  # Store the original data in leaf nodes
+
+class Trie:
+    """Trie data structure for efficient prefix-based searching."""
+    def __init__(self):
+        self.root = TrieNode()
+        self._lock = Lock()  # Thread safety for concurrent access
+
+    def insert(self, key: str, payload: Any):
+        """Insert a key with payload into the Trie."""
+        with self._lock:
+            node = self.root
+            for char in key.lower():
+                if char not in node.children:
+                    node.children[char] = TrieNode()
+                node = node.children[char]
+            node.is_end_word = True
+            node.payload = payload
+
+    def search_prefix(self, prefix: str) -> list[tuple[str, Any]]:
+        """Find all entries with the given prefix."""
+        with self._lock:
+            node = self.root
+            for char in prefix.lower():
+                if char not in node.children:
+                    return []
+                node = node.children[char]
+
+            results = []
+            self._collect_all(node, prefix.lower(), results)
+            return results
+
+    def _collect_all(self, node: TrieNode, current_prefix: str, results: list[tuple[str, Any]]):
+        """Collect all entries from the current node."""
+        if node.is_end_word:
+            results.append((current_prefix, node.payload))
+
+        for char, child_node in node.children.items():
+            self._collect_all(child_node, current_prefix + char, results)
+
+class FuzzySearchIndex:
+    """Index for fuzzy string matching with rapidfuzz."""
+    def __init__(self, items: list[tuple[str, Any]] | None = None):
+        self.items: list[tuple[str, Any]] = items or []
+        self.normalized_items: list[tuple[str, Any]] = []
+        self._lock = Lock()
+        self._build_normalized_index()
+
+    def _build_normalized_index(self):
+        """Build a normalized index for fuzzy matching."""
+        with self._lock:
+            self.normalized_items = [(self._normalize(item[0]), item[1]) for item in self.items]
+
+    def _normalize(self, s: str) -> str:
+        """Normalize string for fuzzy matching."""
+        s = s.lower()
+        for ch in ["™", "®"]:
+            s = s.replace(ch, "")
+        for ch in ["-", ":", ","]:
+            s = s.replace(ch, " ")
+        s = " ".join(s.split())
+        for suffix in ["bin", "app"]:
+            if s.endswith(suffix):
+                s = s[:-len(suffix)].strip()
+        keywords_to_remove = {"ultimate", "edition", "definitive", "complete", "remastered"}
+        words = s.split()
+        filtered_words = [word for word in words if word not in keywords_to_remove]
+        return " ".join(filtered_words)
+
+    def fuzzy_search(self, query: str, limit: int = 5, min_score: float = 60.0) -> list[tuple[str, Any, float]]:
+        """Perform fuzzy search using rapidfuzz."""
+        with self._lock:
+            if not query or not self.normalized_items:
+                return []
+
+            query_normalized = self._normalize(query)
+            results = []
+
+            for i, (item_text, item_data) in enumerate(self.normalized_items):
+                score = fuzz.ratio(query_normalized, item_text)
+                if score >= min_score:
+                    results.append((self.items[i][0], item_data, score))
+
+            # Sort by score descending
+            results.sort(key=lambda x: x[2], reverse=True)
+            return results[:limit]
+
+class SearchOptimizer:
+    """Main search optimization class combining multiple approaches."""
+    def __init__(self):
+        self.hash_index: dict[str, Any] = {}
+        self.trie_index = Trie()
+        self.fuzzy_index = None
+        self._lock = Lock()
+
+    def build_indices(self, items: list[tuple[str, Any]]):
+        """Build all search indices from items."""
+        with self._lock:
+            self.hash_index = {item[0].lower(): item[1] for item in items}
+            self.trie_index = Trie()
+            for key, value in self.hash_index.items():
+                self.trie_index.insert(key, value)
+            self.fuzzy_index = FuzzySearchIndex(items)
+
+    def exact_search(self, key: str) -> Any | None:
+        """Perform exact hash-based lookup."""
+        with self._lock:
+            return self.hash_index.get(key.lower())
+
+    def prefix_search(self, prefix: str) -> list[tuple[str, Any]]:
+        """Perform prefix search using Trie."""
+        with self._lock:
+            return self.trie_index.search_prefix(prefix)
+
+    def fuzzy_search(self, query: str, limit: int = 5, min_score: float = 60.0) -> list[tuple[str, Any, float]]:
+        """Perform fuzzy search."""
+        if self.fuzzy_index:
+            return self.fuzzy_index.fuzzy_search(query, limit, min_score)
+        return []
+
+
+class RequestRunnable(QRunnable):
+    """Runnable for executing HTTP requests in a thread."""
+
+    def __init__(self, method: str, url: str, on_success=None, on_error=None, **kwargs):
+        super().__init__()
+        self.method = method
+        self.url = url
+        self.kwargs = kwargs
+        self.result = None
+        self.error = None
+        self.on_success: Callable | None = on_success
+        self.on_error: Callable | None = on_error
+
+    def run(self):
+        try:
+            if self.method.lower() == 'get':
+                self.result = requests.get(self.url, **self.kwargs)
+            elif self.method.lower() == 'post':
+                self.result = requests.post(self.url, **self.kwargs)
+            else:
+                raise ValueError(f"Unsupported HTTP method: {self.method}")
+
+            # Execute success callback if provided
+            if self.on_success is not None:
+                success_callback = self.on_success  # Capture the callback
+                def success_handler():
+                    if success_callback is not None:  # Re-check to satisfy Pyright
+                        success_callback(self.result)
+                QTimer.singleShot(0, success_handler)
+        except Exception as e:
+            self.error = e
+            # Execute error callback if provided
+            if self.on_error is not None:
+                error_callback = self.on_error  # Capture the callback
+                captured_error = e  # Capture the exception
+                def error_handler():
+                    error_callback(captured_error)
+                QTimer.singleShot(0, error_handler)
+
+
+def run_request_in_thread(method: str, url: str, on_success: Callable | None = None, on_error: Callable | None = None, **kwargs):
+    """Run HTTP request in a separate thread using Qt's thread system."""
+    runnable = RequestRunnable(method, url, on_success=on_success, on_error=on_error, **kwargs)
+
+    # Use QThreadPool to execute the runnable
+    from PySide6.QtCore import QThreadPool
+    thread_pool = QThreadPool.globalInstance()
+    thread_pool.start(runnable)
+
+    return runnable  # Return the runnable to allow for potential cancellation if needed
+
+
+def run_function_in_thread(func, *args, on_success: Callable | None = None, on_error: Callable | None = None, **kwargs):
+    """Run a function in a separate thread."""
+    def execute():
+        try:
+            result = func(*args, **kwargs)
+            if on_success:
+                on_success(result)
+        except Exception as e:
+            if on_error:
+                on_error(e)
+
+    thread = threading.Thread(target=execute)
+    thread.daemon = True
+    thread.start()
+    return thread
+
+
+def run_in_thread(func, *args, **kwargs):
+    """Run a function in a separate thread."""
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        future = executor.submit(func, *args, **kwargs)
+        return future.result()
+
+
+def run_in_thread_async(func, *args, callback: Callable | None = None, **kwargs):
+    """Run a function in a separate thread asynchronously."""
+    import threading
+    def target():
+        try:
+            result = func(*args, **kwargs)
+            if callback:
+                callback(result)
+        except Exception as e:
+            if callback:
+                callback(None)  # or handle error in callback
+            logger.error(f"Error in threaded operation: {e}")
+
+    thread = threading.Thread(target=target)
+    thread.daemon = True
+    thread.start()
+    return thread
+
+
+# Threaded search implementation using QThread for performance optimization
+
+
+class ThreadedSearchWorker(QObject):
+    """
+    A threaded worker for performing search operations without blocking the UI.
+    """
+    search_started = Signal()
+    search_finished = Signal(list)
+    search_error = Signal(str)
+
+    def __init__(self):
+        super().__init__()
+        self.search_optimizer = SearchOptimizer()
+        self.games_data = []
+
+    def set_games_data(self, games_data: list):
+        """Set the games data to be searched."""
+        self.games_data = games_data
+        # Build indices from the games data (name, description, etc.)
+        items = [(game[0], game) for game in games_data]  # game[0] is the name
+        self.search_optimizer.build_indices(items)
+
+    def execute_search(self, search_text: str, search_type: str = "auto"):
+        """
+        Execute search in a separate thread.
+
+        Args:
+            search_text: Text to search for
+            search_type: Type of search ("exact", "prefix", "fuzzy", "auto")
+        """
+        try:
+            self.search_started.emit()
+            import time
+            start_time = time.time()
+
+            results = []
+
+            if search_type == "exact" or (search_type == "auto" and len(search_text) > 2):
+                exact_result = self.search_optimizer.exact_search(search_text)
+                if exact_result:
+                    results = [exact_result]
+            elif search_type == "prefix":
+                results = self.search_optimizer.prefix_search(search_text)
+            elif search_type == "fuzzy" or search_type == "auto":
+                results = self.search_optimizer.fuzzy_search(search_text, limit=20, min_score=50.0)
+            else:
+                # Auto-detect search type based on input
+                if len(search_text) < 3:
+                    results = self.search_optimizer.prefix_search(search_text)
+                else:
+                    # Try exact first, then fuzzy
+                    exact_result = self.search_optimizer.exact_search(search_text)
+                    if exact_result:
+                        results = [exact_result]
+                    else:
+                        results = self.search_optimizer.fuzzy_search(search_text, limit=20, min_score=50.0)
+
+            end_time = time.time()
+            print(f"Search completed in {end_time - start_time:.4f} seconds")
+
+            self.search_finished.emit(results)
+        except Exception as e:
+            self.search_error.emit(str(e))
+
+
+class ThreadedSearch(QThread):
+    """
+    QThread implementation for running search operations in the background.
+    """
+    search_started = Signal()
+    search_finished = Signal(list)
+    search_error = Signal(str)
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self.worker = ThreadedSearchWorker()
+        self.search_text = ""
+        self.search_type = "auto"
+        self.games_data = []
+
+        # Connect worker signals to thread signals
+        self.worker.search_started.connect(self.search_started)
+        self.worker.search_finished.connect(self.search_finished)
+        self.worker.search_error.connect(self.search_error)
+
+    def set_search_params(self, search_text: str, games_data: list, search_type: str = "auto"):
+        """Set parameters for the search operation."""
+        self.search_text = search_text
+        self.games_data = games_data
+        self.search_type = search_type
+
+    def set_games_data(self, games_data: list):
+        """Set the games data to be searched."""
+        self.games_data = games_data
+        self.worker.set_games_data(games_data)
+
+    def run(self):
+        """Run the search operation in the thread."""
+        self.worker.execute_search(self.search_text, self.search_type)
+
+
+class SearchThreadPool:
+    """
+    A simple thread pool for managing multiple search operations.
+    """
+    def __init__(self, max_threads: int = 3):
+        self.max_threads = max_threads
+        self.active_threads = []
+        self.thread_queue = []
+
+    def submit_search(self, search_text: str, games_data: list, search_type: str = "auto",
+                     on_start: Callable | None = None, on_finish: Callable | None = None, on_error: Callable | None = None):
+        """
+        Submit a search operation to the pool.
+
+        Args:
+            search_text: Text to search for
+            games_data: List of game data tuples to search in
+            search_type: Type of search ("exact", "prefix", "fuzzy", "auto")
+            on_start: Callback when search starts
+            on_finish: Callback when search finishes (receives results)
+            on_error: Callback when search errors (receives error message)
+        """
+        search_thread = ThreadedSearch()
+        search_thread.set_search_params(search_text, games_data, search_type)
+
+        # Connect callbacks if provided
+        if on_start:
+            search_thread.search_started.connect(on_start)
+        if on_finish:
+            search_thread.search_finished.connect(on_finish)
+        if on_error:
+            search_thread.search_error.connect(on_error)
+
+        # Start the thread
+        search_thread.start()
+        self.active_threads.append(search_thread)
+
+        # Clean up finished threads
+        self.active_threads = [thread for thread in self.active_threads if thread.isRunning()]
+
+        return search_thread