fix(dev-scripts): parse all topics from linux-gaming

Signed-off-by: Boris Yumankulov <boria138@altlinux.org>

dev-scripts/get_id.py

@@ -5,12 +5,19 @@ import json
 import asyncio
 import aiohttp
 import tarfile
+import ssl
 
 # Получаем ключи и данные из переменных окружения
 STEAM_KEY = os.environ.get('STEAM_KEY')
 LINUX_GAMING_API_KEY = os.environ.get('LINUX_GAMING_API_KEY')
 LINUX_GAMING_API_USERNAME = os.environ.get('LINUX_GAMING_API_USERNAME')
 
+# Флаги для включения/отключения источников
+ENABLE_STEAM = os.environ.get('ENABLE_STEAM', 'true').lower() == 'true'
+ENABLE_ANTICHEAT = os.environ.get('ENABLE_ANTICHEAT', 'true').lower() == 'true'
+ENABLE_LINUX_GAMING = os.environ.get('ENABLE_LINUX_GAMING', 'true').lower() == 'true'
+DEBUG_MODE = os.environ.get('DEBUG_MODE', 'false').lower() == 'true'
+
 # Конфигурация API
 STEAM_BASE_URL = "https://api.steampowered.com/IStoreService/GetAppList/v1/?"
 LINUX_GAMING_BASE_URL = "https://linux-gaming.ru"
@@ -21,6 +28,10 @@ LINUX_GAMING_HEADERS = {
     "Api-Username": LINUX_GAMING_API_USERNAME
 }
 
+# Отключаем предупреждения об SSL в дебаг-режиме
+if DEBUG_MODE:
+    print("DEBUG_MODE enabled: SSL verification is disabled (insecure, use for debugging only).")
+
 def normalize_name(s):
     """
     Приведение строки к нормальному виду:
@@ -69,7 +80,7 @@ async def get_app_list(session, last_appid, endpoint):
     url = endpoint
     if last_appid:
         url = f"{url}&last_appid={last_appid}"
-    async with session.get(url) as response:
+    async with session.get(url, verify_ssl=not DEBUG_MODE) as response:
         response.raise_for_status()
         return await response.json()
 
@@ -79,7 +90,7 @@ async def fetch_games_json(session):
     """
     url = "https://raw.githubusercontent.com/AreWeAntiCheatYet/AreWeAntiCheatYet/HEAD/games.json"
     try:
-        async with session.get(url) as response:
+        async with session.get(url, verify_ssl=not DEBUG_MODE) as response:
             response.raise_for_status()
             text = await response.text()
             data = json.loads(text)
@@ -89,52 +100,130 @@ async def fetch_games_json(session):
         return []
 
 async def get_linux_gaming_topics(session, category_slug):
-    """
-    Получает все темы из указанной категории linux-gaming.ru.
-    Сохраняет только нормализованное название (normalized_title) и slug.
-    """
     page = 0
     all_topics = []
+    max_pages = 100
 
-    while True:
-        page += 1
-        url = f"{LINUX_GAMING_BASE_URL}/c/{category_slug}/l/latest.json?page={page}"
-        try:
-            async with session.get(url, headers=LINUX_GAMING_HEADERS) as response:
-                response.raise_for_status()
-                data = await response.json()
-                topics = data.get("topic_list", {}).get("topics", [])
-                if not topics:
+    while page < max_pages:
+        # Пробуем несколько вариантов URL
+        urls_to_try = [
+            f"{LINUX_GAMING_BASE_URL}/c/{category_slug}/5/l/latest.json",  # с id категории
+            f"{LINUX_GAMING_BASE_URL}/c/{category_slug}/l/latest.json",     # только slug
+            f"{LINUX_GAMING_BASE_URL}/c/5/l/latest.json",                  # только id
+            f"{LINUX_GAMING_BASE_URL}/latest.json"                         # все темы
+        ]
+
+        success = False
+        data = None
+
+        for url in urls_to_try:
+            try:
+                # Добавляем параметры пагинации
+                params = {
+                    'page': page,
+                    'order': 'default'
+                }
+
+                async with session.get(url, headers=LINUX_GAMING_HEADERS,
+                                     params=params, verify_ssl=not DEBUG_MODE) as response:
+                    if response.status == 429:
+                        print(f"Слишком много запросов на странице {page}, ожидание...")
+                        await asyncio.sleep(5)
+                        continue
+
+                    if response.status == 404:
+                        if DEBUG_MODE:
+                            print(f"URL не найден: {url}")
+                        continue
+
+                    response.raise_for_status()
+                    data = await response.json()
+
+                    # Проверяем структуру ответа
+                    topic_list = data.get("topic_list", {})
+                    topics = topic_list.get("topics", [])
+
+                    if not topics:
+                        if page == 0:
+                            if DEBUG_MODE:
+                                print(f"Нет тем в URL: {url}")
+                            continue
+                        else:
+                            print(f"Страница {page} пуста, завершаем пагинацию.")
+                            return all_topics
+
+                    if DEBUG_MODE and page == 0:
+                        print(f"Успешно подключились к URL: {url}")
+
+                    success = True
                     break
-                for topic in topics:
-                    all_topics.append({
-                        "normalized_title": normalize_name(topic["title"]),
-                        "slug": topic["slug"]
-                    })
-                print(f"Обработано {len(topics)} тем на странице {page}, всего: {len(all_topics)}.")
-        except Exception as error:
-            print(f"Ошибка получения тем для страницы {page}: {error}")
+
+            except Exception as e:
+                if DEBUG_MODE:
+                    print(f"Ошибка с URL {url}: {e}")
+                continue
+
+        if not success:
+            print(f"Не удалось загрузить страницу {page}")
             break
+
+        # Обрабатываем темы (этот блок должен быть внутри основного цикла)
+        try:
+            topic_list = data.get("topic_list", {})
+            topics = topic_list.get("topics", [])
+
+            page_topics_added = 0
+            for topic in topics:
+                slug = topic["slug"]
+
+                # Пропускаем тему описания категории
+                if slug is None or slug == "opisanie-kategorii-portprotondb":
+                    if DEBUG_MODE:
+                        print(f"Пропущена тема описания категории")
+                    continue
+
+                normalized_title = normalize_name(topic["title"])
+
+                # Добавляем только валидные темы
+                all_topics.append({
+                    "normalized_title": normalized_title,
+                    "slug": slug,
+                })
+                page_topics_added += 1
+
+                if DEBUG_MODE and page_topics_added <= 3:  # Показываем первые 3 темы
+                    print(f"Добавлена тема: {normalized_title} (slug: {slug}")
+
+            print(f"Обработано {len(topics)} тем на странице {page}, добавлено: {page_topics_added}, всего: {len(all_topics)}.")
+
+            # Проверяем, есть ли еще страницы
+            more_topics_url = topic_list.get("more_topics_url")
+            if not more_topics_url:
+                print("Больше тем нет, завершаем пагинацию.")
+                break
+
+            page += 1
+
+            # Добавляем небольшую задержку между запросами
+            await asyncio.sleep(0.5)
+
+        except Exception as e:
+            print(f"Ошибка при обработке тем на странице {page}: {e}")
+            break
+
+    if not all_topics:
+        print("Предупреждение: не удалось получить ни одной темы из linux-gaming.ru.")
+    else:
+        print(f"Всего получено {len(all_topics)} тем из категории {category_slug}")
+
     return all_topics
 
+
 async def request_data():
     """
     Получает данные из Steam, AreWeAntiCheatYet и linux-gaming.ru,
     обрабатывает их и сохраняет в JSON-файлы и tar.xz архивы.
     """
-    # Параметры запроса для Steam
-    game_param = "&include_games=true"
-    dlc_param = "&include_dlc=false"
-    software_param = "&include_software=false"
-    videos_param = "&include_videos=false"
-    hardware_param = "&include_hardware=false"
-
-    endpoint = (
-        f"{STEAM_BASE_URL}key={STEAM_KEY}"
-        f"{game_param}{dlc_param}{software_param}{videos_param}{hardware_param}"
-        f"&max_results=50000"
-    )
-
     output_json = []
     total_parsed = 0
     linux_gaming_topics = []
@@ -143,26 +232,48 @@ async def request_data():
     try:
         async with aiohttp.ClientSession() as session:
             # Загружаем данные Steam
-            have_more_results = True
-            last_appid_val = None
-            while have_more_results:
-                app_list = await get_app_list(session, last_appid_val, endpoint)
-                apps = app_list['response']['apps']
-                apps = process_steam_apps(apps)
-                output_json.extend(apps)
-                total_parsed += len(apps)
-                have_more_results = app_list['response'].get('have_more_results', False)
-                last_appid_val = app_list['response'].get('last_appid')
-                print(f"Обработано {len(apps)} игр Steam, всего: {total_parsed}.")
+            if ENABLE_STEAM:
+                # Параметры запроса для Steam
+                game_param = "&include_games=true"
+                dlc_param = "&include_dlc=false"
+                software_param = "&include_software=false"
+                videos_param = "&include_videos=false"
+                hardware_param = "&include_hardware=false"
+
+                endpoint = (
+                    f"{STEAM_BASE_URL}key={STEAM_KEY}"
+                    f"{game_param}{dlc_param}{software_param}{videos_param}{hardware_param}"
+                    f"&max_results=50000"
+                )
+
+                have_more_results = True
+                last_appid_val = None
+                while have_more_results:
+                    app_list = await get_app_list(session, last_appid_val, endpoint)
+                    apps = app_list['response']['apps']
+                    apps = process_steam_apps(apps)
+                    output_json.extend(apps)
+                    total_parsed += len(apps)
+                    have_more_results = app_list['response'].get('have_more_results', False)
+                    last_appid_val = app_list['response'].get('last_appid')
+                    print(f"Обработано {len(apps)} игр Steam, всего: {total_parsed}.")
+            else:
+                print("Пропущена загрузка данных Steam (ENABLE_STEAM=false).")
 
             # Загружаем данные AreWeAntiCheatYet
-            anticheat_games = await fetch_games_json(session)
+            if ENABLE_ANTICHEAT:
+                anticheat_games = await fetch_games_json(session)
+            else:
+                print("Пропущена загрузка данных AreWeAntiCheatYet (ENABLE_ANTICHEAT=false).")
 
             # Загружаем данные linux-gaming.ru
-            if LINUX_GAMING_API_KEY and LINUX_GAMING_API_USERNAME:
-                linux_gaming_topics = await get_linux_gaming_topics(session, CATEGORY_LINUX_GAMING)
+            if ENABLE_LINUX_GAMING:
+                if LINUX_GAMING_API_KEY and LINUX_GAMING_API_USERNAME:
+                    linux_gaming_topics = await get_linux_gaming_topics(session, CATEGORY_LINUX_GAMING)
+                else:
+                    print("Предупреждение: LINUX_GAMING_API_KEY или LINUX_GAMING_API_USERNAME не установлены.")
             else:
-                print("Предупреждение: LINUX_GAMING_API_KEY или LINUX_GAMING_API_USERNAME не установлены.")
+                print("Пропущена загрузка данных linux-gaming.ru (ENABLE_LINUX_GAMING=false).")
 
     except Exception as error:
         print(f"Ошибка получения данных: {error}")
@@ -173,55 +284,55 @@ async def request_data():
     os.makedirs(data_dir, exist_ok=True)
 
     # Сохранение данных Steam
-    output_json_full = os.path.join(data_dir, f"{CATEGORY_STEAM}_appid.json")
-    output_json_min = os.path.join(data_dir, f"{CATEGORY_STEAM}_appid_min.json")
-    with open(output_json_full, "w", encoding="utf-8") as f:
-        json.dump(output_json, f, ensure_ascii=False, indent=2)
-    with open(output_json_min, "w", encoding="utf-8") as f:
-        json.dump(output_json, f, ensure_ascii=False, separators=(',',':'))
+    if ENABLE_STEAM and output_json:
+        output_json_full = os.path.join(data_dir, f"{CATEGORY_STEAM}_appid.json")
+        output_json_min = os.path.join(data_dir, f"{CATEGORY_STEAM}_appid_min.json")
+        with open(output_json_full, "w", encoding="utf-8") as f:
+            json.dump(output_json, f, ensure_ascii=False, indent=2)
+        with open(output_json_min, "w", encoding="utf-8") as f:
+            json.dump(output_json, f, ensure_ascii=False, separators=(',',':'))
+
+        # Упаковка минифицированного JSON Steam в tar.xz архив
+        steam_archive_path = os.path.join(data_dir, f"{CATEGORY_STEAM}_appid.tar.xz")
+        try:
+            with tarfile.open(steam_archive_path, "w:xz", preset=9) as tar:
+                tar.add(output_json_min, arcname=os.path.basename(output_json_min))
+            print(f"Упаковано минифицированное JSON Steam в архив: {steam_archive_path}")
+            os.remove(output_json_min)
+        except Exception as e:
+            print(f"Ошибка при упаковке архива Steam: {e}")
+            return False
 
     # Сохранение данных AreWeAntiCheatYet
-    anticheat_json_full = os.path.join(data_dir, "anticheat_games.json")
-    anticheat_json_min = os.path.join(data_dir, "anticheat_games_min.json")
-    with open(anticheat_json_full, "w", encoding="utf-8") as f:
-        json.dump(anticheat_games, f, ensure_ascii=False, indent=2)
-    with open(anticheat_json_min, "w", encoding="utf-8") as f:
-        json.dump(anticheat_games, f, ensure_ascii=False, separators=(',',':'))
+    if ENABLE_ANTICHEAT and anticheat_games:
+        anticheat_json_full = os.path.join(data_dir, "anticheat_games.json")
+        anticheat_json_min = os.path.join(data_dir, "anticheat_games_min.json")
+        with open(anticheat_json_full, "w", encoding="utf-8") as f:
+            json.dump(anticheat_games, f, ensure_ascii=False, indent=2)
+        with open(anticheat_json_min, "w", encoding="utf-8") as f:
+            json.dump(anticheat_games, f, ensure_ascii=False, separators=(',',':'))
+
+        # Упаковка минифицированного JSON AreWeAntiCheatYet в tar.xz архив
+        anticheat_archive_path = os.path.join(data_dir, "anticheat_games.tar.xz")
+        try:
+            with tarfile.open(anticheat_archive_path, "w:xz", preset=9) as tar:
+                tar.add(anticheat_json_min, arcname=os.path.basename(anticheat_json_min))
+            print(f"Упаковано минифицированное JSON AreWeAntiCheatYet в архив: {anticheat_archive_path}")
+            os.remove(anticheat_json_min)
+        except Exception as e:
+            print(f"Ошибка при упаковке архива AreWeAntiCheatYet: {e}")
+            return False
 
     # Сохранение данных linux-gaming.ru
-    linux_gaming_json_full = os.path.join(data_dir, "linux_gaming_topics.json")
-    linux_gaming_json_min = os.path.join(data_dir, "linux_gaming_topics_min.json")
-    if linux_gaming_topics:
+    if ENABLE_LINUX_GAMING and linux_gaming_topics:
+        linux_gaming_json_full = os.path.join(data_dir, "linux_gaming_topics.json")
+        linux_gaming_json_min = os.path.join(data_dir, "linux_gaming_topics_min.json")
         with open(linux_gaming_json_full, "w", encoding="utf-8") as f:
             json.dump(linux_gaming_topics, f, ensure_ascii=False, indent=2)
         with open(linux_gaming_json_min, "w", encoding="utf-8") as f:
             json.dump(linux_gaming_topics, f, ensure_ascii=False, separators=(',',':'))
 
-    # Упаковка минифицированных JSON в tar.xz архивы
-    # Архив для Steam
-    steam_archive_path = os.path.join(data_dir, f"{CATEGORY_STEAM}_appid.tar.xz")
-    try:
-        with tarfile.open(steam_archive_path, "w:xz", preset=9) as tar:
-            tar.add(output_json_min, arcname=os.path.basename(output_json_min))
-        print(f"Упаковано минифицированное JSON Steam в архив: {steam_archive_path}")
-        os.remove(output_json_min)
-    except Exception as e:
-        print(f"Ошибка при упаковке архива Steam: {e}")
-        return False
-
-    # Архив для AreWeAntiCheatYet
-    anticheat_archive_path = os.path.join(data_dir, "anticheat_games.tar.xz")
-    try:
-        with tarfile.open(anticheat_archive_path, "w:xz", preset=9) as tar:
-            tar.add(anticheat_json_min, arcname=os.path.basename(anticheat_json_min))
-        print(f"Упаковано минифицированное JSON AreWeAntiCheatYet в архив: {anticheat_archive_path}")
-        os.remove(anticheat_json_min)
-    except Exception as e:
-        print(f"Ошибка при упаковке архива AreWeAntiCheatYet: {e}")
-        return False
-
-    # Архив для linux-gaming.ru
-    if linux_gaming_topics:
+        # Упаковка минифицированного JSON linux-gaming.ru в tar.xz архив
         linux_gaming_archive_path = os.path.join(data_dir, "linux_gaming_topics.tar.xz")
         try:
             with tarfile.open(linux_gaming_archive_path, "w:xz", preset=9) as tar: