.gitignore

@@ -1,7 +1,3 @@
 .env
 *.log
 **/__pycache__/
-/web/static/*.mp4
-/celerybeat-schedule.bak
-/celerybeat-schedule.dat
-/celerybeat-schedule.dir

Dockerfile

@@ -1,6 +0,0 @@
-FROM python:3.12-slim
-WORKDIR /app
-COPY . /app
-RUN pip install --no-cache-dir -r requirements.txt
-EXPOSE 5000
-CMD ["python", "web/app.py"]

README.md

@@ -1,68 +1,10 @@
-
-# Tempora
-
 Новицкий Глеб Павлович
 
-Терехов Павел Сергеевич
-
-Система мониторинга утечек данных предназначена для выявления инцидентов, связанных с публикацией конфиденциальной информации в открытых источниках (форумы, даркнет, Telegram-каналы). Система осуществляет непрерывный сбор, анализ и визуализацию упоминаний, связанных с потенциальными утечками.
-
-Проект разработан в рамках учебной практики студентом направления «Информационная безопасность автоматизированных систем».
 ```
-├── Driver/                         Основной компонент для работы с веб-ресурсами
+ ____________  ______  ____  ___  ___ 
-│   ├── chrome_proxy.py            Работа с Chrome через undetected-chromedriver + прокси
+/_  __/ __/  |/  / _ \/ __ \/ _ \/ _ |
-│   └── driver_creator.py          Создание и настройка экземпляра драйвера
+ / / / _// /|_/ / ___/ /_/ / , _/ __ |
-├── Tasks/                         Модуль задач парсинга
+/_/ /___/_/  /_/_/   \____/_/|_/_/ |_|
-│   ├── forum_crawler.py          Парсинг данных с форумов
+                                      
-│   └── tg_crawler.py             Парсинг Telegram-каналов
-├── Utils/                         Вспомогательные утилиты
-│   └── logg.py                   Логирование работы системы
-├── Web/                           Веб-интерфейс
-│   ├── Static/                   CSS/JS/Изображения
-│   ├── Templates/                HTML-шаблоны
-│   └── app.py                    Flask приложение
-└── Основные файлы:
-    ├── .env                      Конфигурации, секреты приложения
-    ├── .gitignore              Игнорируемые файлы для Git
-    ├── celery_app.py            Конфигурация Celery 
-    ├── celery_beat.py          Планировщик задач Celery
-    ├── initiator.py            Точка входа
-    ├── requirements.txt        Зависимости Python
-    └── README.md              Описание проекта
 ```
-### Основные функции
                                       
-* Мониторинг 3 телеграм каналов, одного форума
-
-* Визуализация данных на веб-интерфейсе
-* Централизованное логирование и уведомления об ошибках
-
-
-### Запуск проекта 
-
-pip install -r requirements.txt
-
-В отдельных окнах 
-
-python celery_beat.py
-python initiator.py
-
-### Зависимости
-
-| Библиотека                 | Назначение                                                            | Область применения                           |
-| -------------------------- | --------------------------------------------------------------------- | -------------------------------------------- |
-| `requests`                 | HTTP-клиент для отправки запросов и получения ответов                 | Парсинг, работа с API                        |
-| `setuptools`               | Инструмент для упаковки Python-проектов и управления зависимостями    | Сборка, установка пакетов                    |
-| `python-dotenv`            | Загрузка переменных окружения из файла `.env`                         | Конфигурация, безопасное хранение секретов   |
-| `tls-client`               | Кастомный HTTP-клиент с поддержкой TLS и антибот-защиты               | Парсинг сайтов с защитой                     |
-| `undetected-chromedriver`  | Модифицированный ChromeDriver, обходящий антибот-защиту               | Веб-скрапинг с защитой (Cloudflare и др.)    |
-| `selenium`                 | Управление браузером для автоматизации действий                       | Парсинг, тестирование веб-интерфейсов        |
-| `grpc_interceptor_headers` | Расширение gRPC-перехватчиков с поддержкой заголовков                 | gRPC API, метаданные                         |
-| `telethon`                 | Асинхронная библиотека для работы с Telegram API через MTProto        | Мониторинг Telegram-каналов                  |
-| `schedule`                 | Простое планирование задач в Python                                   | Локальный запуск задач по расписанию         |
-| `psycopg2-binary`          | PostgreSQL-драйвер для Python                                         | Работа с СУБД PostgreSQL                     |
-| `docker`                   | Python-интерфейс к Docker Engine API                                  | Управление контейнерами программно           |
-| `asyncio`                  | Базовая библиотека Python для асинхронного программирования           | Асинхронные задачи, неблокирующий ввод/вывод |
-| `pytz`                     | Работа с часовыми поясами                                             | Корректная работа с временем                 |
-| `flask`                    | Легковесный веб-фреймворк                                             | Веб-интерфейсы, REST API                     |
-| `apscheduler`              | Планировщик задач с расширенными возможностями (cron, interval, date) | Автоматизация периодических задач            |

celery_app.py

@@ -1,37 +0,0 @@
-from celery import Celery
-from kombu import Queue
-
-app = Celery(
-    'leak_monitor',
-    broker='memory://localhost',
-    # backend='rpc://',
-    include=['tasks.tg_crawler', 'tasks.forum_crawler']
-)
-
-app.conf.update(
-    worker_pool='solo',
-    worker_max_tasks_per_child=100,
-    task_serializer='json',
-    result_serializer='json',
-    accept_content=['json'],
-    timezone='Europe/Moscow',
-    enable_utc=True,
-)
-
-app.conf.task_queues = (
-    Queue('telegram', routing_key='telegram.#'),
-    Queue('forum', routing_key='forum.#'),
-)
-
-app.conf.beat_schedule = {
-    'monitor-telegram-channels': {
-        'task': 'tasks.tg_crawler.monitor_channels',
-        'schedule': 10.0,
-        'options': {'queue': 'telegram'}
-    },
-    'crawl-forum': {
-        'task': 'forum_crawler.crawl_forum_task',
-        'schedule': 3600.0 * 24,   # час
-        'args': ('https://thehackernews.com/search/label/data%20breach', [])
-    },
-}

celery_beat.py

@@ -1,19 +0,0 @@
-from celery_app import app
-from utils.logg import LoggerSingleton
-
-logger = LoggerSingleton.get_logger()
-
-if __name__ == '__main__':
-    logger.info("Starting Celery beat scheduler...")
-    try:
-        app.loader.import_default_modules()
-        
-        beat = app.Beat(
-            logfile=None, 
-            loglevel='info',
-            socket_timeout=30
-        )
-        beat.run()
-    except Exception as e:
-        logger.error(f"Beat failed: {e}")
-        raise

docker-compose.yaml

@@ -1,47 +0,0 @@
-version: '3.8'
-
-services:
-  web:
-    build: .
-    container_name: tg-monitor
-    ports:
-      - "5000:5000"
-    volumes:
-      - ./web:/app/web
-      - /var/run/docker.sock:/var/run/docker.sock
-      - type: bind
-        source: ./tg_nodes.log
-        target: /app/tg_nodes.log
-        read_only: true
-    environment:
-      - PYTHONUNBUFFERED=1
-    privileged: true
-    restart: unless-stopped
-    command: python web/app.py
-
-  tg_node_0:
-    build: .
-    volumes:
-      - type: bind
-        source: ./tg_nodes.log
-        target: /app/tg_nodes.log
-    environment:
-      - PYTHONUNBUFFERED=1
-    container_name: tg-node-0
-    restart: unless-stopped
-    command: python tg/tg_node_0.py
-
-  tg_node_1:
-    build: .
-    volumes:
-      - type: bind
-        source: ./tg_nodes.log
-        target: /app/tg_nodes.log
-    environment:
-      - PYTHONUNBUFFERED=1
-    container_name: tg-node-1
-    restart: unless-stopped
-    command: python tg/tg_node_1.py
-
-volumes:
-  pg_data:

driver/driver_creator.py

@@ -27,21 +27,34 @@ class DriverCreator:
         ).create_extension()
 
     def get_driver(self):
-        extension_path = self._switch_proxy()
+        '''
+        Отключает JS
+        Каждый запрос получает `сырой` html
+        '''
+        # extension_path = self._switch_proxy()
 
         options = uc.ChromeOptions()
-        options.add_argument(f"--load-extension={extension_path}") # временно
+        # options.add_argument(f"--load-extension={extension_path}") # временно
-        # options.add_argument("--headless=new")
+        options.add_argument("--headless=new")
         options.add_argument("--disable-gpu")
         options.add_argument("--disable-dev-shm-usage")
         options.add_argument("--no-sandbox")
+        options.add_argument("--disable-webgl")
+        options.add_argument("--disable-software-rasterizer")
+        # options.add_argument("--disable-extensions")
         
+        prefs = {"profile.managed_default_content_settings.javascript": 2}
+        options.experimental_options["prefs"] = prefs
         
         driver = uc.Chrome(
             options=options,
-            version_main=135,
+            version_main=132,
             # user_multi_procs=True
         )
 
+        driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
+            "source": "Object.defineProperty(navigator, 'javaEnabled', {get: () => false});"
+        })
+        driver.execute_cdp_cmd("Emulation.setScriptExecutionDisabled", {"value": True})
         
         return driver

initiator.py

@@ -1,29 +0,0 @@
-from celery_app import app
-from utils.logg import LoggerSingleton
-
-logger = LoggerSingleton.get_logger()
-
-# Брокер центральный узел, который:
-# Принимает задачи от beat (планировщика)
-# Распределяет их по очередям (telegram, forum).
-# Передаёт задачи воркерам, которые подписаны на эти очереди.
-
-def main():
-    try:
-        logger.info("Starting Celery worker...")
-        app.worker_main(
-            argv=[
-                'worker',
-                '--loglevel=debug', 
-                '--pool=solo',
-                '-Q', 'telegram',
-                '--without-heartbeat',
-                '--without-gossip',
-                # '-B'
-            ]
-        )
-    except Exception as e:
-        logger.error(f"Failed to start worker: {e}")
-
-if __name__ == '__main__':
-    main()

patcher.py

@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+# this module is part of undetected_chromedriver
+
+from distutils.version import LooseVersion
+import io
+import json
+import logging
+import os
+import pathlib
+import platform
+import random
+import re
+import shutil
+import string
+import sys
+import time
+from urllib.request import urlopen
+from urllib.request import urlretrieve
+import zipfile
+from multiprocessing import Lock
+import secrets
+
+logger = logging.getLogger(__name__)
+
+IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2"))
+
+
+class Patcher(object):
+    lock = Lock()
+    exe_name = "chromedriver%s"
+
+    platform = sys.platform
+    if platform.endswith("win32"):
+        d = "~/appdata/roaming/undetected_chromedriver"
+    elif "LAMBDA_TASK_ROOT" in os.environ:
+        d = "/tmp/undetected_chromedriver"
+    elif platform.startswith(("linux", "linux2")):
+        d = "~/.local/share/undetected_chromedriver"
+    elif platform.endswith("darwin"):
+        d = "~/Library/Application Support/undetected_chromedriver"
+    else:
+        d = "~/.undetected_chromedriver"
+    data_path = os.path.abspath(os.path.expanduser(d))
+
+    def __init__(
+        self,
+        executable_path=None,
+        force=False,
+        version_main: int = 0,
+        user_multi_procs=False,
+    ):
+        """
+        Args:
+            executable_path: None = automatic
+                             a full file path to the chromedriver executable
+            force: False
+                    terminate processes which are holding lock
+            version_main: 0 = auto
+                specify main chrome version (rounded, ex: 82)
+        """
+        self.force = force
+        self._custom_exe_path = False
+        prefix = secrets.token_hex(8)
+        self.user_multi_procs = user_multi_procs
+
+        self.is_old_chromedriver = version_main and version_main <= 114
+        # Needs to be called before self.exe_name is accessed
+        self._set_platform_name()
+
+        if not os.path.exists(self.data_path):
+            os.makedirs(self.data_path, exist_ok=True)
+
+        if not executable_path:
+            self.executable_path = os.path.join(
+                self.data_path, "_".join([prefix, self.exe_name])
+            )
+
+        if not IS_POSIX:
+            if executable_path:
+                if not executable_path[-4:] == ".exe":
+                    executable_path += ".exe"
+
+        self.zip_path = os.path.join(self.data_path, prefix)
+
+        if not executable_path:
+            if not self.user_multi_procs:
+                self.executable_path = os.path.abspath(
+                    os.path.join(".", self.executable_path)
+                )
+
+        if executable_path:
+            self._custom_exe_path = True
+            self.executable_path = executable_path
+
+        # Set the correct repository to download the Chromedriver from
+        if self.is_old_chromedriver:
+            self.url_repo = "https://chromedriver.storage.googleapis.com"
+        else:
+            self.url_repo = "https://googlechromelabs.github.io/chrome-for-testing"
+
+        self.version_main = version_main
+        self.version_full = None
+
+    def _set_platform_name(self):
+        """
+        Set the platform and exe name based on the platform undetected_chromedriver is running on
+        in order to download the correct chromedriver.
+        """
+        if self.platform.endswith("win32"):
+            self.platform_name = "win32"
+            self.exe_name %= ".exe"
+        if self.platform.endswith(("linux", "linux2")):
+            self.platform_name = "linux64"
+            self.exe_name %= ""
+        if self.platform.endswith("darwin"):
+            if self.is_old_chromedriver:
+                self.platform_name = "mac64"
+            else:
+                self.platform_name = "mac-x64"
+            self.exe_name %= ""
+
+    def auto(self, executable_path=None, force=False, version_main=None, _=None):
+        """
+
+        Args:
+            executable_path:
+            force:
+            version_main:
+
+        Returns:
+
+        """
+        p = pathlib.Path(self.data_path)
+        if self.user_multi_procs:
+            with Lock():
+                files = list(p.rglob("*chromedriver*"))
+                most_recent = max(files, key=lambda f: f.stat().st_mtime)
+                files.remove(most_recent)
+                list(map(lambda f: f.unlink(), files))
+                if self.is_binary_patched(most_recent):
+                    self.executable_path = str(most_recent)
+                    return True
+
+        if executable_path:
+            self.executable_path = executable_path
+            self._custom_exe_path = True
+
+        if self._custom_exe_path:
+            ispatched = self.is_binary_patched(self.executable_path)
+            if not ispatched:
+                return self.patch_exe()
+            else:
+                return
+
+        if version_main:
+            self.version_main = version_main
+        if force is True:
+            self.force = force
+
+        try:
+            os.unlink(self.executable_path)
+        except PermissionError:
+            if self.force:
+                self.force_kill_instances(self.executable_path)
+                return self.auto(force=not self.force)
+            try:
+                if self.is_binary_patched():
+                    # assumes already running AND patched
+                    return True
+            except PermissionError:
+                pass
+            # return False
+        except FileNotFoundError:
+            pass
+
+        release = self.fetch_release_number()
+        self.version_main = release.version[0]
+        self.version_full = release
+        self.unzip_package(self.fetch_package())
+        return self.patch()
+
+    def driver_binary_in_use(self, path: str = None) -> bool:
+        """
+        naive test to check if a found chromedriver binary is
+        currently in use
+
+        Args:
+            path: a string or PathLike object to the binary to check.
+                  if not specified, we check use this object's executable_path
+        """
+        if not path:
+            path = self.executable_path
+        p = pathlib.Path(path)
+
+        if not p.exists():
+            raise OSError("file does not exist: %s" % p)
+        try:
+            with open(p, mode="a+b") as fs:
+                exc = []
+                try:
+
+                    fs.seek(0, 0)
+                except PermissionError as e:
+                    exc.append(e)  # since some systems apprently allow seeking
+                    # we conduct another test
+                try:
+                    fs.readline()
+                except PermissionError as e:
+                    exc.append(e)
+
+                if exc:
+
+                    return True
+                return False
+            # ok safe to assume this is in use
+        except Exception as e:
+            # logger.exception("whoops ", e)
+            pass
+
+    def cleanup_unused_files(self):
+        p = pathlib.Path(self.data_path)
+        items = list(p.glob("*undetected*"))
+        for item in items:
+            try:
+                item.unlink()
+            except:
+                pass
+
+    def patch(self):
+        self.patch_exe()
+        return self.is_binary_patched()
+
+    def fetch_release_number(self):
+        """
+        Gets the latest major version available, or the latest major version of self.target_version if set explicitly.
+        :return: version string
+        :rtype: LooseVersion
+        """
+        # Endpoint for old versions of Chromedriver (114 and below)
+        if self.is_old_chromedriver:
+            path = f"/latest_release_{self.version_main}"
+            path = path.upper()
+            logger.debug("getting release number from %s" % path)
+            return LooseVersion(urlopen(self.url_repo + path).read().decode())
+
+        # Endpoint for new versions of Chromedriver (115+)
+        if not self.version_main:
+            # Fetch the latest version
+            path = "/last-known-good-versions-with-downloads.json"
+            logger.debug("getting release number from %s" % path)
+            with urlopen(self.url_repo + path) as conn:
+                response = conn.read().decode()
+
+            last_versions = json.loads(response)
+            return LooseVersion(last_versions["channels"]["Stable"]["version"])
+
+        # Fetch the latest minor version of the major version provided
+        path = "/latest-versions-per-milestone-with-downloads.json"
+        logger.debug("getting release number from %s" % path)
+        with urlopen(self.url_repo + path) as conn:
+            response = conn.read().decode()
+
+        major_versions = json.loads(response)
+        return LooseVersion(major_versions["milestones"][str(self.version_main)]["version"])
+
+    def parse_exe_version(self):
+        with io.open(self.executable_path, "rb") as f:
+            for line in iter(lambda: f.readline(), b""):
+                match = re.search(rb"platform_handle\x00content\x00([0-9.]*)", line)
+                if match:
+                    return LooseVersion(match[1].decode())
+
+    def fetch_package(self):
+        """
+        Downloads ChromeDriver from source
+
+        :return: path to downloaded file
+        """
+        zip_name = f"chromedriver_{self.platform_name}.zip"
+        if self.is_old_chromedriver:
+            download_url = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, zip_name)
+        else:
+            zip_name = zip_name.replace("_", "-", 1)
+            download_url = "https://storage.googleapis.com/chrome-for-testing-public/%s/%s/%s"
+            download_url %= (self.version_full.vstring, self.platform_name, zip_name)
+
+        logger.debug("downloading from %s" % download_url)
+        return urlretrieve(download_url)[0]
+
+    def unzip_package(self, fp):
+        """
+        Does what it says
+
+        :return: path to unpacked executable
+        """
+        exe_path = self.exe_name
+        if not self.is_old_chromedriver:
+            # The new chromedriver unzips into its own folder
+            zip_name = f"chromedriver-{self.platform_name}"
+            exe_path = os.path.join(zip_name, self.exe_name)
+
+        logger.debug("unzipping %s" % fp)
+        try:
+            os.unlink(self.zip_path)
+        except (FileNotFoundError, OSError):
+            pass
+
+        os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
+        with zipfile.ZipFile(fp, mode="r") as zf:
+            zf.extractall(self.zip_path)
+        os.rename(os.path.join(self.zip_path, exe_path), self.executable_path)
+        os.remove(fp)
+        shutil.rmtree(self.zip_path)
+        os.chmod(self.executable_path, 0o755)
+        return self.executable_path
+
+    @staticmethod
+    def force_kill_instances(exe_name):
+        """
+        kills running instances.
+        :param: executable name to kill, may be a path as well
+
+        :return: True on success else False
+        """
+        exe_name = os.path.basename(exe_name)
+        if IS_POSIX:
+            r = os.system("kill -f -9 $(pidof %s)" % exe_name)
+        else:
+            r = os.system("taskkill /f /im %s" % exe_name)
+        return not r
+
+    @staticmethod
+    def gen_random_cdc():
+        cdc = random.choices(string.ascii_letters, k=27)
+        return "".join(cdc).encode()
+
+    def is_binary_patched(self, executable_path=None):
+        executable_path = executable_path or self.executable_path
+        try:
+            with io.open(executable_path, "rb") as fh:
+                return fh.read().find(b"undetected chromedriver") != -1
+        except FileNotFoundError:
+            return False
+
+    def patch_exe(self):
+        start = time.perf_counter()
+        logger.info("patching driver executable %s" % self.executable_path)
+        with io.open(self.executable_path, "r+b") as fh:
+            content = fh.read()
+            # match_injected_codeblock = re.search(rb"{window.*;}", content)
+            match_injected_codeblock = re.search(rb"\{window\.cdc.*?;\}", content)
+            if match_injected_codeblock:
+                target_bytes = match_injected_codeblock[0]
+                new_target_bytes = (
+                    b'{console.log("undetected chromedriver 1337!")}'.ljust(
+                        len(target_bytes), b" "
+                    )
+                )
+                new_content = content.replace(target_bytes, new_target_bytes)
+                if new_content == content:
+                    logger.warning(
+                        "something went wrong patching the driver binary. could not find injection code block"
+                    )
+                else:
+                    logger.debug(
+                        "found block:\n%s\nreplacing with:\n%s"
+                        % (target_bytes, new_target_bytes)
+                    )
+                fh.seek(0)
+                fh.write(new_content)
+        logger.debug(
+            "patching took us {:.2f} seconds".format(time.perf_counter() - start)
+        )
+
+    def __repr__(self):
+        return "{0:s}({1:s})".format(
+            self.__class__.__name__,
+            self.executable_path,
+        )
+
+    def __del__(self):
+        if self._custom_exe_path:
+            # if the driver binary is specified by user
+            # we assume it is important enough to not delete it
+            return
+        else:
+            timeout = 3  # stop trying after this many seconds
+            t = time.monotonic()
+            now = lambda: time.monotonic()
+            while now() - t > timeout:
+                # we don't want to wait until the end of time
+                try:
+                    if self.user_multi_procs:
+                        break
+                    os.unlink(self.executable_path)
+                    logger.debug("successfully unlinked %s" % self.executable_path)
+                    break
+                except (OSError, RuntimeError, PermissionError):
+                    time.sleep(0.01)
+                    continue
+                except FileNotFoundError:
+                    break

requirements.txt

@@ -5,11 +5,3 @@ tls-client
 undetected-chromedriver
 selenium
 grpc_interceptor_headers
-telethon
-schedule
-psycopg2-binary
-docker
-asyncio
-pytz
-flask
-apscheduler

runner.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+TARGET_DIR=$(python -c "import undetected_chromedriver as uc; print(uc.__path__[0])" 2>/dev/null)
+
+if [ -d "$TARGET_DIR" ]; then
+    echo "patcher.py в $TARGET_DIR"
+    cp patcher.py "$TARGET_DIR"
+else
+    echo "undetected_chromedriver не найден"
+fi
+
+pip install -r requirements.txt
+
+python initiator.py

tasks/forum_crawler.py

@@ -1,97 +0,0 @@
-from bs4 import BeautifulSoup
-import os
-import psycopg2
-from dotenv import load_dotenv
-from celery import Celery
-from utils.logg import LoggerSingleton
-from driver.driver_creator import DriverCreator
-from selenium.webdriver.common.by import By
-from time import sleep
-from celery_app import app
-
-load_dotenv()
-
-logger = LoggerSingleton.get_logger()
-
-class ForumCrawler:
-    def __init__(self, forum_url, proxy_list):
-        self.forum_url = forum_url
-        self.proxy_list = proxy_list
-        self.db_config = {
-            'host': os.getenv('DB_HOST'),
-            'port': os.getenv('DB_PORT'),
-            'database': os.getenv('DB_NAME'),
-            'user': os.getenv('DB_USER'),
-            'password': os.getenv('DB_PASSWORD')
-        }
-        self.conn = None
-        self._connect()
-
-    def _connect(self):
-        try:
-            self.conn = psycopg2.connect(**self.db_config)
-            logger.info("Database connection established")
-        except Exception as e:
-            logger.error(f"Database connection error: {e}")
-            raise
-
-    def _close(self):
-        if self.conn and not self.conn.closed:
-            try:
-                self.conn.close()
-                logger.info("Database connection closed")
-            except Exception as e:
-                logger.error(f"Error closing database connection: {e}")
-
-    def _crawl_leaks(self):
-        driver_creator = DriverCreator(self.proxy_list)
-        driver = driver_creator.get_driver()
-
-        try:
-            logger.info(f"Starting forum crawl: {self.forum_url}")
-            driver.get(self.forum_url)
-            sleep(2)
-
-            posts = driver.find_elements(By.CSS_SELECTOR, 'a.story-link h2.home-title')
-            if not posts:
-                logger.info("No posts found on the page")
-                return
-            last_post = posts[0]
-            title = last_post.text.strip()
-            link = last_post.get_attribute('href')
-            post_content = f"{title} - {link}"
-
-            if 'data breach' in title.lower() or 'leak' in title.lower():
-                self._save_to_db('Hacker News', post_content)
-                logger.info(f"New leak found: {title} - {link}")
-            else:
-                logger.info("Last post is not about leaks")
-
-        except Exception as e:
-            logger.error(f"Error during forum crawling: {e}")
-            raise
-        finally:
-            driver.quit()
-            logger.info("WebDriver session closed")
-        
-    def _save_to_db(self, source, message):
-        if not self.conn or self.conn.closed:
-            self._connect()
-        try:
-            with self.conn.cursor() as cur:
-                cur.execute(
-                    "INSERT INTO leaks (resource_name, message) VALUES (%s, %s)",
-                    (source, message)
-                )
-            self.conn.commit()
-            logger.info(f"Leak from {source} saved in the database")
-        except Exception as e:
-            logger.error(f"Error writing to the database: {e}")
-
-@app.task(name='forum_crawler.crawl_forum_task')
-def crawl_forum_task(forum_url, proxy_list):
-    crawler = ForumCrawler(forum_url, proxy_list)
-    try:
-        crawler._crawl_leaks()
-    finally:
-        crawler._close()

tasks/tg_crawler.py

@@ -1,121 +0,0 @@
-import logging
-import os
-import asyncio
-import psycopg2
-from datetime import datetime
-from telethon.sync import TelegramClient
-from telethon.tl.functions.messages import GetHistoryRequest
-from dotenv import load_dotenv
-from celery import Celery
-from utils.logg import LoggerSingleton
-from celery_app import app
-
-load_dotenv()
-
-logger = LoggerSingleton.get_logger()
-
-class TelegramMonitor:
-    def __init__(self):
-        self.db_config = {
-            'host': os.getenv('DB_HOST'),
-            'port': os.getenv('DB_PORT'),
-            'database': os.getenv('DB_NAME'),
-            'user': os.getenv('DB_USER'),
-            'password': os.getenv('DB_PASSWORD')
-        }
-        self.conn = None
-        self._connect()
-
-    def _connect(self):
-        try:
-            self.conn = psycopg2.connect(**self.db_config)
-            logger.info("Database connection established")
-        except Exception as e:
-            logger.error(f"Database connection error: {e}")
-            raise
-
-    def _close(self):
-        if self.conn and not self.conn.closed:
-            try:
-                self.conn.close()
-                logger.info("Database connection closed")
-            except Exception as e:
-                logger.error(f"Error closing database connection: {e}")
-
-    def _fetch_post(self, channel_username, source_name):
-        try:
-            with TelegramClient('session', os.getenv('API_ID'), os.getenv('API_HASH')) as client:
-                entity = client.get_entity(channel_username)
-                history = client(GetHistoryRequest(
-                    peer=entity,
-                    limit=1,
-                    offset_date=None,
-                    offset_id=0,
-                    max_id=0,
-                    min_id=0,
-                    add_offset=0,
-                    hash=0
-                ))
-                if history.messages:
-                    self._save_to_db(source_name, history.messages[0].message)
-        except Exception as e:
-            logger.error(f"Error fetching post from {source_name}: {e}")
-            raise
-
-    def _save_to_db(self, source, message):
-        if not self.conn or self.conn.closed:
-            self._connect()
-            
-        try:
-            with self.conn.cursor() as cur:
-                cur.execute(
-                    "INSERT INTO leaks (resource_name, message) VALUES (%s, %s)",
-                    (source, message)
-                )
-            self.conn.commit()
-            logger.info(f"Data from {source} saved successfully")
-        except Exception as e:
-            logger.error(f"Database save error for {source}: {e}")
-            self.conn.rollback()
-            raise
-
-@app.task(bind=True, name='tasks.tg_crawler.monitor_channels')
-def monitor_channels(self):
-    channels = [
-        ('trueosint', 'trueosint'),
-        ('dataleak', 'dataleak'),
-        ('Vaultofdataleaksss', 'Vaultofdataleaksss')
-    ]
-    
-    logger.info("Starting Telegram channels monitoring")
-    
-    for channel, source in channels:
-        try:
-            monitor_channel.apply_async(
-                args=(channel, source),
-                queue='telegram',
-                retry=True,
-                retry_policy={
-                    'max_retries': 3,
-                    'interval_start': 2,
-                    'interval_step': 5,
-                    'interval_max': 20
-                }
-            )
-            logger.info(f"Sent task for channel: {channel}")
-        except Exception as e:
-            logger.error(f"Failed to send task for {channel}: {e}")
-            raise self.retry(exc=e)
-
-@app.task(bind=True, name='tasks.tg_crawler.monitor_channel')
-def monitor_channel(self, channel, source):
-    logger.info(f"Starting monitoring channel: {channel}")
-    monitor = TelegramMonitor()
-    try:
-        monitor._fetch_post(channel, source)
-        logger.info(f"Successfully monitored channel: {channel}")
-    except Exception as e:
-        logger.error(f"Channel monitoring failed for {channel}: {e}")
-        raise self.retry(exc=e)
-    finally:
-        monitor._close()

utils/logg.py

@@ -1,27 +0,0 @@
-import logging
-
-class LoggerSingleton:
-    _logger = None
-
-    @staticmethod
-    def get_logger():
-        if LoggerSingleton._logger is None:
-            LoggerSingleton._logger = logging.getLogger(__name__)
-
-            LoggerSingleton._logger.setLevel(logging.INFO)
-
-            formatter = logging.Formatter(
-                "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-                datefmt="%Y-%m-%d %H:%M:%S"
-            )
-
-            console_handler = logging.StreamHandler()
-            console_handler.setFormatter(formatter)
-            
-            file_handler = logging.FileHandler('tempora.log', encoding='utf-8')
-            file_handler.setFormatter(formatter)
-            
-            LoggerSingleton._logger.addHandler(console_handler)
-            LoggerSingleton._logger.addHandler(file_handler)
-
-        return LoggerSingleton._logger

web/app.py

@@ -1,141 +0,0 @@
-import os
-from flask import Flask, render_template
-import psycopg2
-from datetime import datetime
-import docker
-
-app = Flask(__name__)
-
-docker_client = docker.from_env()
-
-DB_CONFIG = {
-    'host': os.getenv("HOST"),
-    'port': os.getenv("PORT"),
-    'database': os.getenv("DBNAME"),
-    'user': os.getenv("USER"),
-    'password': os.getenv("PASSWORD")
-}
-
-def parse_log_line(line):
-    """Парсит строку лога в формате: [timestamp] сообщение"""
-    try:
-        line = line.strip()
-        if line.startswith('[') and ']' in line:
-            return line
-        return f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {line}"
-    except:
-        return line
-
-def get_parser_status():
-    """Получает статус парсеров из Docker"""
-    try:
-        containers = docker_client.containers.list(all=True)
-        status = {
-            'tg-node-0': False,
-            'tg-node-1': False,
-            'total': 0,
-            'active': 0,
-            'errors': 0
-        }
-        
-        for container in containers:
-            if container.name == 'tg-node-0':
-                status['tg-node-0'] = container.status == 'running'
-            elif container.name == 'tg-node-1':
-                status['tg-node-1'] = container.status == 'running'
-        
-        status['active'] = sum([status['tg-node-0'], status['tg-node-1']])
-        status['errors'] = 2 - status['active']  # Просто для примера
-        status['total'] = 2
-        
-        return status
-    except Exception as e:
-        print(f"Error getting docker status: {e}")
-        return None
-
-def get_leaks_stats():
-    """Получает статистику по утечкам из базы данных"""
-    try:
-        conn = psycopg2.connect(**DB_CONFIG)
-        cursor = conn.cursor()
-        
-        cursor.execute("SELECT COUNT(*) FROM leaks")
-        total_leaks = cursor.fetchone()[0]
-        
-        cursor.execute("""
-            SELECT DATE(created_at) as day, COUNT(*) 
-            FROM leaks 
-            WHERE created_at >= CURRENT_DATE - INTERVAL '30 days'
-            GROUP BY day 
-            ORDER BY day
-        """)
-        leaks_by_day = cursor.fetchall()
-        
-        cursor.execute("""
-            SELECT resource_name, COUNT(*) 
-            FROM leaks 
-            GROUP BY resource_name 
-            ORDER BY COUNT(*) DESC 
-            LIMIT 3
-        """)
-        leaks_by_source = cursor.fetchall()
-        
-        cursor.execute("""
-            SELECT resource_name, message, created_at 
-            FROM leaks 
-            ORDER BY created_at DESC 
-            LIMIT 10
-        """)
-        recent_leaks = cursor.fetchall()
-        
-        return {
-            'total_leaks': total_leaks,
-            'leaks_by_day': leaks_by_day,
-            'leaks_by_source': leaks_by_source,
-            'recent_leaks': recent_leaks
-        }
-    except Exception as e:
-        print(f"Database error: {e}")
-        return None
-    finally:
-        if 'conn' in locals():
-            conn.close()
-
-@app.route("/")
-@app.route("/index.html")
-def index():
-    parser_status = get_parser_status()
-    leaks_stats = get_leaks_stats()
-    
-    if not leaks_stats:
-        leaks_stats = {
-            'total_leaks': 150,
-            'leaks_by_day': [(datetime.now().date(), 5)],
-            'leaks_by_source': [('Telegram', 80), ('Форум', 50), ('Даркнет', 20)],
-            'recent_leaks': [('Telegram', 'Новая утечка данных', datetime.now())]
-        }
-    
-    return render_template(
-        "index.html",
-        parser_status=parser_status,
-        leaks_stats=leaks_stats
-    )
-
-@app.route("/logs.html")
-@app.route("/logs")
-def logs():
-    log_path = '/app/tg_nodes.log'
-    try:
-        with open(log_path, 'r', encoding='utf-8') as f:
-            lines = f.readlines()[-100:]
-            parsed_logs = [parse_log_line(line) for line in lines if line.strip()]
-    except FileNotFoundError:
-        parsed_logs = ["[ERROR] Лог-файл не найден"]
-    except Exception as e:
-        parsed_logs = [f"[ERROR] Ошибка чтения лог-файла: {str(e)}"]
-    
-    return render_template("logs.html", logs=parsed_logs)
-
-
-if __name__ == "__main__":
-    app.run(host='0.0.0.0', port=5000)

web/templates/index.html

@@ -1,424 +0,0 @@
-<!DOCTYPE html>
-<html lang="ru">
-
-<head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Система мониторинга утечек данных</title>
-  <link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css" />
-  <style>
-    body {
-      background-color: #1e1e1e;
-      color: #ffffff;
-      font-family: 'Arial', sans-serif;
-      margin: 0;
-      padding: 0;
-      display: flex;
-      transition: margin-left 0.3s ease;
-      min-height: 100vh;
-      overflow: hidden;
-    }
-
-    .sidebar {
-      width: 250px;
-      background-color: #2e2e2e;
-      padding: 20px;
-      box-shadow: 2px 0 5px rgba(0, 0, 0, 0.3);
-      transition: width 0.3s ease;
-      overflow-y: auto;
-      position: fixed;
-      top: 0;
-      left: 0;
-      bottom: 0;
-      z-index: 1;
-    }
-
-    .sidebar.collapsed {
-      width: 60px;
-    }
-
-    .sidebar.collapsed h2,
-    .sidebar.collapsed ul li a span {
-      display: none;
-    }
-
-    .sidebar h2 {
-      color: #3399FF;
-      text-align: center;
-      margin-bottom: 20px;
-      font-size: 24px;
-      transition: opacity 0.3s ease;
-    }
-
-    .sidebar ul {
-      list-style: none;
-      padding: 0;
-    }
-
-    .sidebar ul li {
-      margin: 15px 0;
-    }
-
-    .sidebar ul li a {
-      color: #ffffff;
-      text-decoration: none;
-      font-size: 16px;
-      transition: color 0.3s ease;
-      display: flex;
-      align-items: center;
-    }
-
-    .sidebar ul li a:hover {
-      color: #3399FF;
-    }
-
-    .sidebar ul li a i {
-      margin-right: 10px;
-      font-size: 20px;
-    }
-
-    .sidebar ul li a span {
-      transition: opacity 0.3s ease;
-    }
-
-    .toggle-btn {
-      position: fixed;
-      left: 10px;
-      top: 10px;
-      background-color: #3399FF;
-      border: none;
-      color: #fff;
-      padding: 10px;
-      border-radius: 5px;
-      cursor: pointer;
-      z-index: 1000;
-    }
-
-    .container {
-      flex: 1;
-      padding: 20px;
-      margin-left: 250px;
-      transition: margin-left 0.3s ease;
-      overflow-y: auto;
-      height: 100vh;
-      position: relative;
-      z-index: 0;
-    }
-
-    .container.collapsed {
-      margin-left: 60px;
-    }
-
-    h1 {
-      color: #3399FF;
-      text-align: center;
-      margin-bottom: 20px;
-    }
-
-    .grid {
-      display: grid;
-      grid-template-columns: repeat(2, 1fr);
-      gap: 20px;
-      margin-top: 20px;
-    }
-
-    .card {
-      background-color: #2e2e2e;
-      padding: 20px;
-      border-radius: 8px;
-      box-shadow: 0 0 5px rgba(0, 0, 0, 0.2);
-    }
-
-    #map {
-      height: 400px;
-      border-radius: 8px;
-      grid-column: span 2;
-    }
-
-    .screenshots {
-      display: flex;
-      overflow-x: auto;
-      gap: 10px;
-      padding: 10px;
-    }
-
-    .screenshots img {
-      max-height: 200px;
-      border-radius: 5px;
-      border: 2px solid #3399FF;
-    }
-
-    .logs {
-      background-color: #2e2e2e;
-      padding: 15px;
-      border-radius: 8px;
-      height: 200px;
-      overflow-y: auto;
-      font-family: 'Courier New', monospace;
-      font-size: 14px;
-      color: #3399FF;
-    }
-
-    .logs p {
-      margin: 5px 0;
-    }
-
-    .leaflet-marker-icon {
-      background-color: #3399FF;
-      border: 2px solid #ffffff;
-      border-radius: 50%;
-      width: 20px !important;
-      height: 20px !important;
-      box-shadow: 0 0 10px rgba(51, 153, 255, 0.5);
-    }
-
-    .leaflet-popup-content {
-      color: #1e1e1e;
-      font-size: 14px;
-    }
-
-    .leaflet-popup-content-wrapper {
-      border-radius: 8px;
-      box-shadow: 0 0 10px rgba(0, 0, 0, 0.3);
-    }
-  </style>
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css">
-</head>
-
-<body>
-  <!-- Боковое меню -->
-  <div class="sidebar" id="sidebar">
-    <h2>Меню</h2>
-    <ul>
-      <li><a href="#"><i class="fas fa-home"></i><span>Главная</span></a></li>
-      <li><a href="#"><i class="fas fa-cog"></i><span>Настройки</span></a></li>
-      <li><a href="#"><i class="fas fa-chart-line"></i><span>Отчеты</span></a></li>
-      <li><a href="#"><i class="fas fa-code"></i><span>Парсеры</span></a></li>
-      <li><a href="logs.html"><i class="fas fa-file-alt"></i><span>Логи</span></a></li>
-    </ul>
-  </div>
-
-  <!-- Кнопка для сворачивания/разворачивания меню -->
-  <button class="toggle-btn" id="toggle-btn">
-    <i class="fas fa-bars"></i>
-  </button>
-
-  <!-- Основной контент -->
-  <div class="container" id="main-content">
-    <h1>Система мониторинга утечек данных</h1>
-
-    <!-- Карта -->
-    <div class="card">
-      <h2>Карта утечек</h2>
-      <div id="map"></div>
-    </div>
-
-    <!-- Графики, скриншоты и логи -->
-    <div class="grid">
-      <div class="card">
-        <h2>Утечки за месяц</h2>
-        <canvas id="monthlyLeaksChart"></canvas>
-      </div>
-      <div class="card">
-        <h2>Состояние парсеров</h2>
-        <div class="parsers-status">
-          <p>TG Node 0:
-            <span class="status-{% if parser_status['tg-node-0'] %}active{% else %}error{% endif %}">
-              {% if parser_status['tg-node-0'] %}Активен{% else %}Ошибка{% endif %}
-            </span>
-          </p>
-          <p>TG Node 1:
-            <span class="status-{% if parser_status['tg-node-1'] %}active{% else %}error{% endif %}">
-              {% if parser_status['tg-node-1'] %}Активен{% else %}Ошибка{% endif %}
-            </span>
-          </p>
-          <canvas id="parsersStatusChart"></canvas>
-        </div>
-      </div>
-      <div class="card">
-        <h2>Источники утечек</h2>
-        <canvas id="sourcesChart"></canvas>
-      </div>
-      <div class="card">
-        <h2>Скриншоты с форумов</h2>
-        <div class="screenshots">
-          <img src="https://via.placeholder.com/300x200.png?text=Скриншот+1" alt="Скриншот 1">
-          <img src="https://via.placeholder.com/300x200.png?text=Скриншот+2" alt="Скриншот 2">
-          <img src="https://via.placeholder.com/300x200.png?text=Скриншот+3" alt="Скриншот 3">
-          <img src="https://via.placeholder.com/300x200.png?text=Скриншот+4" alt="Скриншот 4">
-          <img src="https://via.placeholder.com/300x200.png?text=Скриншот+5" alt="Скриншот 5">
-        </div>
-      </div>
-      <div class="card" style="grid-column: span 2;">
-        <h2>Логи системы</h2>
-        <div class="logs">
-          <p>[2023-10-01 12:34] Парсер форума запущен.</p>
-          <p>[2023-10-01 12:35] Найдена новая утечка на форуме X.</p>
-          <p>[2023-10-01 12:36] Ошибка парсера Telegram: timeout.</p>
-          <p>[2023-10-01 12:37] Утечка данных в Москве зафиксирована.</p>
-          <p>[2023-10-01 12:38] Парсер даркнета завершил работу.</p>
-          <p>[2023-10-01 12:39] Новый скриншот добавлен в базу.</p>
-        </div>
-      </div>
-    </div>
-  </div>
-
-  <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
-  <script src="https://unpkg.com/leaflet@1.7.1/dist/leaflet.js"></script>
-  <script>
-    const monthlyLeaksCtx = document.getElementById('monthlyLeaksChart').getContext('2d');
-    new Chart(monthlyLeaksCtx, {
-      type: 'line',
-      data: {
-        labels: [
-          {% for day in leaks_stats['leaks_by_day'] %}
-            "{{ day[0].strftime('%d.%m') }}"{% if not loop.last %}, {% endif %}
-          {% endfor %}
-        ],
-        datasets: [{
-          label: 'Утечки',
-          data: [
-            {% for day in leaks_stats['leaks_by_day'] %}
-              {{ day[1] }}{% if not loop.last %}, {% endif %}
-            {% endfor %}
-          ],
-          borderColor: '#3399FF',
-          tension: 0.1,
-          fill: true,
-          backgroundColor: 'rgba(51, 153, 255, 0.1)'
-        }]
-      },
-      options: {
-        responsive: true,
-        plugins: {
-          legend: {
-            labels: {
-              color: '#fff'
-            }
-          }
-        },
-        scales: {
-          y: {
-            beginAtZero: true,
-            grid: {
-              color: '#444'
-            },
-            ticks: {
-              color: '#fff'
-            }
-          },
-          x: {
-            grid: {
-              color: '#444'
-            },
-            ticks: {
-              color: '#fff'
-            }
-          }
-        }
-      }
-    });
-    // График состояния парсеров
-    const parsersStatusCtx = document.getElementById('parsersStatusChart').getContext('2d');
-    new Chart(parsersStatusCtx, {
-      type: 'doughnut',
-      data: {
-        labels: ['Активны', 'Ошибки', 'Неактивны'],
-        datasets: [{
-          label: 'Состояние',
-          data: [
-            {{ parser_status['active'] }},
-            {{ parser_status['errors'] }},
-            0  // Неактивных нет в текущей логике
-          ],
-          backgroundColor: ['#3399FF', '#ff4444', '#666666'],
-        }]
-      },
-      options: {
-        responsive: true,
-        plugins: {
-          legend: {
-            labels: {
-              color: '#fff',
-            }
-          }
-        }
-      }
-    });
-
-    // График источников утечек
-    const sourcesCtx = document.getElementById('sourcesChart').getContext('2d');
-    new Chart(sourcesCtx, {
-      type: 'bar',
-      data: {
-        labels: [
-          {% for source in leaks_stats['leaks_by_source'] %}
-            '{{ source[0] }}'{% if not loop.last %},{% endif %}
-          {% endfor %}
-        ],
-        datasets: [{
-          label: 'Количество утечек',
-          data: [
-            {% for source in leaks_stats['leaks_by_source'] %}
-              {{ source[1] }}{% if not loop.last %},{% endif %}
-            {% endfor %}
-          ],
-          backgroundColor: ['#3399FF', '#00cc99', '#0099cc'],
-        }]
-      },
-      options: {
-        responsive: true,
-        plugins: {
-          legend: {
-            labels: {
-              color: '#fff',
-            }
-          }
-        },
-        scales: {
-          y: {
-            beginAtZero: true,
-            grid: {
-              color: '#444',
-            },
-            ticks: {
-              color: '#fff',
-            }
-          },
-          x: {
-            grid: {
-              color: '#444',
-            },
-            ticks: {
-              color: '#fff',
-            }
-          }
-        }
-      }
-    });
-
-    const map = L.map('map').setView([55.7558, 37.6176], 5);
-    L.tileLayer('https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png', {
-      attribution: '© OpenStreetMap contributors, © CARTO'
-    }).addTo(map);
-
-    const marker1 = L.marker([55.7558, 37.6176], {
-      icon: L.divIcon({ className: 'leaflet-marker-icon' })
-    }).addTo(map).bindPopup('Утечка данных в Москве');
-
-    const marker2 = L.marker([59.9343, 30.3351], {
-      icon: L.divIcon({ className: 'leaflet-marker-icon' })
-    }).addTo(map).bindPopup('Утечка данных в Санкт-Петербурге');
-
-    const sidebar = document.getElementById('sidebar');
-    const mainContent = document.getElementById('main-content');
-    const toggleBtn = document.getElementById('toggle-btn');
-
-    toggleBtn.addEventListener('click', () => {
-      sidebar.classList.toggle('collapsed');
-      mainContent.classList.toggle('collapsed');
-    });
-  </script>
-</body>
-
-</html>

web/templates/logs.html

@@ -1,241 +0,0 @@
-<!DOCTYPE html>
-<html lang="ru">
-<head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Логи системы</title>
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css">
-  <style>
-    body {
-      margin: 0;
-      padding: 0;
-      font-family: 'Arial', sans-serif;
-      color: #ffffff;
-      overflow: hidden;
-      display: flex;
-      transition: margin-left 0.3s ease;
-    }
-
-    /* Боковое меню */
-    .sidebar {
-      width: 250px;
-      background-color: #2e2e2e;
-      padding: 20px;
-      box-shadow: 2px 0 5px rgba(0, 0, 0, 0.3);
-      transition: width 0.3s ease;
-      overflow-y: auto;
-      position: fixed;
-      top: 0;
-      left: 0;
-      bottom: 0;
-      z-index: 2;
-    }
-
-    .sidebar.collapsed {
-      width: 60px;
-    }
-
-    .sidebar.collapsed h2,
-    .sidebar.collapsed ul li a span {
-      display: none;
-    }
-
-    .sidebar h2 {
-      color: #3399FF;
-      text-align: center;
-      margin-bottom: 20px;
-      font-size: 24px;
-      transition: opacity 0.3s ease;
-    }
-
-    .sidebar ul {
-      list-style: none;
-      padding: 0;
-    }
-
-    .sidebar ul li {
-      margin: 15px 0;
-    }
-
-    .sidebar ul li a {
-      color: #ffffff;
-      text-decoration: none;
-      font-size: 16px;
-      transition: color 0.3s ease;
-      display: flex;
-      align-items: center;
-    }
-
-    .sidebar ul li a:hover {
-      color: #3399FF;
-    }
-
-    .sidebar ul li a i {
-      margin-right: 10px;
-      font-size: 20px;
-    }
-
-    .sidebar ul li a span {
-      transition: opacity 0.3s ease;
-    }
-
-    /* Кнопка для сворачивания/разворачивания меню */
-    .toggle-btn {
-      position: fixed;
-      left: 10px;
-      top: 10px;
-      background-color: #3399FF;
-      border: none;
-      color: #fff;
-      padding: 10px;
-      border-radius: 5px;
-      cursor: pointer;
-      z-index: 1000;
-    }
-
-    /* Основной контент */
-    .container {
-      flex: 1;
-      padding: 20px;
-      margin-left: 250px;
-      transition: margin-left 0.3s ease;
-      min-height: 100vh;
-      display: flex;
-      justify-content: center;
-      align-items: center;
-    }
-
-    .container.collapsed {
-      margin-left: 60px;
-    }
-
-    /* Видео на фоне */
-    .background-video {
-      position: fixed;
-      top: 0;
-      left: 0;
-      width: 100%;
-      height: 100%;
-      object-fit: cover;
-      z-index: -1;
-      opacity: 0.8;
-    }
-
-    /* Затемнение поверх видео */
-    .overlay {
-      position: fixed;
-      top: 0;
-      left: 0;
-      width: 100%;
-      height: 100%;
-      background: rgba(0, 0, 0, 0.7);
-      z-index: -1;
-    }
-
-    /* Контейнер для логов */
-    .logs-container {
-      width: 90%;
-      max-width: 800px;
-      background-color: rgba(46, 46, 46, 0.9);
-      padding: 20px;
-      border-radius: 10px;
-      box-shadow: 0 0 20px rgba(51, 153, 255, 0.5);
-    }
-
-    .logs-container h2 {
-      color: #3399FF;
-      margin-bottom: 20px;
-      font-size: 24px;
-    }
-
-    /* Стили для логов */
-    .logs {
-      height: 60vh;
-      overflow-y: auto;
-      font-family: 'Courier New', monospace;
-      font-size: 14px;
-      color: #3399FF;
-      padding-right: 10px;
-    }
-
-    .logs p {
-      margin: 10px 0;
-      padding: 5px;
-      background-color: rgba(0, 0, 0, 0.3);
-      border-radius: 5px;
-    }
-
-    /* Стили для скроллбара */
-    .logs::-webkit-scrollbar {
-      width: 8px;
-    }
-
-    .logs::-webkit-scrollbar-track {
-      background: rgba(0, 0, 0, 0.2);
-      border-radius: 5px;
-    }
-
-    .logs::-webkit-scrollbar-thumb {
-      background: #00ffcc;
-      border-radius: 5px;
-    }
-
-    .logs::-webkit-scrollbar-thumb:hover {
-      background: #00cc99;
-    }
-  </style>
-</head>
-<body>
-  <!-- Боковое меню -->
-  <div class="sidebar" id="sidebar">
-    <h2>Меню</h2>
-    <ul>
-      <li><a href="index.html"><i class="fas fa-home"></i><span>Главная</span></a></li>
-      <li><a href="#"><i class="fas fa-cog"></i><span>Настройки</span></a></li>
-      <li><a href="#"><i class="fas fa-chart-line"></i><span>Отчеты</span></a></li>
-      <li><a href="#"><i class="fas fa-code"></i><span>Парсеры</span></a></li>
-      <li><a href="logs.html"><i class="fas fa-file-alt"></i><span>Логи</span></a></li>
-    </ul>
-  </div>
-
-  <!-- Кнопка для сворачивания/разворачивания меню -->
-  <button class="toggle-btn" id="toggle-btn">
-    <i class="fas fa-bars"></i>
-  </button>
-
-  <!-- Основной контент -->
-  <div class="container" id="main-content">
-    <!-- Видео на фоне -->
-    <video class="background-video" autoplay loop muted>
-      <source src="{{ url_for('static', filename='alb_glob0411_1080p_24fps.mp4') }}" type="video/mp4">
-      Ваш браузер не поддерживает видео.
-    </video>
-    <!-- Затемнение поверх видео -->
-    <div class="overlay"></div>
-
-    <!-- Контейнер для логов -->
-    <div class="logs-container">
-      <h2>Логи системы (tg_nodes.log)</h2>
-      <div class="logs" id="logs-container">
-        {% for log in logs %}
-        <p>{{ log }}</p>
-        {% else %}
-        <p>Нет записей в логах</p>
-        {% endfor %}
-      </div>
-    </div>
-  </div>
-
-  <script>
-    // Сворачивание/разворачивание бокового меню
-    const sidebar = document.getElementById('sidebar');
-    const mainContent = document.getElementById('main-content');
-    const toggleBtn = document.getElementById('toggle-btn');
-
-    toggleBtn.addEventListener('click', () => {
-      sidebar.classList.toggle('collapsed');
-      mainContent.classList.toggle('collapsed');
-    });
-  </script>
-</body>
-</html>