37 lines
927 B
Python
37 lines
927 B
Python
from celery import Celery
|
|
from kombu import Queue
|
|
|
|
app = Celery(
|
|
'leak_monitor',
|
|
broker='memory://localhost',
|
|
# backend='rpc://',
|
|
include=['tasks.tg_crawler', 'tasks.forum_crawler']
|
|
)
|
|
|
|
app.conf.update(
|
|
worker_pool='solo',
|
|
worker_max_tasks_per_child=100,
|
|
task_serializer='json',
|
|
result_serializer='json',
|
|
accept_content=['json'],
|
|
timezone='Europe/Moscow',
|
|
enable_utc=True,
|
|
)
|
|
|
|
app.conf.task_queues = (
|
|
Queue('telegram', routing_key='telegram.#'),
|
|
Queue('forum', routing_key='forum.#'),
|
|
)
|
|
|
|
app.conf.beat_schedule = {
|
|
'monitor-telegram-channels': {
|
|
'task': 'tasks.tg_crawler.monitor_channel',
|
|
'schedule': 10.0,
|
|
'options': {'queue': 'telegram'}
|
|
},
|
|
'crawl-forum': {
|
|
'task': 'forum_crawler.crawl_forum_task',
|
|
'schedule': 3600.0 * 24, # час
|
|
'args': ('https://thehackernews.com/search/label/data%20breach', [])
|
|
},
|
|
} |