SyncKafkaBroker

Sync Kafka Broker.

SyncKafkaBroker

Bases: BaseBroker, SyncPluginMixin

A broker that listens to Kafka and adds tasks to the queue.

Example

from qtasks import QueueTasks
from qtasks.brokers import SyncKafkaBroker

broker = SyncKafkaBroker(name="QueueTasks", url="localhost:9092")

app = QueueTasks(broker=broker)

Source code in src/qtasks/brokers/sync_kafka.py

class SyncKafkaBroker(BaseBroker, SyncPluginMixin):
    """
    A broker that listens to Kafka and adds tasks to the queue.

    ## Example

    ```python
    from qtasks import QueueTasks
    from qtasks.brokers import SyncKafkaBroker

    broker = SyncKafkaBroker(name="QueueTasks", url="localhost:9092")

    app = QueueTasks(broker=broker)
    ```
    """

    def __init__(
        self,
        name: Annotated[
            str,
            Doc("""
                    Project name. This name is also used by the broker.

                    Default: `QueueTasks`.
                    """),
        ] = "QueueTasks",
        url: Annotated[
            str,
            Doc("""
                    URL to connect to Kafka.

                    Default: `localhost:9092`.
                    """),
        ] = "localhost:9092",
        storage: Annotated[
            Optional[BaseStorage],
            Doc("""
                    Storage.

                    Default: `SyncRedisStorage`.
                    """),
        ] = None,
        topic: Annotated[
            str,
            Doc("""
                    Kafka topic.

                    Default: `task_queue`.
                    """),
        ] = "task_queue",
        log: Annotated[
            Logger | None,
            Doc("""
                    Logger.

                    Default: `qtasks.logs.Logger`.
                    """),
        ] = None,
        config: Annotated[
            QueueConfig | None,
            Doc("""
                    Config.

                    Default: `qtasks.configs.config.QueueConfig`.
                    """),
        ] = None,
        events: Annotated[
            Optional[BaseEvents],
            Doc("""
                    Events.

                    Default: `qtasks.events.SyncEvents`.
                    """),
        ] = None,
    ):
        """
        Initializing SyncKafkaBroker.

        Args:
            name (str, optional): Project name. Default: `QueueTasks`.
            url (str, optional): URL to connect to Kafka. Default: `None`.
            storage (BaseStorage, optional): Storage. Default: `None`.
            topic (str, optional): Kafka topic. Default: `task_queue`.
            log (Logger, optional): Logger. Default: `None`.
            config (QueueConfig, optional): Config. Default: `None`.
            events (BaseEvents, optional): Events. Default: `qtasks.events.SyncEvents`.
        """
        self.url = url
        storage = storage or SyncRedisStorage(
            name=name, log=log, config=config, events=events
        )
        super().__init__(
            name=name, log=log, config=config, events=events, storage=storage
        )

        self.storage: BaseStorage[Literal[False]]

        self.events = self.events or SyncEvents()
        self.topic = f"{self.name}_{topic}"

        self.consumer = KafkaConsumer(
            self.topic,
            bootstrap_servers=self.url,
            group_id=f"{self.name}_group",
            auto_offset_reset="earliest",
            enable_auto_commit=True,
            value_deserializer=lambda m: m.decode("utf-8"),
        )
        self.producer = KafkaProducer(
            bootstrap_servers=self.url,
            auto_offset_reset="earliest",
            enable_auto_commit=True,
            value_deserializer=lambda m: m.decode("utf-8"),
        )

        self.running = False

    def listen(
        self,
        worker: Annotated[
            BaseWorker[Literal[False]],
            Doc("""
                    Worker class.
                    """),
        ],
    ):
        """
        Listens to Kafka and transfers tasks to the worker.

        Args:
            worker (BaseWorker): Worker class.
        """
        self._plugin_trigger("broker_listen_start", broker=self, worker=worker)
        self.running = True
        for msg in self.consumer:
            if not self.running:
                break
            task_data = msg.value
            task_name, uuid_str, priority = task_data.split(":")
            uuid = UUID(uuid_str)
            model_get = self.get(uuid=uuid)
            if model_get is None:
                if self.log:
                    self.log.warning(f"Task {uuid} not found in repository.")
                continue
            args, kwargs, created_at = (
                model_get.args or (),
                model_get.kwargs or {},
                model_get.created_at.timestamp(),
            )
            if self.log:
                self.log.info(f"Received new task: {uuid}")
            new_args = self._plugin_trigger(
                "broker_add_worker",
                broker=self,
                worker=worker,
                task_name=task_name,
                uuid=uuid,
                priority=int(priority),
                args=args,
                kw=kwargs,
                created_at=created_at,
                return_last=True,
            )
            if new_args:
                task_name = new_args.get("task_name", task_name)
                uuid = new_args.get("uuid", uuid)
                priority = new_args.get("priority", priority)
                args = new_args.get("args", args)
                kwargs = new_args.get("kw", kwargs)
                created_at = new_args.get("created_at", created_at)

            worker.add(
                name=task_name,
                uuid=uuid,
                priority=int(priority),
                args=args,
                kwargs=kwargs,
                created_at=created_at,
            )

    def add(
        self,
        task_name: Annotated[
            str,
            Doc("""
                    Task name.
                    """),
        ],
        priority: Annotated[
            int,
            Doc("""
                    Task priority.

                    Default: `0`.
                    """),
        ] = 0,
        extra: Annotated[
            dict | None,
            Doc("""
                    Additional task parameters.

                    Default: `None`.
                    """),
        ] = None,
        args: Annotated[
            tuple | None,
            Doc("""
                    Task arguments of type args.

                    Default: `()`.
                    """),
        ] = None,
        kwargs: Annotated[
            dict | None,
            Doc("""
                    Task arguments of type kwargs.

                    Default: `{}`.
                    """),
        ] = None,
    ) -> Task:
        """
        Adds a task to the broker.

        Args:
            task_name (str): The name of the task.
            priority (int, optional): Task priority. Default: `0`.
            extra (dict, optional): Additional task parameters. Default: `None`.
            args (tuple, optional): Task arguments of type args. Default: `()`.
            kwargs (dict, optional): Task arguments of type kwargs. Default: `{}`.

        Returns:
            Task: `schemas.task.Task`

        Raises:
            ValueError: Invalid task status.
        """
        args, kwargs = args or (), kwargs or {}
        uuid = uuid4()
        uuid_str = str(uuid)
        created_at = time()

        model = TaskStatusNewSchema(
            task_name=task_name,
            priority=priority,
            created_at=created_at,
            updated_at=created_at,
            args=str(args),
            kwargs=str(kwargs),
        )

        if extra:
            model = self._dynamic_model(model=model, extra=extra)

        new_model = self._plugin_trigger(
            "broker_add_before",
            broker=self,
            storage=self.storage,
            model=model,
            return_last=True,
        )
        if new_model:
            model = new_model.get("model", model)

        self.storage.add(uuid=uuid, task_status=model)

        task_data = f"{task_name}:{uuid_str}:{priority}"
        self.producer.send(self.topic, task_data)
        self.producer.flush()

        self._plugin_trigger(
            "broker_add_after", broker=self, storage=self.storage, model=model
        )

        return Task(
            status=TaskStatusEnum.NEW.value,
            task_name=task_name,
            uuid=uuid,
            priority=priority,
            args=args,
            kwargs=kwargs,
            created_at=datetime.fromtimestamp(created_at),
            updated_at=datetime.fromtimestamp(created_at),
        )

    def get(
        self,
        uuid: Annotated[
            UUID | str,
            Doc("""
                    UUID of the task.
                    """),
        ],
    ) -> Task | None:
        """
        Obtaining information about a task.

        Args:
            uuid (UUID|str): UUID of the task.

        Returns:
            Task|None: If there is task information, returns `schemas.task.Task`, otherwise `None`.
        """
        if isinstance(uuid, str):
            uuid = UUID(uuid)
        task = self.storage.get(uuid=uuid)
        new_task = self._plugin_trigger(
            "broker_get", broker=self, task=task, return_last=True
        )
        if new_task:
            task = new_task.get("task", task)
        return task

    def update(
        self,
        **kwargs: Annotated[
            Any,
            Doc("""
                    Update arguments for storage type kwargs.
                    """),
        ],
    ) -> None:
        """
        Updates task information.

        Args:
            kwargs (dict, optional): task data of type kwargs.
        """
        new_kw = self._plugin_trigger(
            "broker_update", broker=self, kw=kwargs, return_last=True
        )
        if new_kw:
            kwargs = new_kw.get("kw", kwargs)
        return self.storage.update(**kwargs)

    def start(
        self,
        worker: Annotated[
            BaseWorker,
            Doc("""
                    Worker class.
                    """),
        ],
    ) -> None:
        """
        Launches the broker.

        Args:
            worker (BaseWorker): Worker class.
        """
        self._plugin_trigger("broker_start", broker=self, worker=worker)
        self.storage.start()

        if self.config.delete_finished_tasks:
            self.storage._delete_finished_tasks()

        if self.config.running_older_tasks:
            self.storage._running_older_tasks(worker)

        self.listen(worker)

    def stop(self):
        """The broker stops."""
        self._plugin_trigger("broker_stop", broker=self)
        self.running = False
        self.consumer.stop()
        self.producer.stop()

    def remove_finished_task(
        self,
        task_broker: Annotated[
            TaskPrioritySchema,
            Doc("""
                    Priority task diagram.
                    """),
        ],
        model: Annotated[
            TaskStatusSuccessSchema | TaskStatusErrorSchema,
            Doc("""
                    Model of the task result.
                    """),
        ],
    ) -> None:
        """
        Updates storage data via the `self.storage.remove_finished_task` function.

        Args:
            task_broker (TaskPrioritySchema): The priority task schema.
            model (TaskStatusSuccessSchema | TaskStatusErrorSchema): Model of the task result.
        """
        new_model = self._plugin_trigger(
            "broker_remove_finished_task",
            broker=self,
            storage=self.storage,
            model=model,
            return_last=True,
        )
        if new_model:
            model = new_model.get("model", model)

        self.storage.remove_finished_task(task_broker, model)

    def _running_older_tasks(self, worker):
        self._plugin_trigger("broker_running_older_tasks", broker=self, worker=worker)
        return self.storage._running_older_tasks(worker)

    def flush_all(self) -> None:
        """Delete all data."""
        self._plugin_trigger("broker_flush_all", broker=self)
        self.storage.flush_all()

__init__(name='QueueTasks', url='localhost:9092', storage=None, topic='task_queue', log=None, config=None, events=None)

Initializing SyncKafkaBroker.

Parameters:

Name	Type	Description	Default
name	str	Project name. Default: QueueTasks.	'QueueTasks'
url	str	URL to connect to Kafka. Default: None.	'localhost:9092'
storage	BaseStorage	Storage. Default: None.	None
topic	str	Kafka topic. Default: task_queue.	'task_queue'
log	Logger	Logger. Default: None.	None
config	QueueConfig	Config. Default: None.	None
events	BaseEvents	Events. Default: qtasks.events.SyncEvents.	None

Source code in src/qtasks/brokers/sync_kafka.py

def __init__(
    self,
    name: Annotated[
        str,
        Doc("""
                Project name. This name is also used by the broker.

                Default: `QueueTasks`.
                """),
    ] = "QueueTasks",
    url: Annotated[
        str,
        Doc("""
                URL to connect to Kafka.

                Default: `localhost:9092`.
                """),
    ] = "localhost:9092",
    storage: Annotated[
        Optional[BaseStorage],
        Doc("""
                Storage.

                Default: `SyncRedisStorage`.
                """),
    ] = None,
    topic: Annotated[
        str,
        Doc("""
                Kafka topic.

                Default: `task_queue`.
                """),
    ] = "task_queue",
    log: Annotated[
        Logger | None,
        Doc("""
                Logger.

                Default: `qtasks.logs.Logger`.
                """),
    ] = None,
    config: Annotated[
        QueueConfig | None,
        Doc("""
                Config.

                Default: `qtasks.configs.config.QueueConfig`.
                """),
    ] = None,
    events: Annotated[
        Optional[BaseEvents],
        Doc("""
                Events.

                Default: `qtasks.events.SyncEvents`.
                """),
    ] = None,
):
    """
    Initializing SyncKafkaBroker.

    Args:
        name (str, optional): Project name. Default: `QueueTasks`.
        url (str, optional): URL to connect to Kafka. Default: `None`.
        storage (BaseStorage, optional): Storage. Default: `None`.
        topic (str, optional): Kafka topic. Default: `task_queue`.
        log (Logger, optional): Logger. Default: `None`.
        config (QueueConfig, optional): Config. Default: `None`.
        events (BaseEvents, optional): Events. Default: `qtasks.events.SyncEvents`.
    """
    self.url = url
    storage = storage or SyncRedisStorage(
        name=name, log=log, config=config, events=events
    )
    super().__init__(
        name=name, log=log, config=config, events=events, storage=storage
    )

    self.storage: BaseStorage[Literal[False]]

    self.events = self.events or SyncEvents()
    self.topic = f"{self.name}_{topic}"

    self.consumer = KafkaConsumer(
        self.topic,
        bootstrap_servers=self.url,
        group_id=f"{self.name}_group",
        auto_offset_reset="earliest",
        enable_auto_commit=True,
        value_deserializer=lambda m: m.decode("utf-8"),
    )
    self.producer = KafkaProducer(
        bootstrap_servers=self.url,
        auto_offset_reset="earliest",
        enable_auto_commit=True,
        value_deserializer=lambda m: m.decode("utf-8"),
    )

    self.running = False

add(task_name, priority=0, extra=None, args=None, kwargs=None)

Adds a task to the broker.

Parameters:

Name	Type	Description	Default
task_name	str	The name of the task.	required
priority	int	Task priority. Default: 0.	0
extra	dict	Additional task parameters. Default: None.	None
args	tuple	Task arguments of type args. Default: ().	None
kwargs	dict	Task arguments of type kwargs. Default: {}.	None

Returns:

Name	Type	Description
Task	Task	schemas.task.Task

Raises:

Type	Description
ValueError	Invalid task status.

Source code in src/qtasks/brokers/sync_kafka.py

def add(
    self,
    task_name: Annotated[
        str,
        Doc("""
                Task name.
                """),
    ],
    priority: Annotated[
        int,
        Doc("""
                Task priority.

                Default: `0`.
                """),
    ] = 0,
    extra: Annotated[
        dict | None,
        Doc("""
                Additional task parameters.

                Default: `None`.
                """),
    ] = None,
    args: Annotated[
        tuple | None,
        Doc("""
                Task arguments of type args.

                Default: `()`.
                """),
    ] = None,
    kwargs: Annotated[
        dict | None,
        Doc("""
                Task arguments of type kwargs.

                Default: `{}`.
                """),
    ] = None,
) -> Task:
    """
    Adds a task to the broker.

    Args:
        task_name (str): The name of the task.
        priority (int, optional): Task priority. Default: `0`.
        extra (dict, optional): Additional task parameters. Default: `None`.
        args (tuple, optional): Task arguments of type args. Default: `()`.
        kwargs (dict, optional): Task arguments of type kwargs. Default: `{}`.

    Returns:
        Task: `schemas.task.Task`

    Raises:
        ValueError: Invalid task status.
    """
    args, kwargs = args or (), kwargs or {}
    uuid = uuid4()
    uuid_str = str(uuid)
    created_at = time()

    model = TaskStatusNewSchema(
        task_name=task_name,
        priority=priority,
        created_at=created_at,
        updated_at=created_at,
        args=str(args),
        kwargs=str(kwargs),
    )

    if extra:
        model = self._dynamic_model(model=model, extra=extra)

    new_model = self._plugin_trigger(
        "broker_add_before",
        broker=self,
        storage=self.storage,
        model=model,
        return_last=True,
    )
    if new_model:
        model = new_model.get("model", model)

    self.storage.add(uuid=uuid, task_status=model)

    task_data = f"{task_name}:{uuid_str}:{priority}"
    self.producer.send(self.topic, task_data)
    self.producer.flush()

    self._plugin_trigger(
        "broker_add_after", broker=self, storage=self.storage, model=model
    )

    return Task(
        status=TaskStatusEnum.NEW.value,
        task_name=task_name,
        uuid=uuid,
        priority=priority,
        args=args,
        kwargs=kwargs,
        created_at=datetime.fromtimestamp(created_at),
        updated_at=datetime.fromtimestamp(created_at),
    )

flush_all()

Delete all data.

Source code in src/qtasks/brokers/sync_kafka.py

def flush_all(self) -> None:
    """Delete all data."""
    self._plugin_trigger("broker_flush_all", broker=self)
    self.storage.flush_all()

get(uuid)

Obtaining information about a task.

Parameters:

Name	Type	Description	Default
uuid	UUID | str	UUID of the task.	required

Returns:

Type	Description
Task | None	Task|None: If there is task information, returns schemas.task.Task, otherwise None.

Source code in src/qtasks/brokers/sync_kafka.py

def get(
    self,
    uuid: Annotated[
        UUID | str,
        Doc("""
                UUID of the task.
                """),
    ],
) -> Task | None:
    """
    Obtaining information about a task.

    Args:
        uuid (UUID|str): UUID of the task.

    Returns:
        Task|None: If there is task information, returns `schemas.task.Task`, otherwise `None`.
    """
    if isinstance(uuid, str):
        uuid = UUID(uuid)
    task = self.storage.get(uuid=uuid)
    new_task = self._plugin_trigger(
        "broker_get", broker=self, task=task, return_last=True
    )
    if new_task:
        task = new_task.get("task", task)
    return task

listen(worker)

Listens to Kafka and transfers tasks to the worker.

Parameters:

Name	Type	Description	Default
worker	BaseWorker	Worker class.	required

Source code in src/qtasks/brokers/sync_kafka.py

def listen(
    self,
    worker: Annotated[
        BaseWorker[Literal[False]],
        Doc("""
                Worker class.
                """),
    ],
):
    """
    Listens to Kafka and transfers tasks to the worker.

    Args:
        worker (BaseWorker): Worker class.
    """
    self._plugin_trigger("broker_listen_start", broker=self, worker=worker)
    self.running = True
    for msg in self.consumer:
        if not self.running:
            break
        task_data = msg.value
        task_name, uuid_str, priority = task_data.split(":")
        uuid = UUID(uuid_str)
        model_get = self.get(uuid=uuid)
        if model_get is None:
            if self.log:
                self.log.warning(f"Task {uuid} not found in repository.")
            continue
        args, kwargs, created_at = (
            model_get.args or (),
            model_get.kwargs or {},
            model_get.created_at.timestamp(),
        )
        if self.log:
            self.log.info(f"Received new task: {uuid}")
        new_args = self._plugin_trigger(
            "broker_add_worker",
            broker=self,
            worker=worker,
            task_name=task_name,
            uuid=uuid,
            priority=int(priority),
            args=args,
            kw=kwargs,
            created_at=created_at,
            return_last=True,
        )
        if new_args:
            task_name = new_args.get("task_name", task_name)
            uuid = new_args.get("uuid", uuid)
            priority = new_args.get("priority", priority)
            args = new_args.get("args", args)
            kwargs = new_args.get("kw", kwargs)
            created_at = new_args.get("created_at", created_at)

        worker.add(
            name=task_name,
            uuid=uuid,
            priority=int(priority),
            args=args,
            kwargs=kwargs,
            created_at=created_at,
        )

remove_finished_task(task_broker, model)

Updates storage data via the self.storage.remove_finished_task function.

Parameters:

Name	Type	Description	Default
task_broker	TaskPrioritySchema	The priority task schema.	required
model	TaskStatusSuccessSchema | TaskStatusErrorSchema	Model of the task result.	required

Source code in src/qtasks/brokers/sync_kafka.py

def remove_finished_task(
    self,
    task_broker: Annotated[
        TaskPrioritySchema,
        Doc("""
                Priority task diagram.
                """),
    ],
    model: Annotated[
        TaskStatusSuccessSchema | TaskStatusErrorSchema,
        Doc("""
                Model of the task result.
                """),
    ],
) -> None:
    """
    Updates storage data via the `self.storage.remove_finished_task` function.

    Args:
        task_broker (TaskPrioritySchema): The priority task schema.
        model (TaskStatusSuccessSchema | TaskStatusErrorSchema): Model of the task result.
    """
    new_model = self._plugin_trigger(
        "broker_remove_finished_task",
        broker=self,
        storage=self.storage,
        model=model,
        return_last=True,
    )
    if new_model:
        model = new_model.get("model", model)

    self.storage.remove_finished_task(task_broker, model)

start(worker)

Launches the broker.

Parameters:

Name	Type	Description	Default
worker	BaseWorker	Worker class.	required

Source code in src/qtasks/brokers/sync_kafka.py

def start(
    self,
    worker: Annotated[
        BaseWorker,
        Doc("""
                Worker class.
                """),
    ],
) -> None:
    """
    Launches the broker.

    Args:
        worker (BaseWorker): Worker class.
    """
    self._plugin_trigger("broker_start", broker=self, worker=worker)
    self.storage.start()

    if self.config.delete_finished_tasks:
        self.storage._delete_finished_tasks()

    if self.config.running_older_tasks:
        self.storage._running_older_tasks(worker)

    self.listen(worker)

stop()

The broker stops.

Source code in src/qtasks/brokers/sync_kafka.py

def stop(self):
    """The broker stops."""
    self._plugin_trigger("broker_stop", broker=self)
    self.running = False
    self.consumer.stop()
    self.producer.stop()

update(**kwargs)

Updates task information.

Parameters:

Name	Type	Description	Default
kwargs	dict	task data of type kwargs.	{}

Source code in src/qtasks/brokers/sync_kafka.py

def update(
    self,
    **kwargs: Annotated[
        Any,
        Doc("""
                Update arguments for storage type kwargs.
                """),
    ],
) -> None:
    """
    Updates task information.

    Args:
        kwargs (dict, optional): task data of type kwargs.
    """
    new_kw = self._plugin_trigger(
        "broker_update", broker=self, kw=kwargs, return_last=True
    )
    if new_kw:
        kwargs = new_kw.get("kw", kwargs)
    return self.storage.update(**kwargs)