Source code for s2_sdk._read_session

from __future__ import annotations

import asyncio
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable
from contextlib import suppress
from dataclasses import dataclass, field
from typing import Self

from s2_sdk._exceptions import (
    S2ClientError,
    normalize_exception,
    set_and_retrieve_future_exception,
)
from s2_sdk._types import ReadBatch, StreamPosition


class _ReadSessionRetrying:
    pass


@dataclass(frozen=True, slots=True)
class _ReadSessionHeartbeat:
    tail: StreamPosition


@dataclass(frozen=True, slots=True)
class _ReadSessionBatch:
    batch: ReadBatch


_ReadSessionEvent = _ReadSessionRetrying | _ReadSessionHeartbeat | _ReadSessionBatch


@dataclass(slots=True)
class _ReadDelivery:
    batch: ReadBatch
    caught_up_tail: StreamPosition | None
    delivered: asyncio.Event = field(default_factory=asyncio.Event)


[docs] class ReadSession(AsyncIterator[ReadBatch]): """Async iterator that yields :class:`ReadBatch` values and tracks caught-up state. Use it as an async context manager, or call :meth:`close` explicitly to close the session. Caution: Returned by :meth:`S2Stream.read_session`. Do not instantiate directly. """ __slots__ = ( "_caught_up_tail", "_closed", "_delivery_queue", "_error", "_ignore_command_records", "_task", "_events", "_caught_up_futs", ) def __init__( self, events: AsyncGenerator[_ReadSessionEvent, None], *, ignore_command_records: bool = False, ) -> None: self._events = events self._ignore_command_records = ignore_command_records self._delivery_queue: asyncio.Queue[_ReadDelivery | BaseException | None] = ( asyncio.Queue(maxsize=1) ) self._task: asyncio.Task[None] | None = None self._closed = False self._error: BaseException | None = None self._caught_up_tail: StreamPosition | None = None self._caught_up_futs: set[asyncio.Future[StreamPosition]] = set()
[docs] def is_caught_up(self) -> bool: """Whether this session has yielded all records with sequence numbers less than the sequence number of the last observed tail position. The stream's tail may have advanced since this session last observed it. Use :meth:`S2Stream.check_tail` if you need the current tail. """ return self._caught_up_tail is not None
[docs] def caught_up(self) -> Awaitable[StreamPosition]: """Get an awaitable that resolves to a tail position once this session is caught up to it. See :meth:`is_caught_up` for the semantics of caught up. This awaitable only signals that the session is caught up. To yield batches, continue iterating over the session. """ loop = asyncio.get_running_loop() caught_up_fut: asyncio.Future[StreamPosition] = loop.create_future() if self._caught_up_tail is not None: caught_up_fut.set_result(_copy_position(self._caught_up_tail)) elif self._closed: set_and_retrieve_future_exception( caught_up_fut, self._error if self._error is not None else _session_closed_error(), ) else: self._caught_up_futs.add(caught_up_fut) caught_up_fut.add_done_callback(self._caught_up_futs.discard) self._ensure_started() return caught_up_fut
[docs] async def close(self) -> None: """Close the session.""" if self._closed: return if self._task is None: self._finish() return current = asyncio.current_task() cancellation_count = current.cancelling() if current is not None else 0 self._task.cancel() with suppress(asyncio.CancelledError): await self._task if not self._closed: self._finish() if current is not None and current.cancelling() > cancellation_count: raise asyncio.CancelledError
def __aiter__(self) -> Self: self._ensure_started() return self async def __anext__(self) -> ReadBatch: session = self.__aiter__() try: delivery = await session._delivery_queue.get() except asyncio.CancelledError: await self.close() raise if delivery is None: self._delivery_queue.put_nowait(None) raise StopAsyncIteration if isinstance(delivery, BaseException): self._delivery_queue.put_nowait(None) raise delivery if delivery.caught_up_tail is not None: self._mark_caught_up(delivery.caught_up_tail) delivery.delivered.set() return delivery.batch async def __aenter__(self) -> Self: return self.__aiter__() async def __aexit__(self, exc_type, exc_val, exc_tb) -> bool: await self.close() return False def _ensure_started(self) -> None: if self._task is None and not self._closed: self._task = asyncio.get_running_loop().create_task(self._run()) async def _run(self) -> None: error: BaseException | None = None cancelled = False events = self._events try: async for event in events: await self._handle_event(event) except asyncio.CancelledError: cancelled = True except BaseException as exc: error = normalize_exception(exc) finally: try: await events.aclose() except BaseException as exc: if error is None and not cancelled: error = normalize_exception(exc) self._finish(error) async def _handle_event(self, event: _ReadSessionEvent) -> None: if isinstance(event, _ReadSessionRetrying): self._mark_not_caught_up() elif isinstance(event, _ReadSessionHeartbeat): self._mark_caught_up(event.tail) else: batch = event.batch caught_up_tail = ( batch.tail if batch.tail is not None and batch.records[-1].seq_num + 1 == batch.tail.seq_num else None ) if self._ignore_command_records: batch = ReadBatch( records=[r for r in batch.records if not r.is_command_record()], tail=batch.tail, ) if batch.records: self._mark_not_caught_up() delivery = _ReadDelivery(batch, caught_up_tail) await self._delivery_queue.put(delivery) await delivery.delivered.wait() elif caught_up_tail is not None: self._mark_caught_up(caught_up_tail) else: self._mark_not_caught_up() def _mark_caught_up(self, tail: StreamPosition) -> None: if self._closed: return self._caught_up_tail = _copy_position(tail) caught_up_futs = tuple(self._caught_up_futs) self._caught_up_futs.clear() for caught_up_fut in caught_up_futs: if not caught_up_fut.done(): caught_up_fut.set_result(_copy_position(tail)) def _mark_not_caught_up(self) -> None: if not self._closed: self._caught_up_tail = None def _finish(self, error: BaseException | None = None) -> None: if self._closed: return self._closed = True self._error = error if self._error is not None: self._caught_up_tail = None caught_up_futs = tuple(self._caught_up_futs) self._caught_up_futs.clear() for caught_up_fut in caught_up_futs: set_and_retrieve_future_exception( caught_up_fut, self._error if self._error is not None else _session_closed_error(), ) while not self._delivery_queue.empty(): self._delivery_queue.get_nowait() self._delivery_queue.put_nowait(self._error)
def _copy_position(position: StreamPosition) -> StreamPosition: return StreamPosition(position.seq_num, position.timestamp) def _session_closed_error() -> S2ClientError: return S2ClientError("ReadSession is closed")