diff --git a/README.rst b/README.rst index cd4fc96..ca1ca97 100644 --- a/README.rst +++ b/README.rst @@ -133,6 +133,7 @@ Now you can type commands, for instance, ``help``:: ps (p) Show task table ps-terminated (pst,pt) List recently terminated/cancelled tasks signal Send a Unix signal + snapshot (snap) Manage point-in-time snapshots of the running/terminated task state stacktrace (st,stack) Print a stack trace from the event loop thread where (w) Show stack frames and the task creation chain of a task where-terminated (wt) Show stack frames and the termination/cancellation chain of a task diff --git a/aiomonitor/monitor.py b/aiomonitor/monitor.py index 5a2ad70..31f4595 100644 --- a/aiomonitor/monitor.py +++ b/aiomonitor/monitor.py @@ -21,6 +21,7 @@ from typing import ( Dict, Final, Generator, + Iterable, List, Optional, Sequence, @@ -42,6 +43,9 @@ from .types import ( FormattedLiveTaskInfo, FormattedStackItem, FormattedTerminatedTaskInfo, + SnapshotDiff, + SnapshotSummary, + TaskSnapshot, TerminatedTaskInfo, ) from .utils import ( @@ -110,6 +114,8 @@ class Monitor: _canceller_chain: Dict[str, str] _canceller_stacks: Dict[str, List[traceback.FrameSummary] | None] _cancellation_chain_queue: janus.Queue[CancellationChain] + _snapshots: Dict[int, TaskSnapshot] + _snapshot_order: List[int] def __init__( self, @@ -122,6 +128,7 @@ class Monitor: console_enabled: bool = True, hook_task_factory: bool = False, max_termination_history: int = 1000, + max_snapshots: int = 10, locals: Optional[Dict[str, Any]] = None, ) -> None: self._monitored_loop = loop or asyncio.get_running_loop() @@ -158,6 +165,11 @@ class Monitor: self._terminated_history = [] self._max_termination_history = max_termination_history + self._snapshots = {} + self._snapshot_order = [] + self._next_snapshot_id = 1 + self._max_snapshots = max_snapshots + self._ui_started = threading.Event() self._ui_thread = threading.Thread(target=self._ui_main, args=(), daemon=True) @@ -233,19 +245,29 @@ class Monitor: self._ui_thread.join() self._closed = True - def format_running_task_list( - self, filter_: str, persistent: bool - ) -> Sequence[FormattedLiveTaskInfo]: - all_running_tasks = asyncio.all_tasks(loop=self._monitored_loop) + def _format_task_list( + self, + task_iterable: Iterable["asyncio.Task[Any]"], + filter_: str, + persistent: bool, + ) -> List[FormattedLiveTaskInfo]: tasks = [] - for task in sorted(all_running_tasks, key=id): + for task in sorted(task_iterable, key=id): taskid = str(id(task)) if isinstance(task, TracedTask): coro_repr = _format_coroutine(task._orig_coro).partition(" ")[0] if persistent and task._orig_coro not in persistent_coro: continue else: - coro_repr = _format_coroutine(task.get_coro()).partition(" ")[0] + # A task's underlying coroutine reference may be released by + # the event loop internals once the task is done (e.g., a + # task captured by a snapshot that has since finished), so we + # need a fallback here. + task_coro = task.get_coro() + if task_coro is None: + coro_repr = "" + else: + coro_repr = _format_coroutine(task_coro).partition(" ")[0] if persistent: # untracked tasks should be skipped when showing persistent ones only continue @@ -283,13 +305,21 @@ class Monitor: ) return tasks - def format_terminated_task_list( + def format_running_task_list( self, filter_: str, persistent: bool - ) -> Sequence[FormattedTerminatedTaskInfo]: - terminated_tasks = self._terminated_tasks.values() + ) -> Sequence[FormattedLiveTaskInfo]: + all_running_tasks = asyncio.all_tasks(loop=self._monitored_loop) + return self._format_task_list(all_running_tasks, filter_, persistent) + + def _format_terminated_task_list( + self, + info_iterable: Iterable[TerminatedTaskInfo], + filter_: str, + persistent: bool, + ) -> List[FormattedTerminatedTaskInfo]: tasks = [] for item in sorted( - terminated_tasks, + info_iterable, key=lambda info: info.terminated_at, reverse=True, ): @@ -314,6 +344,12 @@ class Monitor: ) return tasks + def format_terminated_task_list( + self, filter_: str, persistent: bool + ) -> Sequence[FormattedTerminatedTaskInfo]: + terminated_tasks = self._terminated_tasks.values() + return self._format_terminated_task_list(terminated_tasks, filter_, persistent) + async def cancel_monitored_task(self, task_id: str | int) -> str: task_id_ = int(task_id) task = task_by_id(task_id_, self._monitored_loop) @@ -335,20 +371,17 @@ class Monitor: else: raise ValueError("Invalid or non-existent task ID", task_id) - def format_running_task_stack( + def _format_task_stack_chain( self, - task_id: str | int, - ) -> Sequence[FormattedStackItem]: + task: "asyncio.Task[Any]", + ) -> List[FormattedStackItem]: depth = 0 - task_id_ = int(task_id) - task = task_by_id(task_id_, self._monitored_loop) - if task is None: - raise MissingTask(task_id_) task_chain: List[asyncio.Task[Any]] = [] - while task is not None: - task_chain.append(task) - task_ref = self._created_traceback_chains.get(task) - task = task_ref() if task_ref is not None else None + current: Optional[asyncio.Task[Any]] = task + while current is not None: + task_chain.append(current) + task_ref = self._created_traceback_chains.get(current) + current = task_ref() if task_ref is not None else None prev_task = None formatted_stack_list = [] for task in reversed(task_chain): @@ -419,6 +452,16 @@ class Monitor: ) return formatted_stack_list + def format_running_task_stack( + self, + task_id: str | int, + ) -> Sequence[FormattedStackItem]: + task_id_ = int(task_id) + task = task_by_id(task_id_, self._monitored_loop) + if task is None: + raise MissingTask(task_id_) + return self._format_task_stack_chain(task) + def format_terminated_task_stack( self, trace_id: str, @@ -502,6 +545,168 @@ class Monitor: ) return formatted_stack_list + def _evict_snapshots(self) -> None: + """ + Evicts the oldest unnamed snapshot(s) while the number of stored + snapshots exceeds ``max_snapshots``. Named snapshots are always + preserved and never evicted automatically. + """ + while len(self._snapshot_order) > self._max_snapshots: + for idx, snapshot_id in enumerate(self._snapshot_order): + if self._snapshots[snapshot_id].name is None: + del self._snapshot_order[idx] + del self._snapshots[snapshot_id] + break + else: + # All the remaining snapshots are named, so we cannot + # evict anything further to honor the cap. + break + + async def capture_snapshot(self, name: Optional[str] = None) -> int: + """ + Captures the current running and terminated task state as a new, + immutable snapshot and returns its auto-incrementing ID (starting + from 1). + + If ``max_snapshots`` would be exceeded by adding this snapshot, the + oldest unnamed snapshot is evicted first. Named snapshots are + preserved and are not automatically evicted. + """ + running_tasks = list(asyncio.all_tasks(loop=self._monitored_loop)) + terminated_tasks = dict(self._terminated_tasks) + snapshot_id = self._next_snapshot_id + self._next_snapshot_id += 1 + self._snapshots[snapshot_id] = TaskSnapshot( + id=snapshot_id, + name=name, + created_at=time.perf_counter(), + running_tasks=running_tasks, + terminated_tasks=terminated_tasks, + ) + self._snapshot_order.append(snapshot_id) + self._evict_snapshots() + return snapshot_id + + def list_snapshots(self) -> Sequence[SnapshotSummary]: + """ + Returns the summaries (id, name, running/terminated task counts) of + all the currently stored snapshots, in the order they were created. + """ + return [ + SnapshotSummary( + id=snapshot.id, + name=snapshot.name, + running_count=len(snapshot.running_tasks), + terminated_count=len(snapshot.terminated_tasks), + ) + for snapshot in ( + self._snapshots[snapshot_id] for snapshot_id in self._snapshot_order + ) + ] + + def get_snapshot(self, snapshot_id: int) -> TaskSnapshot: + """ + Returns the raw :class:`~aiomonitor.types.TaskSnapshot` for the + given ID. + + :raises KeyError: if there is no snapshot with the given ID. + """ + return self._snapshots[snapshot_id] + + def delete_snapshot(self, snapshot_id: int) -> None: + """ + Deletes the snapshot with the given ID, regardless of whether it is + named or not. + + :raises KeyError: if there is no snapshot with the given ID. + """ + if snapshot_id not in self._snapshots: + raise KeyError(snapshot_id) + del self._snapshots[snapshot_id] + self._snapshot_order.remove(snapshot_id) + + def format_snapshot_task_list( + self, + snapshot_id: int, + ) -> Sequence[FormattedLiveTaskInfo]: + """ + Formats the running tasks captured by the given snapshot, in the + same shape as :meth:`format_running_task_list`. + + :raises KeyError: if there is no snapshot with the given ID. + """ + snapshot = self.get_snapshot(snapshot_id) + return self._format_task_list(snapshot.running_tasks, "", False) + + def format_snapshot_terminated_task_list( + self, + snapshot_id: int, + ) -> Sequence[FormattedTerminatedTaskInfo]: + """ + Formats the terminated tasks captured by the given snapshot, in the + same shape as :meth:`format_terminated_task_list`. + + :raises KeyError: if there is no snapshot with the given ID. + """ + snapshot = self.get_snapshot(snapshot_id) + return self._format_terminated_task_list( + snapshot.terminated_tasks.values(), "", False + ) + + def format_snapshot_task_stack( + self, + snapshot_id: int, + task_id: str | int, + ) -> Sequence[FormattedStackItem]: + """ + Formats the stack and creation chain of the given task, as long as + it was running at the time the given snapshot was captured, in the + same shape as :meth:`format_running_task_stack`. + + :raises KeyError: if there is no snapshot with the given ID, or no + running task with the given ID inside the snapshot. + """ + snapshot = self.get_snapshot(snapshot_id) + task_id_ = int(task_id) + task = next((t for t in snapshot.running_tasks if id(t) == task_id_), None) + if task is None: + raise KeyError(task_id) + return self._format_task_stack_chain(task) + + def format_snapshot_diff( + self, + snapshot_id_1: int, + snapshot_id_2: int, + ) -> SnapshotDiff: + """ + Compares the running tasks captured by the two given snapshots by + the identity (``id()``) of the task objects and returns a + :class:`~aiomonitor.types.SnapshotDiff` listing the tasks that were + added, removed, or common between the two snapshots. + + :raises KeyError: if there is no snapshot with either of the given + IDs. + """ + snapshot1 = self.get_snapshot(snapshot_id_1) + snapshot2 = self.get_snapshot(snapshot_id_2) + tasks1 = {id(t): t for t in snapshot1.running_tasks} + tasks2 = {id(t): t for t in snapshot2.running_tasks} + ids1 = set(tasks1.keys()) + ids2 = set(tasks2.keys()) + added_ids = ids2 - ids1 + removed_ids = ids1 - ids2 + common_ids = ids1 & ids2 + added = self._format_task_list( + (tasks2[task_id] for task_id in added_ids), "", False + ) + removed = self._format_task_list( + (tasks1[task_id] for task_id in removed_ids), "", False + ) + common = self._format_task_list( + (tasks2[task_id] for task_id in common_ids), "", False + ) + return SnapshotDiff(added=added, removed=removed, common=common) + async def _coro_wrapper(self, coro: Awaitable[T_co]) -> T_co: myself = asyncio.current_task() assert isinstance(myself, TracedTask) @@ -636,6 +841,7 @@ def start_monitor( console_enabled: bool = True, hook_task_factory: bool = False, max_termination_history: Optional[int] = None, + max_snapshots: Optional[int] = None, locals: Optional[Dict[str, Any]] = None, ) -> Monitor: """ @@ -649,6 +855,9 @@ def start_monitor( :param int console_port: python REPL port, by default 20103 :param bool console_enabled: flag indicates if python REPL is requred to start with instance of monitor. + :param int max_snapshots: the maximum number of unnamed task state + snapshots to keep, by default 10. Named snapshots are always + preserved. :param dict locals: dictionary with variables exposed in python console environment """ @@ -665,6 +874,11 @@ def start_monitor( if max_termination_history is not None else get_default_args(monitor_cls.__init__)["max_termination_history"] ), + max_snapshots=( + max_snapshots + if max_snapshots is not None + else get_default_args(monitor_cls.__init__)["max_snapshots"] + ), locals=locals, ) m.start() diff --git a/aiomonitor/termui/commands.py b/aiomonitor/termui/commands.py index dfcf9d6..6cbe1c9 100644 --- a/aiomonitor/termui/commands.py +++ b/aiomonitor/termui/commands.py @@ -10,7 +10,7 @@ import sys import textwrap import traceback from contextvars import copy_context -from typing import TYPE_CHECKING, List, TextIO, Tuple +from typing import TYPE_CHECKING, List, Optional, Sequence, TextIO, Tuple import click from prompt_toolkit import PromptSession @@ -26,12 +26,19 @@ from ..exceptions import MissingTask from .completion import ( ClickCompleter, complete_signal_names, + complete_snapshot_id, + complete_snapshot_task_id, complete_task_id, complete_trace_id, ) if TYPE_CHECKING: from ..monitor import Monitor + from ..types import ( + FormattedLiveTaskInfo, + FormattedStackItem, + FormattedTerminatedTaskInfo, + ) log = logging.getLogger(__name__) @@ -392,17 +399,9 @@ def do_console(ctx: click.Context) -> None: self._termui_tasks.add(task) -@monitor_cli.command(name="ps", aliases=["p"]) -@click.option("-f", "--filter", "filter_", help="filter by coroutine or task name") -@click.option("-p", "--persistent", is_flag=True, help="show only persistent tasks") -@custom_help_option -@auto_command_done -def do_ps( - ctx: click.Context, - filter_: str, - persistent: bool, -) -> None: - """Show task table""" +def _render_running_task_table( + tasks: "Sequence[FormattedLiveTaskInfo]", +) -> AsciiTable: headers = ( "Task ID", "State", @@ -411,10 +410,7 @@ def do_ps( "Created Location", "Since", ) - self: Monitor = ctx.obj - stdout = _get_current_stdout() table_data: List[Tuple[str, str, str, str, str, str]] = [headers] - tasks = self.format_running_task_list(filter_, persistent) for task in tasks: table_data.append(( task.task_id, @@ -427,28 +423,12 @@ def do_ps( table = AsciiTable(table_data) table.inner_row_border = False table.inner_column_border = False - if filter_ or persistent: - stdout.write( - f"{len(tasks)} tasks running (showing {len(table_data) - 1} tasks)\n" - ) - else: - stdout.write(f"{len(tasks)} tasks running\n") - stdout.write(table.table) - stdout.write("\n") - stdout.flush() + return table -@monitor_cli.command(name="ps-terminated", aliases=["pt", "pst"]) -@click.option("-f", "--filter", "filter_", help="filter by coroutine or task name") -@click.option("-p", "--persistent", is_flag=True, help="show only persistent tasks") -@custom_help_option -@auto_command_done -def do_ps_terminated( - ctx: click.Context, - filter_: str, - persistent: bool, -) -> None: - """List recently terminated/cancelled tasks""" +def _render_terminated_task_table( + tasks: "Sequence[FormattedTerminatedTaskInfo]", +) -> AsciiTable: headers = ( "Trace ID", "Name", @@ -456,10 +436,7 @@ def do_ps_terminated( "Since Started", "Since Terminated", ) - self: Monitor = ctx.obj - stdout = _get_current_stdout() table_data: List[Tuple[str, str, str, str, str]] = [headers] - tasks = self.format_terminated_task_list(filter_, persistent) for task in tasks: table_data.append(( task.task_id, @@ -471,30 +448,61 @@ def do_ps_terminated( table = AsciiTable(table_data) table.inner_row_border = False table.inner_column_border = False + return table + + +@monitor_cli.command(name="ps", aliases=["p"]) +@click.option("-f", "--filter", "filter_", help="filter by coroutine or task name") +@click.option("-p", "--persistent", is_flag=True, help="show only persistent tasks") +@custom_help_option +@auto_command_done +def do_ps( + ctx: click.Context, + filter_: str, + persistent: bool, +) -> None: + """Show task table""" + self: Monitor = ctx.obj + stdout = _get_current_stdout() + tasks = self.format_running_task_list(filter_, persistent) + table = _render_running_task_table(tasks) if filter_ or persistent: - stdout.write( - f"{len(tasks)} tasks terminated (showing {len(table_data) - 1} tasks)\n" - ) + stdout.write(f"{len(tasks)} tasks running (showing {len(tasks)} tasks)\n") else: - stdout.write(f"{len(tasks)} tasks terminated (old ones may be stripped)\n") + stdout.write(f"{len(tasks)} tasks running\n") stdout.write(table.table) stdout.write("\n") stdout.flush() -@monitor_cli.command(name="where", aliases=["w"]) -@click.argument("taskid", shell_complete=complete_task_id) +@monitor_cli.command(name="ps-terminated", aliases=["pt", "pst"]) +@click.option("-f", "--filter", "filter_", help="filter by coroutine or task name") +@click.option("-p", "--persistent", is_flag=True, help="show only persistent tasks") @custom_help_option @auto_command_done -def do_where(ctx: click.Context, taskid: str) -> None: - """Show stack frames and the task creation chain of a task""" +def do_ps_terminated( + ctx: click.Context, + filter_: str, + persistent: bool, +) -> None: + """List recently terminated/cancelled tasks""" self: Monitor = ctx.obj stdout = _get_current_stdout() - try: - formatted_stack_list = self.format_running_task_stack(taskid) - except MissingTask as e: - print_fail(f"No task {e.task_id}") - return + tasks = self.format_terminated_task_list(filter_, persistent) + table = _render_terminated_task_table(tasks) + if filter_ or persistent: + stdout.write(f"{len(tasks)} tasks terminated (showing {len(tasks)} tasks)\n") + else: + stdout.write(f"{len(tasks)} tasks terminated (old ones may be stripped)\n") + stdout.write(table.table) + stdout.write("\n") + stdout.flush() + + +def _print_formatted_stack( + stdout: TextIO, + formatted_stack_list: "Sequence[FormattedStackItem]", +) -> None: for item_type, item_text in formatted_stack_list: if item_type == "header": stdout.write("\n") @@ -508,6 +516,22 @@ def do_where(ctx: click.Context, taskid: str) -> None: stdout.write("\n") +@monitor_cli.command(name="where", aliases=["w"]) +@click.argument("taskid", shell_complete=complete_task_id) +@custom_help_option +@auto_command_done +def do_where(ctx: click.Context, taskid: str) -> None: + """Show stack frames and the task creation chain of a task""" + self: Monitor = ctx.obj + stdout = _get_current_stdout() + try: + formatted_stack_list = self.format_running_task_stack(taskid) + except MissingTask as e: + print_fail(f"No task {e.task_id}") + return + _print_formatted_stack(stdout, formatted_stack_list) + + @monitor_cli.command(name="where-terminated", aliases=["wt"]) @click.argument("trace_id", shell_complete=complete_trace_id) @custom_help_option @@ -521,14 +545,150 @@ def do_where_terminated(ctx: click.Context, trace_id: str) -> None: except MissingTask as e: print_fail(f"No task {e.task_id}") return - for item_type, item_text in formatted_stack_list: - if item_type == "header": - stdout.write("\n") - print_formatted_text( - FormattedText([ - ("ansiwhite", item_text), - ]) - ) + _print_formatted_stack(stdout, formatted_stack_list) + + +@monitor_cli.group(name="snapshot", aliases=["snap"], add_help_option=False) +def do_snapshot() -> None: + """Manage point-in-time snapshots of the running/terminated task state""" + pass + + +@do_snapshot.command(name="save") +@click.option( + "-n", + "--name", + "name", + default=None, + help="An optional name to assign to the snapshot so that it is preserved from automatic eviction", +) +@custom_help_option +def do_snapshot_save(ctx: click.Context, name: Optional[str]) -> None: + """Capture the current running/terminated task state as a new snapshot""" + self: Monitor = ctx.obj + + @auto_async_command_done + async def _do_snapshot_save(ctx: click.Context) -> None: + snapshot_id = await self.capture_snapshot(name) + if name: + print_ok(f"Saved snapshot #{snapshot_id} ({name})") else: - stdout.write(textwrap.indent(item_text.strip("\n"), " ")) - stdout.write("\n") + print_ok(f"Saved snapshot #{snapshot_id}") + + task = self._ui_loop.create_task(_do_snapshot_save(ctx)) + self._termui_tasks.add(task) + + +@do_snapshot.command(name="list", aliases=["ls"]) +@custom_help_option +@auto_command_done +def do_snapshot_list(ctx: click.Context) -> None: + """List the saved snapshots""" + self: Monitor = ctx.obj + stdout = _get_current_stdout() + snapshots = self.list_snapshots() + headers = ("ID", "Name", "Running", "Terminated") + table_data: List[Tuple[str, str, str, str]] = [headers] + for snapshot in snapshots: + table_data.append(( + str(snapshot.id), + snapshot.name if snapshot.name is not None else "-", + str(snapshot.running_count), + str(snapshot.terminated_count), + )) + table = AsciiTable(table_data) + table.inner_row_border = False + table.inner_column_border = False + stdout.write(f"{len(snapshots)} snapshots saved\n") + stdout.write(table.table) + stdout.write("\n") + stdout.flush() + + +@do_snapshot.command(name="show") +@click.argument("snapshot_id", type=int, shell_complete=complete_snapshot_id) +@custom_help_option +@auto_command_done +def do_snapshot_show(ctx: click.Context, snapshot_id: int) -> None: + """Show the running and terminated tasks captured in a snapshot""" + self: Monitor = ctx.obj + stdout = _get_current_stdout() + try: + running_tasks = self.format_snapshot_task_list(snapshot_id) + terminated_tasks = self.format_snapshot_terminated_task_list(snapshot_id) + except KeyError: + print_fail(f"No snapshot {snapshot_id}") + return + + running_table = _render_running_task_table(running_tasks) + stdout.write(f"Snapshot #{snapshot_id}: {len(running_tasks)} tasks running\n") + stdout.write(running_table.table) + stdout.write("\n") + + terminated_table = _render_terminated_task_table(terminated_tasks) + stdout.write(f"Snapshot #{snapshot_id}: {len(terminated_tasks)} tasks terminated\n") + stdout.write(terminated_table.table) + stdout.write("\n") + stdout.flush() + + +@do_snapshot.command(name="where") +@click.argument("snapshot_id", type=int, shell_complete=complete_snapshot_id) +@click.argument("task_id", shell_complete=complete_snapshot_task_id) +@custom_help_option +@auto_command_done +def do_snapshot_where(ctx: click.Context, snapshot_id: int, task_id: str) -> None: + """Show stack frames and the task creation chain of a task captured in a snapshot""" + self: Monitor = ctx.obj + stdout = _get_current_stdout() + try: + formatted_stack_list = self.format_snapshot_task_stack(snapshot_id, task_id) + except KeyError: + print_fail(f"No task {task_id} in snapshot {snapshot_id}") + return + _print_formatted_stack(stdout, formatted_stack_list) + + +@do_snapshot.command(name="diff") +@click.argument("snapshot_id_1", type=int, shell_complete=complete_snapshot_id) +@click.argument("snapshot_id_2", type=int, shell_complete=complete_snapshot_id) +@custom_help_option +@auto_command_done +def do_snapshot_diff( + ctx: click.Context, + snapshot_id_1: int, + snapshot_id_2: int, +) -> None: + """Compare the running tasks captured by two snapshots""" + self: Monitor = ctx.obj + stdout = _get_current_stdout() + try: + diff = self.format_snapshot_diff(snapshot_id_1, snapshot_id_2) + except KeyError: + print_fail(f"No snapshot {snapshot_id_1} or {snapshot_id_2}") + return + for title, tasks in ( + ("Added", diff.added), + ("Removed", diff.removed), + ("Common", diff.common), + ): + table = _render_running_task_table(tasks) + stdout.write(f"{title} ({len(tasks)} tasks)\n") + stdout.write(table.table) + stdout.write("\n") + stdout.flush() + + +@do_snapshot.command(name="delete", aliases=["del", "rm"]) +@click.argument("snapshot_id", type=int, shell_complete=complete_snapshot_id) +@custom_help_option +@auto_command_done +def do_snapshot_delete(ctx: click.Context, snapshot_id: int) -> None: + """Delete a saved snapshot""" + self: Monitor = ctx.obj + try: + self.delete_snapshot(snapshot_id) + except KeyError: + print_fail(f"No snapshot {snapshot_id}") + return + print_ok(f"Deleted snapshot #{snapshot_id}") diff --git a/aiomonitor/termui/completion.py b/aiomonitor/termui/completion.py index 7c4938b..477bb22 100644 --- a/aiomonitor/termui/completion.py +++ b/aiomonitor/termui/completion.py @@ -88,3 +88,40 @@ def complete_signal_names( incomplete: str, ) -> Iterable[str]: return [sig.name for sig in signal.Signals if sig.name.startswith(incomplete)] + + +def complete_snapshot_id( + ctx: click.Context, + param: click.Parameter, + incomplete: str, +) -> Iterable[str]: + try: + self: Monitor = current_monitor.get() + except LookupError: + return [] + return [ + snapshot_id + for snapshot_id in map(str, self._snapshot_order) + if snapshot_id.startswith(incomplete) + ][:10] + + +def complete_snapshot_task_id( + ctx: click.Context, + param: click.Parameter, + incomplete: str, +) -> Iterable[str]: + try: + self: Monitor = current_monitor.get() + except LookupError: + return [] + try: + snapshot_id = int(ctx.params["snapshot_id"]) + snapshot = self.get_snapshot(snapshot_id) + except (KeyError, TypeError, ValueError): + return [] + return [ + task_id + for task_id in map(str, sorted(map(id, snapshot.running_tasks))) + if task_id.startswith(incomplete) + ][:10] diff --git a/aiomonitor/types.py b/aiomonitor/types.py index 04e5b04..721edbd 100644 --- a/aiomonitor/types.py +++ b/aiomonitor/types.py @@ -1,9 +1,10 @@ from __future__ import annotations +import asyncio import sys import traceback -from dataclasses import dataclass -from typing import List, NamedTuple, Optional +from dataclasses import dataclass, field +from typing import Any, Dict, List, NamedTuple, Optional if sys.version_info >= (3, 11): from enum import StrEnum @@ -59,3 +60,41 @@ class CancellationChain: target_id: str canceller_id: str canceller_stack: Optional[List[traceback.FrameSummary]] = None + + +@dataclass +class TaskSnapshot: + """ + A frozen-in-time capture of the running and terminated task state of a + :class:`~aiomonitor.monitor.Monitor`. + """ + + id: int + name: Optional[str] + created_at: float + running_tasks: List["asyncio.Task[Any]"] = field(default_factory=list) + terminated_tasks: Dict[str, TerminatedTaskInfo] = field(default_factory=dict) + + +@dataclass +class SnapshotSummary: + """ + A short summary of a :class:`TaskSnapshot`, used to render snapshot listings. + """ + + id: int + name: Optional[str] + running_count: int + terminated_count: int + + +@dataclass +class SnapshotDiff: + """ + The result of comparing the running tasks captured by two snapshots, + keyed by the identity (``id()``) of the task objects. + """ + + added: List[FormattedLiveTaskInfo] + removed: List[FormattedLiveTaskInfo] + common: List[FormattedLiveTaskInfo] diff --git a/aiomonitor/utils.py b/aiomonitor/utils.py index 1dab00f..8b460e6 100644 --- a/aiomonitor/utils.py +++ b/aiomonitor/utils.py @@ -18,7 +18,14 @@ def _format_task(task: "asyncio.Task[Any]") -> str: """ A simpler version of task's repr() """ - coro = _format_coroutine(task.get_coro()).partition(" ")[0] + task_coro = task.get_coro() + # A task's underlying coroutine reference may be released by the event + # loop internals once the task is done (e.g., a task captured by a + # snapshot that has since finished), so we need a fallback here. + if task_coro is None: + coro = "" + else: + coro = _format_coroutine(task_coro).partition(" ")[0] return f"" diff --git a/aiomonitor/webui/app.py b/aiomonitor/webui/app.py index d11a766..cf90df5 100644 --- a/aiomonitor/webui/app.py +++ b/aiomonitor/webui/app.py @@ -5,7 +5,7 @@ import dataclasses import sys from importlib.metadata import version from pathlib import Path -from typing import TYPE_CHECKING, Dict, Mapping, Tuple +from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple if sys.version_info >= (3, 11): from enum import StrEnum @@ -20,6 +20,7 @@ from .utils import APIParams, check_params if TYPE_CHECKING: from ..monitor import Monitor + from ..types import FormattedLiveTaskInfo, FormattedTerminatedTaskInfo @dataclasses.dataclass @@ -46,6 +47,33 @@ class ListFilterParams(APIParams): persistent: bool = Field(default=False) +class SnapshotSaveParams(APIParams): + name: Optional[str] = Field(default=None) + + +class SnapshotIdParams(APIParams): + snapshot_id: int + + +class SnapshotTaskListParams(APIParams): + snapshot_id: int + task_type: TaskTypes = Field(default=TaskTypes.RUNNING) + + +class SnapshotTraceParams(APIParams): + snapshot_id: int + task_id: str + + +class SnapshotDiffParams(APIParams): + snapshot_id_1: int + snapshot_id_2: int + + +class SnapshotPageParams(APIParams): + task_type: TaskTypes = Field(default=TaskTypes.RUNNING) + + @dataclasses.dataclass class NavigationItem: title: str @@ -57,6 +85,10 @@ nav_menus: Mapping[str, NavigationItem] = { title="Dashboard", current=False, ), + "/snapshots": NavigationItem( + title="Snapshots", + current=False, + ), "/about": NavigationItem( title="About", current=False, @@ -134,6 +166,57 @@ async def show_trace_page(request: web.Request) -> web.Response: return web.Response(body=output, content_type="text/html") +async def show_snapshot_list_page(request: web.Request) -> web.Response: + ctx: WebUIContext = request.app[ctx_key] + nav_info, nav_items = get_navigation_info(request.path) + template = ctx.jenv.get_template("snapshots.html") + async with check_params(request, SnapshotPageParams) as params: + output = template.render( + navigation=nav_items, + page={ + "title": nav_info.title, + }, + current_list_type=params.task_type, + list_types=[ + {"id": TaskTypes.RUNNING, "title": "Running"}, + {"id": TaskTypes.TERMINATED, "title": "Terminated"}, + ], + ) + return web.Response(body=output, content_type="text/html") + + +async def show_snapshot_trace_page(request: web.Request) -> web.Response: + ctx: WebUIContext = request.app[ctx_key] + template = ctx.jenv.get_template("trace.html") + async with check_params(request, SnapshotTraceParams) as params: + try: + trace_data = ctx.monitor.format_snapshot_task_stack( + params.snapshot_id, params.task_id + ) + except KeyError: + # Returning (instead of raising) an error response here is + # intentional: check_params() would otherwise re-wrap a raised + # HTTPException as an unrelated 500 Internal Server Error. + return web.json_response( + status=404, + data={ + "msg": ( + f"No task {params.task_id} in snapshot {params.snapshot_id}" + ) + }, + ) + output = template.render( + navigation=nav_menus, + page={ + "title": ( + f"Task trace for {params.task_id} in snapshot #{params.snapshot_id}" + ), + }, + trace_data=trace_data, + ) + return web.Response(body=output, content_type="text/html") + + async def get_version(request: web.Request) -> web.Response: return web.json_response( data={ @@ -166,20 +249,7 @@ async def get_live_task_list(request: web.Request) -> web.Response: params.persistent, ) return web.json_response( - data={ - "tasks": [ - { - "task_id": t.task_id, - "state": t.state, - "name": t.name, - "coro": t.coro, - "created_location": t.created_location, - "since": t.since, - "is_root": t.created_location == "-", - } - for t in tasks - ] - } + data={"tasks": [_serialize_live_task(t) for t in tasks]} ) @@ -191,18 +261,7 @@ async def get_terminated_task_list(request: web.Request) -> web.Response: params.persistent, ) return web.json_response( - data={ - "tasks": [ - { - "task_id": t.task_id, - "name": t.name, - "coro": t.coro, - "started_since": t.started_since, - "terminated_since": t.terminated_since, - } - for t in tasks - ] - } + data={"tasks": [_serialize_terminated_task(t) for t in tasks]} ) @@ -224,6 +283,147 @@ async def cancel_task(request: web.Request) -> web.Response: ) +def _serialize_live_task(t: "FormattedLiveTaskInfo") -> Dict[str, object]: + return { + "task_id": t.task_id, + "state": t.state, + "name": t.name, + "coro": t.coro, + "created_location": t.created_location, + "since": t.since, + "is_root": t.created_location == "-", + } + + +def _serialize_terminated_task(t: "FormattedTerminatedTaskInfo") -> Dict[str, object]: + return { + "task_id": t.task_id, + "name": t.name, + "coro": t.coro, + "started_since": t.started_since, + "terminated_since": t.terminated_since, + } + + +async def save_snapshot(request: web.Request) -> web.Response: + ctx: WebUIContext = request.app[ctx_key] + async with check_params(request, SnapshotSaveParams) as params: + snapshot_id = await ctx.monitor.capture_snapshot(params.name or None) + return web.json_response(data={"id": snapshot_id}) + + +async def get_snapshot_list(request: web.Request) -> web.Response: + ctx: WebUIContext = request.app[ctx_key] + snapshots = ctx.monitor.list_snapshots() + return web.json_response( + data={ + "snapshots": [ + { + "id": s.id, + "name": s.name, + "running_count": s.running_count, + "terminated_count": s.terminated_count, + } + for s in snapshots + ] + } + ) + + +async def get_snapshot_task_list(request: web.Request) -> web.Response: + ctx: WebUIContext = request.app[ctx_key] + async with check_params(request, SnapshotTaskListParams) as params: + try: + if params.task_type == TaskTypes.RUNNING: + running_tasks = ctx.monitor.format_snapshot_task_list( + params.snapshot_id + ) + tasks: List[Dict[str, object]] = [ + _serialize_live_task(t) for t in running_tasks + ] + elif params.task_type == TaskTypes.TERMINATED: + terminated_tasks = ctx.monitor.format_snapshot_terminated_task_list( + params.snapshot_id + ) + tasks = [_serialize_terminated_task(t) for t in terminated_tasks] + else: + raise RuntimeError("should not reach here") + except KeyError: + return web.json_response( + status=404, + data={"msg": f"No snapshot {params.snapshot_id}"}, + ) + for t in tasks: + t["snapshot_id"] = params.snapshot_id + return web.json_response(data={"tasks": tasks}) + + +async def get_snapshot_trace(request: web.Request) -> web.Response: + ctx: WebUIContext = request.app[ctx_key] + async with check_params(request, SnapshotTraceParams) as params: + try: + trace_data = ctx.monitor.format_snapshot_task_stack( + params.snapshot_id, params.task_id + ) + except KeyError: + return web.json_response( + status=404, + data={ + "msg": ( + f"No task {params.task_id} in snapshot {params.snapshot_id}" + ) + }, + ) + return web.json_response( + data={ + "trace": [ + {"type": str(item_type), "content": content} + for item_type, content in trace_data + ] + } + ) + + +async def get_snapshot_diff(request: web.Request) -> web.Response: + ctx: WebUIContext = request.app[ctx_key] + async with check_params(request, SnapshotDiffParams) as params: + try: + diff = ctx.monitor.format_snapshot_diff( + params.snapshot_id_1, params.snapshot_id_2 + ) + except KeyError: + return web.json_response( + status=404, + data={ + "msg": ( + f"No snapshot {params.snapshot_id_1} or {params.snapshot_id_2}" + ) + }, + ) + return web.json_response( + data={ + "added": [_serialize_live_task(t) for t in diff.added], + "removed": [_serialize_live_task(t) for t in diff.removed], + "common": [_serialize_live_task(t) for t in diff.common], + } + ) + + +async def delete_snapshot(request: web.Request) -> web.Response: + ctx: WebUIContext = request.app[ctx_key] + async with check_params(request, SnapshotIdParams) as params: + try: + ctx.monitor.delete_snapshot(params.snapshot_id) + except KeyError: + return web.json_response( + status=404, + data={"msg": f"No snapshot {params.snapshot_id}"}, + ) + return web.json_response( + data={"msg": f"Successfully deleted snapshot {params.snapshot_id}"}, + ) + + async def init_webui(monitor: Monitor) -> web.Application: jenv = Environment( loader=PackageLoader("aiomonitor.webui"), autoescape=select_autoescape() @@ -235,12 +435,20 @@ async def init_webui(monitor: Monitor) -> web.Application: ) app.router.add_route("GET", "/", show_list_page) app.router.add_route("GET", "/about", show_about_page) + app.router.add_route("GET", "/snapshots", show_snapshot_list_page) app.router.add_route("GET", "/trace-running", show_trace_page) app.router.add_route("GET", "/trace-terminated", show_trace_page) + app.router.add_route("GET", "/trace-snapshot", show_snapshot_trace_page) app.router.add_route("GET", "/api/version", get_version) app.router.add_route("POST", "/api/task-count", get_task_count) app.router.add_route("POST", "/api/live-tasks", get_live_task_list) app.router.add_route("POST", "/api/terminated-tasks", get_terminated_task_list) app.router.add_route("DELETE", "/api/task", cancel_task) + app.router.add_route("POST", "/api/snapshot/save", save_snapshot) + app.router.add_route("GET", "/api/snapshot/list", get_snapshot_list) + app.router.add_route("POST", "/api/snapshot/tasks", get_snapshot_task_list) + app.router.add_route("POST", "/api/snapshot/trace", get_snapshot_trace) + app.router.add_route("POST", "/api/snapshot/diff", get_snapshot_diff) + app.router.add_route("DELETE", "/api/snapshot", delete_snapshot) app.router.add_static("/static", Path(__file__).parent / "static") return app diff --git a/aiomonitor/webui/templates/snapshots.html b/aiomonitor/webui/templates/snapshots.html new file mode 100644 index 0000000..fa64e81 --- /dev/null +++ b/aiomonitor/webui/templates/snapshots.html @@ -0,0 +1,283 @@ +{% extends "layout.html" %} +{% block head_content %} +

+ Snapshots freeze the running/terminated task state at a point in time so that they + can be inspected and compared later. +

+{% endblock %} +{% block content %} +
+ +
+
+ + +
+
+ +
+
+ +
+
+
+
+ + + + + + + + + + + + +
IDNameRunningTerminated + Action +
+
+
+
+
+ +{% raw %} + +{% endraw %} + +

Snapshot tasks

+

Click "Running" or "Terminated" on a snapshot above to inspect its captured tasks.

+
+
+
+
+ + + + + +
+
+
+
+
+ +{% raw %} + + +{% endraw %} + +

Compare two snapshots

+
+
+ + +
+
+ + +
+
+ +
+
+
+ +{% raw %} + +{% endraw %} +
+ + +{% endblock %} diff --git a/changes/456.feature b/changes/456.feature new file mode 100644 index 0000000..ecf9965 --- /dev/null +++ b/changes/456.feature @@ -0,0 +1 @@ +Added task state snapshots to freeze and later compare the running/terminated task state over time, including new ``Monitor`` APIs (``capture_snapshot``, ``list_snapshots``, ``get_snapshot``, ``delete_snapshot``, and the ``format_snapshot_*`` formatters), a ``snapshot`` telnet CLI command group (``save``, ``list``/``ls``, ``show``, ``where``, ``diff``, ``delete``), and matching ``/api/snapshot/*`` web endpoints with a new ``/snapshots`` page diff --git a/docs/reference/monitor.rst b/docs/reference/monitor.rst index b5ff331..ebda0ab 100644 --- a/docs/reference/monitor.rst +++ b/docs/reference/monitor.rst @@ -43,3 +43,27 @@ The Monitor .. autoproperty:: host .. autoproperty:: port + + .. rubric:: Task state snapshots + + .. versionadded:: 0.8.0 + + Snapshots let you freeze the current running/terminated task state so + that it can be inspected and compared (diffed) later. See + :ref:`task-snapshots` for a walk-through. + + .. automethod:: capture_snapshot() + + .. automethod:: list_snapshots() + + .. automethod:: get_snapshot() + + .. automethod:: delete_snapshot() + + .. automethod:: format_snapshot_task_list() + + .. automethod:: format_snapshot_terminated_task_list() + + .. automethod:: format_snapshot_task_stack() + + .. automethod:: format_snapshot_diff() diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 96aa83d..8b17cec 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -76,12 +76,15 @@ Once connection established, one can type commands, for instance ``help``:: ps (p) Show task table ps-terminated (pst,pt) List recently terminated/cancelled tasks signal Send a Unix signal + snapshot (snap) Manage point-in-time snapshots of the running/terminated task state stacktrace (st,stack) Print a stack trace from the event loop thread where (w) Show stack frames and the task creation chain of a task where-terminated (wt) Show stack frames and the termination/cancellation chain of a task Additional commands can be added by defining a Click command function injected into :ref:`the monitor CLI `, see below :ref:`cust-commands`. +See :ref:`task-snapshots` for more about the ``snapshot`` command group. + .. versionchanged:: 0.5.0 As of 0.5.0, you must use a telnet client that implements the actual telnet @@ -158,6 +161,67 @@ To see the recursive task creation and termination history, you should pass ``hook_task_factory=True`` to the ``start_monitor()`` function. +.. _task-snapshots: + +Task state snapshots +-------------------- + +.. versionadded:: 0.8.0 + +Sometimes you want to freeze the current running/terminated task state so that you +can compare it against another point in time later, e.g. to spot task leaks or to see +which tasks were added/removed/kept running between two events in your application. +This is what *snapshots* are for. + +Snapshots are created with the ``snapshot save`` command (``snap save`` for short), with +an optional ``--name``. IDs are assigned starting from 1 and always increase, even if +older snapshots are deleted or evicted. By default, up to 10 snapshots are kept; you can +change this via the ``max_snapshots`` argument of :func:`~aiomonitor.start_monitor` or the +:class:`~aiomonitor.Monitor` constructor. When the limit is exceeded, the oldest snapshot +*without a name* is evicted first; named snapshots are always preserved, so give important +snapshots a name if you want to keep them around:: + + monitor >>> snapshot save --name before-deploy + ✓ Saved snapshot #1 (before-deploy) + monitor >>> snapshot save + ✓ Saved snapshot #2 + monitor >>> snapshot list + 2 snapshots saved + +----+---------------+---------+------------+ + | ID | Name | Running | Terminated | + +----+---------------+---------+------------+ + | 1 | before-deploy | 4 | 0 | + | 2 | - | 5 | 1 | + +----+---------------+---------+------------+ + +You can inspect the running/terminated tasks captured by a snapshot with ``snapshot show``, +look at a task's stack with ``snapshot where ``, and remove a snapshot +with ``snapshot delete`` (aliases: ``snap del``/``snap rm``). All of these commands report +a friendly error (instead of crashing the session) when given an unknown snapshot or task ID. + +To compare two snapshots, use ``snapshot diff``, which reports the tasks that were added, +removed, and kept in common between them (matched by task object identity):: + + monitor >>> snapshot diff 1 2 + Added (2 tasks) + ... + Removed (1 tasks) + ... + Common (3 tasks) + ... + +All of the ``snapshot`` sub-commands are also available as aliases under ``snap`` +(e.g., ``snap save``, ``snap ls``, ``snap rm``). + +The same functionality is available programmatically via +:meth:`~aiomonitor.monitor.Monitor.capture_snapshot`, +:meth:`~aiomonitor.monitor.Monitor.list_snapshots`, +:meth:`~aiomonitor.monitor.Monitor.get_snapshot`, +:meth:`~aiomonitor.monitor.Monitor.delete_snapshot`, +and the ``format_snapshot_*``/``format_snapshot_diff`` methods, as well as through the web UI's +``/snapshots`` page and the JSON endpoints under ``/api/snapshot/``. + + .. _cust-commands: Adding custom commands diff --git a/tests/conftest.py b/tests/conftest.py index 29f788d..b76502f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,25 @@ +from __future__ import annotations + +import asyncio +import contextlib +import contextvars +import functools +import io import socket +import unittest.mock +from typing import Sequence import pytest +from prompt_toolkit.output import DummyOutput + +import aiomonitor.termui.commands +from aiomonitor import Monitor +from aiomonitor.termui.commands import ( + command_done, + current_monitor, + current_stdout, + monitor_cli, +) # import uvloop # TODO: run the entire test suite with both uvloop and the vanilla loop. @@ -13,3 +32,104 @@ def unused_port(): return s.getsockname()[1] return f + + +@contextlib.contextmanager +def monitor_common(*, hook_task_factory: bool = False): + def make_baz(): + return "baz" + + locals_ = {"foo": "bar", "make_baz": make_baz} + # In the tests, we reuse the pytest's event loop as the monitored loop. + # Because of this, all cross-loop coroutine invocations should use the following + # pattern in both tests and the monitor/termui implementation: + # > fut = asyncio.wrap_future(asyncio.run_coroutine_threadsafe(...)) + # > await fut + test_loop = asyncio.get_running_loop() + mon = Monitor(test_loop, locals=locals_, hook_task_factory=hook_task_factory) + with mon: + yield mon + + +@pytest.fixture +async def monitor(request, event_loop): + with monitor_common() as monitor_instance: + yield monitor_instance + + +@pytest.fixture +async def monitor_with_task_factory(request, event_loop): + """ + Same as the ``monitor`` fixture, but with ``hook_task_factory=True`` so + that tasks created afterwards are traced (:class:`~aiomonitor.task.TracedTask`) + with real creation-location and timing information. + """ + with monitor_common(hook_task_factory=True) as monitor_instance: + yield monitor_instance + + +def get_task_ids(event_loop): + return [id(t) for t in asyncio.all_tasks(loop=event_loop)] + + +class BufferedOutput(DummyOutput): + def __init__(self) -> None: + self._buffer = io.StringIO() + + def write(self, data: str) -> None: + self._buffer.write(data) + + def write_raw(self, data: str) -> None: + self._buffer.write(data) + + +async def invoke_command( + monitor: Monitor, + args: Sequence[str], +) -> str: + dummy_stdout = BufferedOutput() + current_monitor_token = current_monitor.set(monitor) + current_stdout_token = current_stdout.set(dummy_stdout._buffer) + + async def _ui_create_event() -> asyncio.Event: + return asyncio.Event() + + fut = asyncio.run_coroutine_threadsafe(_ui_create_event(), monitor._ui_loop) + command_done_event: asyncio.Event = await asyncio.wrap_future(fut) + command_done_token = command_done.set(command_done_event) + try: + with unittest.mock.patch.object( + aiomonitor.termui.commands, + "print_formatted_text", + functools.partial( + aiomonitor.termui.commands.print_formatted_text, output=dummy_stdout + ), + ): + ctx = contextvars.copy_context() + ctx.run( + monitor_cli.main, + args, + prog_name="", + obj=monitor, + standalone_mode=False, # type: ignore + ) + # If Click raises UsageError before running the command, + # there will be no one to set command_done_event. + # In this case, the error is propagated to the upper stack + # immediately here. + fut = asyncio.run_coroutine_threadsafe( + command_done_event.wait(), # type: ignore + monitor._ui_loop, + ) + await asyncio.wrap_future(fut) + finally: + command_done.reset(command_done_token) + current_stdout.reset(current_stdout_token) + current_monitor.reset(current_monitor_token) + with contextlib.closing(dummy_stdout._buffer): + return dummy_stdout._buffer.getvalue() + + +@pytest.fixture(params=[True, False], ids=["console:True", "console:False"]) +def console_enabled(request): + return request.param diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 9ef832d..69c7573 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -1,121 +1,22 @@ from __future__ import annotations import asyncio -import contextlib import contextvars -import functools -import io import sys -import unittest.mock -from typing import Sequence import click import pytest from prompt_toolkit.application import create_app_session from prompt_toolkit.input import create_pipe_input -from prompt_toolkit.output import DummyOutput -import aiomonitor.termui.commands from aiomonitor import Monitor, start_monitor from aiomonitor.termui.commands import ( auto_command_done, - command_done, - current_monitor, - current_stdout, custom_help_option, monitor_cli, print_ok, ) - - -@contextlib.contextmanager -def monitor_common(): - def make_baz(): - return "baz" - - locals_ = {"foo": "bar", "make_baz": make_baz} - # In the tests, we reuse the pytest's event loop as the monitored loop. - # Because of this, all cross-loop coroutine invocations should use the following - # pattern in both tests and the monitor/termui implementation: - # > fut = asyncio.wrap_future(asyncio.run_coroutine_threadsafe(...)) - # > await fut - test_loop = asyncio.get_running_loop() - mon = Monitor(test_loop, locals=locals_) - with mon: - yield mon - - -@pytest.fixture -async def monitor(request, event_loop): - with monitor_common() as monitor_instance: - yield monitor_instance - - -def get_task_ids(event_loop): - return [id(t) for t in asyncio.all_tasks(loop=event_loop)] - - -class BufferedOutput(DummyOutput): - def __init__(self) -> None: - self._buffer = io.StringIO() - - def write(self, data: str) -> None: - self._buffer.write(data) - - def write_raw(self, data: str) -> None: - self._buffer.write(data) - - -async def invoke_command( - monitor: Monitor, - args: Sequence[str], -) -> str: - dummy_stdout = BufferedOutput() - current_monitor_token = current_monitor.set(monitor) - current_stdout_token = current_stdout.set(dummy_stdout._buffer) - - async def _ui_create_event() -> asyncio.Event: - return asyncio.Event() - - fut = asyncio.run_coroutine_threadsafe(_ui_create_event(), monitor._ui_loop) - command_done_event: asyncio.Event = await asyncio.wrap_future(fut) - command_done_token = command_done.set(command_done_event) - try: - with unittest.mock.patch.object( - aiomonitor.termui.commands, - "print_formatted_text", - functools.partial( - aiomonitor.termui.commands.print_formatted_text, output=dummy_stdout - ), - ): - ctx = contextvars.copy_context() - ctx.run( - monitor_cli.main, - args, - prog_name="", - obj=monitor, - standalone_mode=False, # type: ignore - ) - # If Click raises UsageError before running the command, - # there will be no one to set command_done_event. - # In this case, the error is propagated to the upper stack - # immediately here. - fut = asyncio.run_coroutine_threadsafe( - command_done_event.wait(), # type: ignore - monitor._ui_loop, - ) - await asyncio.wrap_future(fut) - finally: - command_done.reset(command_done_token) - current_stdout.reset(current_stdout_token) - current_monitor.reset(current_monitor_token) - with contextlib.closing(dummy_stdout._buffer): - return dummy_stdout._buffer.getvalue() - - -@pytest.fixture(params=[True, False], ids=["console:True", "console:False"]) -def console_enabled(request): - return request.param +from conftest import BufferedOutput, get_task_ids, invoke_command @pytest.mark.asyncio diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py new file mode 100644 index 0000000..5656fce --- /dev/null +++ b/tests/test_snapshot.py @@ -0,0 +1,584 @@ +from __future__ import annotations + +import asyncio +import unittest.mock +from typing import Any, cast + +import pytest + +from aiomonitor import Monitor +from conftest import invoke_command + +# --------------------------------------------------------------------------- +# Monitor API tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_capture_snapshot_ids_auto_increment(monitor: Monitor) -> None: + sid1 = await monitor.capture_snapshot() + sid2 = await monitor.capture_snapshot() + sid3 = await monitor.capture_snapshot() + assert sid1 == 1 + assert sid2 == 2 + assert sid3 == 3 + + +@pytest.mark.asyncio +async def test_capture_snapshot_with_name(monitor: Monitor) -> None: + sid = await monitor.capture_snapshot(name="mysnapshot") + summaries = monitor.list_snapshots() + assert len(summaries) == 1 + assert summaries[0].id == sid + assert summaries[0].name == "mysnapshot" + + +@pytest.mark.asyncio +async def test_list_snapshots_summary_fields(monitor: Monitor) -> None: + async def sleeper(): + await asyncio.sleep(5) + + t = monitor._monitored_loop.create_task(sleeper(), name="sleepy") + await asyncio.sleep(0.05) + sid = await monitor.capture_snapshot() + summaries = monitor.list_snapshots() + assert len(summaries) == 1 + summary = summaries[0] + assert summary.id == sid + assert summary.name is None + # at least the current test task + sleepy task + assert summary.running_count >= 2 + assert summary.terminated_count == 0 + + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_get_snapshot_returns_stored_object(monitor: Monitor) -> None: + sid = await monitor.capture_snapshot(name="foo") + snapshot = monitor.get_snapshot(sid) + assert snapshot.id == sid + assert snapshot.name == "foo" + + +@pytest.mark.asyncio +async def test_get_snapshot_missing_raises_keyerror(monitor: Monitor) -> None: + with pytest.raises(KeyError): + monitor.get_snapshot(12345) + + +@pytest.mark.asyncio +async def test_delete_snapshot(monitor: Monitor) -> None: + sid = await monitor.capture_snapshot() + monitor.delete_snapshot(sid) + with pytest.raises(KeyError): + monitor.get_snapshot(sid) + assert monitor.list_snapshots() == [] + + +@pytest.mark.asyncio +async def test_delete_snapshot_missing_raises_keyerror(monitor: Monitor) -> None: + with pytest.raises(KeyError): + monitor.delete_snapshot(12345) + + +@pytest.mark.asyncio +async def test_format_snapshot_task_list_missing_raises_keyerror( + monitor: Monitor, +) -> None: + with pytest.raises(KeyError): + monitor.format_snapshot_task_list(12345) + + +@pytest.mark.asyncio +async def test_format_snapshot_terminated_task_list_missing_raises_keyerror( + monitor: Monitor, +) -> None: + with pytest.raises(KeyError): + monitor.format_snapshot_terminated_task_list(12345) + + +@pytest.mark.asyncio +async def test_format_snapshot_task_stack_missing_snapshot_raises_keyerror( + monitor: Monitor, +) -> None: + with pytest.raises(KeyError): + monitor.format_snapshot_task_stack(12345, 1) + + +@pytest.mark.asyncio +async def test_format_snapshot_task_stack_missing_task_raises_keyerror( + monitor: Monitor, +) -> None: + sid = await monitor.capture_snapshot() + with pytest.raises(KeyError): + monitor.format_snapshot_task_stack(sid, 999999999) + + +@pytest.mark.asyncio +async def test_format_snapshot_diff_missing_raises_keyerror(monitor: Monitor) -> None: + sid = await monitor.capture_snapshot() + with pytest.raises(KeyError): + monitor.format_snapshot_diff(sid, 999999) + with pytest.raises(KeyError): + monitor.format_snapshot_diff(999999, sid) + + +@pytest.mark.asyncio +async def test_snapshot_freezes_running_task_membership(monitor: Monitor) -> None: + async def sleeper(): + await asyncio.sleep(5) + + t1 = monitor._monitored_loop.create_task(sleeper(), name="frozen1") + await asyncio.sleep(0.05) + sid1 = await monitor.capture_snapshot() + + t2 = monitor._monitored_loop.create_task(sleeper(), name="frozen2") + await asyncio.sleep(0.05) + + # snapshot 1 should not see the task created after it was captured + names_in_snap1 = {t.name for t in monitor.format_snapshot_task_list(sid1)} + assert "frozen1" in names_in_snap1 + assert "frozen2" not in names_in_snap1 + + sid2 = await monitor.capture_snapshot() + names_in_snap2 = {t.name for t in monitor.format_snapshot_task_list(sid2)} + assert "frozen1" in names_in_snap2 + assert "frozen2" in names_in_snap2 + + t1.cancel() + t2.cancel() + await asyncio.gather(t1, t2, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_snapshot_freezes_terminated_task_membership( + monitor_with_task_factory: Monitor, +) -> None: + # Terminated tasks are only tracked when the task factory is hooked, + # since only TracedTask instances report their termination. + monitor = monitor_with_task_factory + + async def quick(): + await asyncio.sleep(0.01) + + t1 = monitor._monitored_loop.create_task(quick(), name="quick_frozen1") + await t1 + await asyncio.sleep(0.05) + sid1 = await monitor.capture_snapshot() + + t2 = monitor._monitored_loop.create_task(quick(), name="quick_frozen2") + await t2 + await asyncio.sleep(0.05) + + names_in_snap1 = { + t.name for t in monitor.format_snapshot_terminated_task_list(sid1) + } + assert "quick_frozen1" in names_in_snap1 + assert "quick_frozen2" not in names_in_snap1 + + sid2 = await monitor.capture_snapshot() + names_in_snap2 = { + t.name for t in monitor.format_snapshot_terminated_task_list(sid2) + } + assert "quick_frozen1" in names_in_snap2 + assert "quick_frozen2" in names_in_snap2 + + +@pytest.mark.asyncio +async def test_format_snapshot_diff_added_removed_common(monitor: Monitor) -> None: + async def sleeper(): + await asyncio.sleep(5) + + t1 = monitor._monitored_loop.create_task(sleeper(), name="stay") + t2 = monitor._monitored_loop.create_task(sleeper(), name="gone") + await asyncio.sleep(0.05) + sid1 = await monitor.capture_snapshot() + + t2.cancel() + await asyncio.gather(t2, return_exceptions=True) + t3 = monitor._monitored_loop.create_task(sleeper(), name="new") + await asyncio.sleep(0.05) + sid2 = await monitor.capture_snapshot() + + diff = monitor.format_snapshot_diff(sid1, sid2) + added_names = {t.name for t in diff.added} + removed_names = {t.name for t in diff.removed} + common_names = {t.name for t in diff.common} + + assert "new" in added_names + assert "gone" not in added_names + assert "gone" in removed_names + assert "stay" in common_names + assert "new" not in common_names + assert "gone" not in common_names + + t1.cancel() + t3.cancel() + await asyncio.gather(t1, t3, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_format_snapshot_diff_object_shape(monitor: Monitor) -> None: + sid1 = await monitor.capture_snapshot() + sid2 = await monitor.capture_snapshot() + diff = monitor.format_snapshot_diff(sid1, sid2) + assert hasattr(diff, "added") + assert hasattr(diff, "removed") + assert hasattr(diff, "common") + assert isinstance(diff.added, list) + assert isinstance(diff.removed, list) + assert isinstance(diff.common, list) + + +@pytest.mark.asyncio +async def test_snapshot_eviction_prefers_unnamed(monitor: Monitor) -> None: + monitor._max_snapshots = 3 + ids = [] + for _i in range(3): + ids.append(await monitor.capture_snapshot()) + named_id = await monitor.capture_snapshot(name="keep-me") + # Adding a 4th unnamed snapshot should evict the oldest *unnamed* one, + # even though the named one was captured most recently. + summaries = monitor.list_snapshots() + summary_ids = [s.id for s in summaries] + assert ids[0] not in summary_ids + assert named_id in summary_ids + assert len(summaries) == 3 + + +@pytest.mark.asyncio +async def test_snapshot_eviction_preserves_named_even_over_cap( + monitor: Monitor, +) -> None: + monitor._max_snapshots = 2 + for i in range(5): + await monitor.capture_snapshot(name=f"named-{i}") + summaries = monitor.list_snapshots() + # No unnamed snapshots exist to evict, so we exceed the cap. + assert len(summaries) == 5 + assert all(s.name is not None for s in summaries) + + +@pytest.mark.asyncio +async def test_snapshot_eviction_does_not_touch_named(monitor: Monitor) -> None: + monitor._max_snapshots = 1 + named_id = await monitor.capture_snapshot(name="persistent") + for _ in range(5): + await monitor.capture_snapshot() + summaries = monitor.list_snapshots() + summary_ids = {s.id for s in summaries} + assert named_id in summary_ids + # only the named one plus at most 1 unnamed should remain + assert len(summaries) <= 2 + + +@pytest.mark.asyncio +async def test_max_snapshots_constructor_default() -> None: + loop = asyncio.get_running_loop() + mon = Monitor(loop, console_enabled=False) + assert mon._max_snapshots == 10 + + +@pytest.mark.asyncio +async def test_max_snapshots_constructor_override() -> None: + loop = asyncio.get_running_loop() + mon = Monitor(loop, console_enabled=False, max_snapshots=2) + assert mon._max_snapshots == 2 + + +@pytest.mark.asyncio +async def test_start_monitor_max_snapshots(event_loop) -> None: + from aiomonitor import start_monitor + + with start_monitor(event_loop, console_enabled=False, max_snapshots=4) as m: + assert m._max_snapshots == 4 + + +# --------------------------------------------------------------------------- +# Timing/creation-location fidelity tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_snapshot_task_list_dashes_without_hook_task_factory( + monitor: Monitor, +) -> None: + # The default `monitor` fixture does not hook the task factory. + async def sleeper(): + await asyncio.sleep(5) + + t = monitor._monitored_loop.create_task(sleeper(), name="not_traced") + await asyncio.sleep(0.05) + sid = await monitor.capture_snapshot() + tasks = monitor.format_snapshot_task_list(sid) + matched = [x for x in tasks if x.name == "not_traced"] + assert len(matched) == 1 + assert matched[0].since == "-" + assert matched[0].created_location == "-" + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_snapshot_task_list_real_timing_with_hook_task_factory( + monitor_with_task_factory: Monitor, +) -> None: + monitor = monitor_with_task_factory + + async def sleeper(): + await asyncio.sleep(5) + + t = monitor._monitored_loop.create_task(sleeper(), name="traced") + await asyncio.sleep(0.05) + sid = await monitor.capture_snapshot() + tasks = monitor.format_snapshot_task_list(sid) + matched = [x for x in tasks if x.name == "traced"] + assert len(matched) == 1 + assert matched[0].since != "-" + assert matched[0].created_location != "-" + assert "test_snapshot.py" in matched[0].created_location + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_snapshot_task_stack_has_headers( + monitor_with_task_factory: Monitor, +) -> None: + monitor = monitor_with_task_factory + + async def sleeper(): + await asyncio.sleep(5) + + t = monitor._monitored_loop.create_task(sleeper(), name="traced") + await asyncio.sleep(0.05) + sid = await monitor.capture_snapshot() + stack = monitor.format_snapshot_task_stack(sid, id(t)) + assert any(item.type == "header" for item in stack) + assert any("traced" in item.content for item in stack if item.type == "header") + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_snapshot_handles_task_that_completed_after_capture( + monitor: Monitor, +) -> None: + """ + A task captured while running may complete by the time the snapshot is + inspected later; get_coro() can then return None for such tasks (this is + known to happen for some internal aiohttp tasks), and formatting must not + crash in that case. + """ + + async def sleeper(): + await asyncio.sleep(5) + + t = monitor._monitored_loop.create_task(sleeper(), name="will_finish") + await asyncio.sleep(0.02) + assert not t.done() + sid = await monitor.capture_snapshot() + + # Simulate the task's coroutine reference having been released, which is + # known to occur for some tasks once they are done. + with unittest.mock.patch.object(t, "get_coro", return_value=None): + tasks = monitor.format_snapshot_task_list(sid) + matched = [x for x in tasks if x.name == "will_finish"] + assert len(matched) == 1 + assert matched[0].coro == "" + + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +# --------------------------------------------------------------------------- +# CLI tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_cli_snapshot_save_and_list(monitor: Monitor) -> None: + resp = await invoke_command(monitor, ["snapshot", "list"]) + assert "0 snapshots saved" in resp + + resp = await invoke_command(monitor, ["snapshot", "save"]) + assert "Saved snapshot #1" in resp + + resp = await invoke_command(monitor, ["snap", "save", "--name", "checkpoint"]) + assert "Saved snapshot #2 (checkpoint)" in resp + + resp = await invoke_command(monitor, ["snap", "ls"]) + assert "2 snapshots saved" in resp + assert "checkpoint" in resp + + +@pytest.mark.asyncio +async def test_cli_snapshot_show(monitor: Monitor) -> None: + async def sleeper(): + await asyncio.sleep(5) + + t = monitor._monitored_loop.create_task(sleeper(), name="cli_task") + await asyncio.sleep(0.05) + resp = await invoke_command(monitor, ["snapshot", "save"]) + assert "Saved snapshot #1" in resp + + resp = await invoke_command(monitor, ["snapshot", "show", "1"]) + assert "cli_task" in resp + assert "tasks running" in resp + assert "tasks terminated" in resp + + resp = await invoke_command(monitor, ["snapshot", "show", "42"]) + assert "No snapshot 42" in resp + + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_cli_snapshot_where(monitor: Monitor) -> None: + async def sleeper(): + await asyncio.sleep(5) + + t = monitor._monitored_loop.create_task(sleeper(), name="cli_where_task") + t_id = id(t) + await asyncio.sleep(0.05) + await invoke_command(monitor, ["snapshot", "save"]) + + resp = await invoke_command(monitor, ["snapshot", "where", "1", str(t_id)]) + assert "Task" in resp + + resp = await invoke_command(monitor, ["snapshot", "where", "1", "999999999"]) + assert "No task 999999999 in snapshot 1" in resp + + resp = await invoke_command(monitor, ["snapshot", "where", "42", str(t_id)]) + assert f"No task {t_id} in snapshot 42" in resp + + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_cli_snapshot_diff(monitor: Monitor) -> None: + async def sleeper(): + await asyncio.sleep(5) + + t1 = monitor._monitored_loop.create_task(sleeper(), name="cli_diff_1") + await asyncio.sleep(0.05) + await invoke_command(monitor, ["snapshot", "save"]) # id 1 + + t2 = monitor._monitored_loop.create_task(sleeper(), name="cli_diff_2") + await asyncio.sleep(0.05) + await invoke_command(monitor, ["snapshot", "save"]) # id 2 + + resp = await invoke_command(monitor, ["snapshot", "diff", "1", "2"]) + assert "Added" in resp + assert "cli_diff_2" in resp + assert "Common" in resp + assert "cli_diff_1" in resp + + resp = await invoke_command(monitor, ["snapshot", "diff", "1", "999"]) + assert "No snapshot 1 or 999" in resp + + t1.cancel() + t2.cancel() + await asyncio.gather(t1, t2, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_cli_snapshot_delete(monitor: Monitor) -> None: + await invoke_command(monitor, ["snapshot", "save"]) + resp = await invoke_command(monitor, ["snapshot", "delete", "1"]) + assert "Deleted snapshot #1" in resp + + resp = await invoke_command(monitor, ["snapshot", "delete", "1"]) + assert "No snapshot 1" in resp + + await invoke_command(monitor, ["snapshot", "save"]) + resp = await invoke_command(monitor, ["snap", "rm", "2"]) + assert "Deleted snapshot #2" in resp + + await invoke_command(monitor, ["snapshot", "save"]) + resp = await invoke_command(monitor, ["snap", "del", "3"]) + assert "Deleted snapshot #3" in resp + + +@pytest.mark.asyncio +async def test_cli_snapshot_help_lists_group(monitor: Monitor) -> None: + resp = await invoke_command(monitor, ["help"]) + assert "snapshot" in resp + assert "snap" in resp + + +@pytest.mark.asyncio +async def test_cli_snapshot_completion(monitor: Monitor) -> None: + import click + + from aiomonitor.context import current_monitor + from aiomonitor.termui.completion import ( + complete_snapshot_id, + complete_snapshot_task_id, + ) + + async def sleeper(): + await asyncio.sleep(5) + + t = monitor._monitored_loop.create_task(sleeper(), name="completion_task") + await asyncio.sleep(0.05) + sid = await monitor.capture_snapshot() + + token = current_monitor.set(monitor) + try: + completions = complete_snapshot_id(cast(Any, None), cast(Any, None), "") + assert str(sid) in completions + + ctx = click.Context(click.Command("where")) + ctx.params["snapshot_id"] = str(sid) + completions = complete_snapshot_task_id(ctx, cast(Any, None), "") + assert str(id(t)) in completions + finally: + current_monitor.reset(token) + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_snapshot_completion_without_monitor_context() -> None: + from aiomonitor.termui.completion import ( + complete_snapshot_id, + complete_snapshot_task_id, + ) + + # current_monitor is not set in this test, so LookupError should be + # handled gracefully by returning an empty completion list. + assert complete_snapshot_id(cast(Any, None), cast(Any, None), "") == [] + assert complete_snapshot_task_id(cast(Any, None), cast(Any, None), "") == [] + + +@pytest.mark.asyncio +async def test_snapshot_task_completion_with_invalid_snapshot_id( + monitor: Monitor, +) -> None: + import click + + from aiomonitor.context import current_monitor + from aiomonitor.termui.completion import complete_snapshot_task_id + + token = current_monitor.set(monitor) + try: + # no snapshot_id in ctx.params at all + ctx = click.Context(click.Command("where")) + assert complete_snapshot_task_id(ctx, cast(Any, None), "") == [] + + # snapshot_id present but does not correspond to an existing snapshot + ctx = click.Context(click.Command("where")) + ctx.params["snapshot_id"] = "123456" + assert complete_snapshot_task_id(ctx, cast(Any, None), "") == [] + + # snapshot_id present but not an integer + ctx = click.Context(click.Command("where")) + ctx.params["snapshot_id"] = "not-an-int" + assert complete_snapshot_task_id(ctx, cast(Any, None), "") == [] + finally: + current_monitor.reset(token) diff --git a/tests/test_webui_snapshot.py b/tests/test_webui_snapshot.py new file mode 100644 index 0000000..72fd07d --- /dev/null +++ b/tests/test_webui_snapshot.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from aiomonitor import Monitor +from aiomonitor.webui.app import init_webui + + +@pytest.fixture +async def webui_client(aiohttp_client, monitor_with_task_factory: Monitor): + """ + A test client wired to the same webui app + monitor combination used by + the live monitor, but served through an ephemeral aiohttp test server + instead of the monitor's own background thread/port. + """ + app = await init_webui(monitor_with_task_factory) + client = await aiohttp_client(app) + return client + + +@pytest.mark.asyncio +async def test_api_snapshot_save_returns_id( + webui_client, monitor_with_task_factory: Monitor +): + resp = await webui_client.post("/api/snapshot/save") + assert resp.status == 200 + data = await resp.json() + assert data == {"id": 1} + + resp = await webui_client.post("/api/snapshot/save", data={"name": "hello"}) + assert resp.status == 200 + data = await resp.json() + assert data == {"id": 2} + assert monitor_with_task_factory.get_snapshot(2).name == "hello" + + +@pytest.mark.asyncio +async def test_api_snapshot_list(webui_client, monitor_with_task_factory: Monitor): + resp = await webui_client.get("/api/snapshot/list") + assert resp.status == 200 + data = await resp.json() + assert data == {"snapshots": []} + + await monitor_with_task_factory.capture_snapshot() + await monitor_with_task_factory.capture_snapshot(name="second") + + resp = await webui_client.get("/api/snapshot/list") + data = await resp.json() + assert len(data["snapshots"]) == 2 + assert data["snapshots"][0]["id"] == 1 + assert data["snapshots"][0]["name"] is None + assert data["snapshots"][1]["id"] == 2 + assert data["snapshots"][1]["name"] == "second" + assert "running_count" in data["snapshots"][0] + assert "terminated_count" in data["snapshots"][0] + + +@pytest.mark.asyncio +async def test_api_snapshot_tasks_running_and_terminated( + webui_client, monitor_with_task_factory: Monitor +): + loop = monitor_with_task_factory._monitored_loop + + async def sleeper(): + await asyncio.sleep(5) + + async def quick(): + await asyncio.sleep(0.01) + + t1 = loop.create_task(sleeper(), name="apitask1") + t2 = loop.create_task(quick(), name="apiquick1") + await t2 + await asyncio.sleep(0.05) + + sid = await monitor_with_task_factory.capture_snapshot() + + resp = await webui_client.post( + "/api/snapshot/tasks", data={"snapshot_id": str(sid), "task_type": "running"} + ) + assert resp.status == 200 + data = await resp.json() + names = {t["name"] for t in data["tasks"]} + assert "apitask1" in names + for t in data["tasks"]: + assert t["snapshot_id"] == sid + + resp = await webui_client.post( + "/api/snapshot/tasks", + data={"snapshot_id": str(sid), "task_type": "terminated"}, + ) + assert resp.status == 200 + data = await resp.json() + names = {t["name"] for t in data["tasks"]} + assert "apiquick1" in names + + t1.cancel() + await asyncio.gather(t1, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_api_snapshot_tasks_missing_snapshot_404(webui_client): + resp = await webui_client.post( + "/api/snapshot/tasks", data={"snapshot_id": "999", "task_type": "running"} + ) + assert resp.status == 404 + data = await resp.json() + assert "999" in data["msg"] + + +@pytest.mark.asyncio +async def test_api_snapshot_trace(webui_client, monitor_with_task_factory: Monitor): + loop = monitor_with_task_factory._monitored_loop + + async def sleeper(): + await asyncio.sleep(5) + + t = loop.create_task(sleeper(), name="tracetask") + await asyncio.sleep(0.05) + sid = await monitor_with_task_factory.capture_snapshot() + + resp = await webui_client.post( + "/api/snapshot/trace", + data={"snapshot_id": str(sid), "task_id": str(id(t))}, + ) + assert resp.status == 200 + data = await resp.json() + assert any(item["type"] == "header" for item in data["trace"]) + + resp = await webui_client.post( + "/api/snapshot/trace", + data={"snapshot_id": str(sid), "task_id": "424242"}, + ) + assert resp.status == 404 + + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_api_snapshot_diff(webui_client, monitor_with_task_factory: Monitor): + loop = monitor_with_task_factory._monitored_loop + + async def sleeper(): + await asyncio.sleep(5) + + t1 = loop.create_task(sleeper(), name="diffstay") + await asyncio.sleep(0.05) + resp = await webui_client.post("/api/snapshot/save") + sid1 = (await resp.json())["id"] + + t2 = loop.create_task(sleeper(), name="diffnew") + await asyncio.sleep(0.05) + resp = await webui_client.post("/api/snapshot/save") + sid2 = (await resp.json())["id"] + + resp = await webui_client.post( + "/api/snapshot/diff", + data={"snapshot_id_1": str(sid1), "snapshot_id_2": str(sid2)}, + ) + assert resp.status == 200 + data = await resp.json() + assert "diffnew" in {t["name"] for t in data["added"]} + assert "diffstay" in {t["name"] for t in data["common"]} + assert "diffstay" not in {t["name"] for t in data["removed"]} + assert "diffnew" not in {t["name"] for t in data["removed"]} + + resp = await webui_client.post( + "/api/snapshot/diff", + data={"snapshot_id_1": str(sid1), "snapshot_id_2": "999"}, + ) + assert resp.status == 404 + + t1.cancel() + t2.cancel() + await asyncio.gather(t1, t2, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_api_snapshot_delete(webui_client, monitor_with_task_factory: Monitor): + resp = await webui_client.post("/api/snapshot/save") + sid = (await resp.json())["id"] + + resp = await webui_client.delete("/api/snapshot", params={"snapshot_id": str(sid)}) + assert resp.status == 200 + + # deleting again -> 404 (missing) + resp = await webui_client.delete("/api/snapshot", params={"snapshot_id": str(sid)}) + assert resp.status == 404 + + # no snapshot_id given at all -> 400 (bad request) + resp = await webui_client.delete("/api/snapshot") + assert resp.status == 400 + + +@pytest.mark.asyncio +async def test_snapshots_nav_page(webui_client): + resp = await webui_client.get("/snapshots") + assert resp.status == 200 + text = await resp.text() + assert "Snapshots" in text + + # the nav page should show up in the layout navigation as well + resp = await webui_client.get("/") + text = await resp.text() + assert "/snapshots" in text + + +@pytest.mark.asyncio +async def test_trace_snapshot_page(webui_client, monitor_with_task_factory: Monitor): + loop = monitor_with_task_factory._monitored_loop + + async def sleeper(): + await asyncio.sleep(5) + + t = loop.create_task(sleeper(), name="pagetrace") + await asyncio.sleep(0.05) + sid = await monitor_with_task_factory.capture_snapshot() + + resp = await webui_client.get( + "/trace-snapshot", + params={"snapshot_id": str(sid), "task_id": str(id(t))}, + ) + assert resp.status == 200 + text = await resp.text() + assert "Stack of" in text + + resp = await webui_client.get( + "/trace-snapshot", + params={"snapshot_id": str(sid), "task_id": "424242"}, + ) + assert resp.status == 404 + + t.cancel() + await asyncio.gather(t, return_exceptions=True)