Coverage for /Users/rik/github/cgse/libs/cgse-common/src/egse/system.py: 23%
774 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-09-15 11:57 +0200
1"""
2The system module defines convenience functions that provide information on system specific
3functionality like, file system interactions, timing, operating system interactions, etc.
5The module has external dependencies to:
7* __distro__: for determining the Linux distribution
8* __psutil__: for system statistics
9* __rich__: for console output
11"""
13from __future__ import annotations
15import asyncio
16import builtins
17import collections
18import contextlib
19import datetime
20import functools
21import importlib
22import importlib.metadata
23import importlib.util
24import inspect
25import itertools
26import logging
27import math
28import operator
29import os
30import platform # For getting the operating system name
31import re
32import shutil
33import socket
34import subprocess # For executing a shell command
35import sys
36import threading
37import time
38import warnings
39from collections import namedtuple
40from contextlib import contextmanager
41from io import SEEK_END
42from io import SEEK_SET
43from pathlib import Path
44from types import FunctionType
45from types import ModuleType
46from typing import Any
47from typing import Callable
48from typing import Iterable
49from typing import List
50from typing import Optional
51from typing import Tuple
52from typing import Type
53from typing import Union
55import distro # For determining the Linux distribution
56import psutil
57from rich.console import Console
58from rich.text import Text
59from rich.tree import Tree
60from typer.core import TyperCommand
62import signal
63from egse.log import logger
65EPOCH_1958_1970 = 378691200
66TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f%z"
69# ACKNOWLEDGEMENT: The class is based on the textual.timer.Timer class from the Textual project.
70class Periodic:
71 """A timer that periodically invokes a function in the background.
73 If no callback is provided, a warning message will be logged. In a future, we might send out an event to the
74 application (will need an event handler).
76 When the function execution takes longer then the interval there re several options:
78 - if skip is True (default) the interval will take precedence and the
79 Args:
80 interval: The time between timer events, in seconds.
81 name: A name to assign the event (for debugging), defaults to `Periodic#`.
82 callback: A optional callback to invoke when the event is handled.
83 repeat: The number of times to repeat the timer, or None to repeat forever.
84 skip: Enable skipping of scheduled function calls that couldn't be sent in time.
85 pause: Start the timer paused. Use `resume()` to activate the timer.
86 """
88 _periodic_count: int = 0
89 """The number of Periodic instances that are created."""
91 def __init__(
92 self,
93 interval: float,
94 *,
95 name: str | None = None,
96 callback: Callable = None,
97 repeat: int | None = None,
98 skip: bool = True,
99 pause: bool = False,
100 ) -> None:
101 self._interval = interval
102 self.name = f"Periodic#{self._periodic_count}" if name is None else name
103 self._periodic_count += 1
104 self._callback = callback
105 self._repeat = repeat
106 self._skip = skip
107 self._active = asyncio.Event()
108 self._task: asyncio.Task | None = None
109 self._reset: bool = False
110 self._logger = logging.getLogger("periodic")
111 if not pause:
112 self._active.set()
114 def start(self) -> None:
115 """Start the timer."""
116 self._task = asyncio.create_task(self._run_timer(), name=self.name)
118 def stop(self) -> None:
119 """Stop the timer."""
120 if self._task is None:
121 return
123 self._active.clear()
124 self._task.cancel()
125 self._task = None
127 def is_running(self):
128 return self._active.is_set() and self._task is not None
130 def is_paused(self):
131 return not self._active.is_set()
133 def pause(self) -> None:
134 """Pause the timer.
136 A paused timer will not send events until it is resumed.
137 """
138 self._active.clear()
140 def reset(self) -> None:
141 """Reset the timer, so it starts from the beginning."""
142 self._active.set()
143 self._reset = True
145 def resume(self) -> None:
146 """Resume a paused timer."""
147 self._active.set()
149 async def _run_timer(self) -> None:
150 """Run the timer task."""
151 try:
152 await self._run()
153 except asyncio.CancelledError:
154 pass
156 async def _run(self) -> None:
157 """Run the timer."""
158 count = 0
159 _repeat = self._repeat
160 _interval = self._interval
161 await self._active.wait()
162 start = time.monotonic()
164 while _repeat is None or count < _repeat:
165 next_timer = start + ((count + 1) * _interval)
166 now = time.monotonic()
167 # self._logger.debug(f"{count = }, {next_timer = }, {now = }")
168 if self._skip and next_timer < now:
169 count = int((now - start) / _interval)
170 # self._logger.debug(f"Recalculated {count = }, {now - start = }, {(now - start) / _interval = }")
171 continue
172 now = time.monotonic()
173 wait_time = max(0.0, next_timer - now)
174 await asyncio.sleep(wait_time)
175 count += 1
176 await self._active.wait()
177 if self._reset:
178 start = time.monotonic()
179 count = 0
180 self._reset = False
181 continue
183 await self._tick()
185 self.stop()
187 async def _tick(self) -> None:
188 """Triggers the Timer's action: either call its callback, or logs a message."""
190 if self._callback is None:
191 self._logger.warning(f"Periodic – No callback provided for interval timer {self.name}.")
192 return
194 try:
195 await await_me_maybe(self._callback)
196 except asyncio.CancelledError:
197 self._logger.debug("Caught CancelledError on callback function in Periodic.")
198 raise
199 except Exception as exc:
200 self._logger.error(f"{type(exc).__name__} caught: {exc}")
202 @property
203 def interval(self):
204 return self._interval
207def round_up(n: float | int, decimals: int = 0):
208 """
209 Round a number up to a specified number of decimal places.
211 This function rounds the input number upward (toward positive infinity)
212 regardless of the value of the digits being rounded. It uses math.ceil()
213 after multiplying by a power of 10 to achieve the specified precision.
215 Args:
216 n (float or int): The number to round up.
217 decimals (int, optional): The number of decimal places to round to.
218 Must be non-negative. Defaults to 0.
220 Returns:
221 float: The rounded number with the specified precision.
223 Examples:
224 >>> round_up(3.14159, 3)
225 3.142
226 >>> round_up(3.1409, 3)
227 3.141
228 >>> round_up(-3.14159, 3)
229 -3.141
230 >>> round_up(5, 2)
231 5.0
233 Note:
234 For negative numbers, "rounding up" means rounding toward zero,
235 so -3.14159 rounded up to 3 decimals is -3.141.
236 """
237 multiplier = 10**decimals
238 return math.ceil(n * multiplier) / multiplier
241async def await_me_maybe(callback: Callable, *params: object) -> Any:
242 """Invoke a callback with an arbitrary number of parameters.
244 The callback can be a coroutine (async def) or a plain old function.
245 The `await_me_maybe` awaits the result of the callback if it's an awaitable,
246 or simply returns the result if not.
248 Args:
249 callback: The callable to be invoked.
251 Returns:
252 The return value of the invoked callable.
253 """
254 result = callback(*params)
255 if inspect.isawaitable(result):
256 result = await result
257 return result
260class TyperAsyncCommand(TyperCommand):
261 """Runs an asyncio Typer command.
263 Example:
265 @add.command(cls=TyperAsyncCommand)
266 async def start():
267 ...
269 """
271 def __init__(self, *args, **kwargs) -> None:
272 super().__init__(*args, **kwargs)
274 old_callback = self.callback
276 def new_callback(*args, **kwargs):
277 return asyncio.run(old_callback(*args, **kwargs))
279 self.callback = new_callback
282@contextmanager
283def all_logging_disabled(highest_level=logging.CRITICAL, flag=True):
284 """
285 Context manager to temporarily disable logging messages during its execution.
287 Args:
288 highest_level (int, optional): The maximum logging level to be disabled.
289 Defaults to logging.CRITICAL.
290 Note: Adjust this only if a custom level greater than CRITICAL is defined.
291 flag (bool, optional): If True, disables all logging; if False, no changes are made.
292 Defaults to True.
294 Example:
295 ```python
296 with all_logging_disabled():
297 ... # Your code with logging messages disabled
298 ```
300 Note:
301 This context manager is designed to prevent any logging messages triggered during its body
302 from being processed. It temporarily disables logging and restores the previous state afterward.
303 """
304 # Code below is copied from https://gist.github.com/simon-weber/7853144
305 # two kind-of hacks here:
306 # * can't get the highest logging level in effect => delegate to the user
307 # * can't get the current module-level override => use an undocumented
308 # (but non-private!) interface
310 previous_level = logging.root.manager.disable
312 if flag:
313 logging.disable(highest_level)
315 try:
316 yield
317 finally:
318 logging.disable(previous_level)
321def get_active_loggers() -> dict:
322 """
323 Retrieves information about active loggers and their respective log levels.
325 Returns a dictionary where keys are the names of active loggers, and values
326 are the corresponding log levels in string format.
328 Returns:
329 dict: A dictionary mapping logger names to their log levels.
331 Note:
332 This function provides a snapshot of the currently active loggers and
333 their log levels at the time of the function call.
335 """
337 return {
338 name: logging.getLevelName(logging.getLogger(name).level) for name in sorted(logging.Logger.manager.loggerDict)
339 }
342# The code below was taken from https://stackoverflow.com/a/69639238/4609203
345def ignore_m_warning(modules=None):
346 """
347 Ignore RuntimeWarning by `runpy` that occurs when executing a module with `python -m package.module`,
348 while that module is also imported.
350 The original warning message is:
352 '<package.module>' found in sys.modules after import of package '<package'>,
353 but prior to execution of '<package.module>'
354 """
355 if not isinstance(modules, (list, tuple)): 355 ↛ 358line 355 didn't jump to line 358 because the condition on line 355 was always true
356 modules = [modules]
358 try:
359 import warnings
360 import re
362 msg = "'{module}' found in sys.modules after import of package"
363 for module in modules:
364 module_msg = re.escape(msg.format(module=module))
365 warnings.filterwarnings("ignore", message=module_msg, category=RuntimeWarning, module="runpy") # ignore -m
366 except (ImportError, KeyError, AttributeError, Exception):
367 pass
370def now(utc: bool = True):
371 """Returns a datetime object for the current time in UTC or local time."""
372 if utc:
373 return datetime.datetime.now(tz=datetime.timezone.utc)
374 else:
375 return datetime.datetime.now()
378def format_datetime(
379 dt: Union[str, datetime.datetime] = None, fmt: str = None, width: int = 6, precision: int = 3
380) -> str:
381 """Format a datetime as YYYY-mm-ddTHH:MM:SS.μs+0000.
383 If the given argument is not timezone aware, the last part, i.e. `+0000` will not be there.
385 If no argument is given, the timestamp is generated as
386 `datetime.datetime.now(tz=datetime.timezone.utc)`.
388 The `dt` argument can also be a string with the following values: today, yesterday, tomorrow,
389 and 'day before yesterday'. The format will then be '%Y%m%d' unless specified.
391 Optionally, a format string can be passed in to customize the formatting of the timestamp.
392 This format string will be used with the `strftime()` method and should obey those conventions.
394 Example:
395 ```python
396 >>> format_datetime(datetime.datetime(2020, 6, 13, 14, 45, 45, 696138))
397 '2020-06-13T14:45:45.696'
398 >>> format_datetime(datetime.datetime(2020, 6, 13, 14, 45, 45, 696138), precision=6)
399 '2020-06-13T14:45:45.696138'
400 >>> format_datetime(datetime.datetime(2020, 6, 13, 14, 45, 59, 999501), precision=3)
401 '2020-06-13T14:45:59.999'
402 >>> format_datetime(datetime.datetime(2020, 6, 13, 14, 45, 59, 999501), precision=6)
403 '2020-06-13T14:45:59.999501'
404 >>> _ = format_datetime()
405 ...
406 >>> format_datetime("yesterday")
407 '20220214'
408 >>> format_datetime("yesterday", fmt="%d/%m/%Y")
409 '14/02/2022'
410 ```
412 Args:
413 dt (datetime): a datetime object or an agreed string like yesterday, tomorrow, ...
414 fmt (str): a format string that is accepted by `strftime()`
415 width (int): the width to use for formatting the microseconds
416 precision (int): the precision for the microseconds
418 Returns:
419 a string representation of the current time in UTC, e.g. `2020-04-29T12:30:04.862+0000`.
421 Raises:
422 ValueError: will be raised when the given dt argument string is not understood.
423 """
424 dt = dt or datetime.datetime.now(tz=datetime.timezone.utc)
425 if isinstance(dt, str):
426 fmt = fmt or "%Y%m%d"
427 if dt.lower() == "yesterday":
428 dt = datetime.date.today() - datetime.timedelta(days=1)
429 elif dt.lower() == "today":
430 dt = datetime.date.today()
431 elif dt.lower() == "day before yesterday":
432 dt = datetime.date.today() - datetime.timedelta(days=2)
433 elif dt.lower() == "tomorrow":
434 dt = datetime.date.today() + datetime.timedelta(days=1)
435 else:
436 raise ValueError(f"Unknown date passed as an argument: {dt}")
438 if fmt:
439 timestamp = dt.strftime(fmt)
440 else:
441 width = min(width, precision)
442 timestamp = (
443 f"{dt.strftime('%Y-%m-%dT%H:%M')}:"
444 f"{dt.second:02d}.{dt.microsecond // 10 ** (6 - precision):0{width}d}{dt.strftime('%z')}"
445 )
447 return timestamp
450SECONDS_IN_A_DAY = 24 * 60 * 60
451SECONDS_IN_AN_HOUR = 60 * 60
452SECONDS_IN_A_MINUTE = 60
455def humanize_seconds(seconds: float, include_micro_seconds: bool = True) -> str:
456 """
457 The number of seconds is represented as `[#D]d [#H]h[#M]m[#S]s.MS` where:
459 * `#D` is the number of days if days > 0
460 * `#H` is the number of hours if hours > 0
461 * `#M` is the number of minutes if minutes > 0 or hours > 0
462 * `#S` is the number of seconds
463 * `MS` is the number of microseconds
465 Args:
466 seconds: the number of seconds
467 include_micro_seconds: True if microseconds shall be included
469 Example:
470 ```python
471 >>> humanize_seconds(20)
472 '20s.000'
473 >>> humanize_seconds(10*24*60*60)
474 '10d 00s.000'
475 >>> humanize_seconds(10*86400 + 3*3600 + 42.023)
476 '10d 03h00m42s.023'
477 >>> humanize_seconds(10*86400 + 3*3600 + 42.023, include_micro_seconds=False)
478 '10d 03h00m42s'
479 ```
481 Returns:
482 a string representation for the number of seconds.
483 """
484 micro_seconds = round((seconds - int(seconds)) * 1000)
485 rest = int(seconds)
487 days = rest // SECONDS_IN_A_DAY
488 rest -= SECONDS_IN_A_DAY * days
490 hours = rest // SECONDS_IN_AN_HOUR
491 rest -= SECONDS_IN_AN_HOUR * hours
493 minutes = rest // SECONDS_IN_A_MINUTE
494 rest -= SECONDS_IN_A_MINUTE * minutes
496 seconds = rest
498 result = ""
499 if days:
500 result += f"{days}d "
502 if hours:
503 result += f"{hours:02d}h"
505 if minutes or hours:
506 result += f"{minutes:02d}m"
508 result += f"{seconds:02d}s"
509 if include_micro_seconds:
510 result += f".{micro_seconds:03d}"
512 return result
515def str_to_datetime(datetime_string: str) -> datetime.datetime:
516 """
517 Convert the given string to a datetime object.
519 Args:
520 datetime_string: String representing a datetime, in the format `%Y-%m-%dT%H:%M:%S.%f%z`.
522 Returns:
523 a datetime object.
524 """
526 return datetime.datetime.strptime(datetime_string.strip("\r"), TIME_FORMAT)
529def duration(dt_start: str | datetime.datetime, dt_end: str | datetime.datetime) -> datetime.timedelta:
530 """
531 Returns a `timedelta` object with the duration, i.e. time difference between dt_start and dt_end.
533 Notes:
534 If you need the number of seconds of your measurement, use the `total_seconds()` method of
535 the timedelta object.
537 Even if you —by accident— switch the start and end time arguments, the duration will
538 be calculated as expected.
540 Args:
541 dt_start: start time of the measurement
542 dt_end: end time of the measurement
544 Returns:
545 The time difference (duration) between dt_start and dt_end.
546 """
547 if isinstance(dt_start, str):
548 dt_start = str_to_datetime(dt_start)
549 if isinstance(dt_end, str):
550 dt_end = str_to_datetime(dt_end)
552 return dt_end - dt_start if dt_end > dt_start else dt_start - dt_end
555def time_since_epoch_1958(datetime_string: str) -> float:
556 """
557 Calculate the time since epoch 1958 for the given string representation of a datetime.
559 Args:
560 datetime_string: String representing a datetime, in the format `%Y-%m-%dT%H:%M:%S.%f%z`.
562 Returns:
563 Time since the 1958 epoch [s].
564 """
566 time_since_epoch_1970 = str_to_datetime(datetime_string).timestamp() # Since Jan 1st, 1970, midnight
568 return time_since_epoch_1970 + EPOCH_1958_1970
571class Timer:
572 """
573 Context manager to benchmark some lines of code.
575 When the context exits, the elapsed time is sent to the default logger (level=INFO).
577 Elapsed time can be logged with the `log_elapsed()` method and requested in fractional seconds
578 by calling the class instance. When the contexts goes out of scope, the elapsed time will not
579 increase anymore.
581 Log messages are sent to the logger (including egse_logger for egse.system) and the logging
582 level can be passed in as an optional argument. Default logging level is INFO.
584 Args:
585 name (str): a name for the Timer, will be printed in the logging message
586 precision (int): the precision for the presentation of the elapsed time
587 (number of digits behind the comma)
588 log_level (int): the log level to report the timing [default=INFO]
590 Example:
591 ```Python
592 with Timer("Some calculation") as timer:
593 # do some calculations
594 timer.log_elapsed()
595 # do some more calculations
596 print(f"Elapsed seconds: {timer()}")
597 Elapsed seconds: ...
598 ```
599 """
601 def __init__(self, name="Timer", precision=3, log_level=logging.INFO):
602 self.name = name
603 self.precision = precision
604 self.log_level = log_level
605 caller_info = get_caller_info(level=2)
606 self.filename = caller_info.filename
607 self.func = caller_info.function
608 self.lineno = caller_info.lineno
610 def __enter__(self):
611 # start is a value containing the start time in fractional seconds
612 # end is a function which returns the time in fractional seconds
613 self.start = time.perf_counter()
614 self.end = time.perf_counter
615 self._last_elapsed = time.perf_counter()
616 return self
618 def __exit__(self, ty, val, tb):
619 # The context goes out of scope here and we fix the elapsed time
620 self._total_elapsed = time.perf_counter()
621 self._last_elapsed = self._total_elapsed
623 # Overwrite self.end() so that it always returns the fixed end time
624 self.end = self._end
626 logger.log(
627 self.log_level,
628 f"{self.name} [ {self.filename}:{self.func}:{self.lineno} ]: "
629 f"{self.end() - self.start:0.{self.precision}f} seconds",
630 )
631 return False
633 def __call__(self):
634 return self.end() - self.start
636 def log_elapsed(self):
637 """Sends the elapsed time info to the default logger."""
638 current_lap = self.end()
639 logger.log(
640 self.log_level,
641 f"{self.name} [ {self.func}:{self.lineno} ]: "
642 f"{current_lap - self.start:0.{self.precision}f} seconds elapsed, "
643 f"{current_lap - self._last_elapsed:0.{self.precision}f}s since last lap.",
644 )
645 self._last_elapsed = current_lap
647 def get_elapsed(self) -> float:
648 """Returns the elapsed time for this timer as a float in seconds."""
649 return self.end() - self.start
651 def _end(self):
652 return self._total_elapsed
655def ping(host, timeout: float = 3.0) -> bool:
656 """
657 Sends a ping request to the given host.
659 Remember that a host may not respond to a ping (ICMP) request even if the host name is valid.
661 Args:
662 host (str): hostname or IP address (as a string)
663 timeout (float): timeout in seconds
665 Returns:
666 True when host responds to a ping request.
668 Reference:
669 [SO – Pinging servers in Python](https://stackoverflow.com/a/32684938)
670 """
672 # Option for the number of packets as a function of
673 param = "-n" if platform.system().lower() == "windows" else "-c"
675 # Building the command. Ex: "ping -c 1 google.com"
676 command = ["ping", param, "1", host]
678 try:
679 return subprocess.call(command, stdout=subprocess.DEVNULL, timeout=timeout) == 0
680 except subprocess.TimeoutExpired:
681 logging.info(f"Ping to {host} timed out in {timeout} seconds.")
682 return False
685def get_host_ip() -> Optional[str]:
686 """Returns the IP address. If no IP address can be found, None will be returned and the caller can try
687 to use localhost."""
689 host_ip = None
691 # The following code needs internet access
693 try:
694 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
695 # sock.connect(("8.8.8.8", 80))
696 sock.connect(("10.255.255.255", 1))
697 host_ip = sock.getsockname()[0]
698 sock.close()
699 except Exception as exc:
700 logger.warning(f"{type(exc).__name__} caught: {exc}")
702 if host_ip:
703 return host_ip
705 # This may still return 127.0.0.1 when hostname is defined in /etc/hosts
706 try:
707 host_name = socket.gethostname()
708 host_ip = socket.gethostbyname(host_name)
709 return host_ip
710 except Exception as exc:
711 logger.warning(f"{type(exc).__name__} caught: {exc}")
713 return None
716def get_current_location():
717 """
718 Returns the location where this function is called, i.e. the filename, line number, and function name.
719 """
720 frame = inspect.currentframe().f_back
722 filename = inspect.getframeinfo(frame).filename
723 line_number = inspect.getframeinfo(frame).lineno
724 function_name = inspect.getframeinfo(frame).function
726 # Clean up to prevent reference cycles
727 del frame
729 return filename, line_number, function_name
732CallerInfo = namedtuple("CallerInfo", "filename function lineno")
735def get_caller_info(level=1) -> CallerInfo:
736 """
737 Returns the filename, function name and lineno of the caller.
739 The level indicates how many levels to go back in the stack.
740 When level is 0 information about this function will be returned. That is usually not
741 what you want so the default level is 1 which returns information about the function
742 where the call to `get_caller_info` was made.
744 Args:
745 level (int): the number of levels to go back in the stack
747 Returns:
748 a namedtuple: `CallerInfo['filename', 'function', 'lineno']`.
749 """
750 frame = inspect.currentframe()
751 for _ in range(level):
752 if frame.f_back is None:
753 break
754 frame = frame.f_back
755 frame_info = inspect.getframeinfo(frame)
757 return CallerInfo(frame_info.filename, frame_info.function, frame_info.lineno)
760def get_caller_breadcrumbs(prefix: str = "call stack: ", limit: int = 5, with_filename: bool = False) -> str:
761 """
762 Returns a string representing the calling sequence of this function. The string contains the calling sequence from
763 left to right. Each entry has the function name and the line number of the line being executed.
764 When the `with_filename` is `True`, also the filename is printed before the function name. If the file
765 is `__init__.py`, also the parent folder name is printed.
767 <filename>:<function name>[<lineno>] <— <filename>:<caller function name>[<lineno>]
769 Use this function for example if you need to find out when and where a function is called in your process.
771 Example:
772 ```text
773 state.py:load_setup[126] <- state.py:setup[103] <- spw.py:__str__[167] <- nfeesim.py:run[575]
774 ```
776 Args:
777 prefix: a prefix for the calling sequence [default='call stack: '].
778 limit: the maximum number of caller to go back up the calling stack [default=5].
779 with_filename: filename is included in the returned string when True [default=False].
781 Returns:
782 A string containing the calling sequence.
783 """
784 frame = inspect.currentframe()
785 msg = []
786 while (frame := frame.f_back) is not None:
787 fi = inspect.getframeinfo(frame)
788 if with_filename:
789 filename = Path(fi.filename)
790 if filename.name == "__init__.py":
791 filename = f"{filename.parent.name}/{filename.name}:"
792 else:
793 filename = f"{filename.name}:"
794 else:
795 filename = ""
796 msg.append(f"{filename}{fi.function}[{fi.lineno}]")
797 if (limit := limit - 1) == 0:
798 break
800 return prefix + " <- ".join(msg)
803def get_referenced_var_name(obj: Any) -> List[str]:
804 """
805 Returns a list of variable names that reference the given object.
806 The names can be both in the local and global namespace of the object.
808 Args:
809 obj (Any): object for which the variable names are returned
811 Returns:
812 a list of variable names.
813 """
814 frame = inspect.currentframe().f_back
815 f_locals = frame.f_locals
816 f_globals = frame.f_globals
817 if "self" in f_locals:
818 f_locals = frame.f_back.f_locals
819 name_set = [k for k, v in {**f_locals, **f_globals}.items() if v is obj]
820 return name_set or []
823class AttributeDict(dict):
824 """
825 This class is and acts like a dictionary but has the additional functionality
826 that all keys in the dictionary are also accessible as instance attributes.
828 >>> ad = AttributeDict({'a': 1, 'b': 2, 'c': 3})
830 >>> assert ad.a == ad['a']
831 >>> assert ad.b == ad['b']
832 >>> assert ad.c == ad['c']
834 Similarly, adding or defining attributes will make them also keys in the dict.
836 >>> ad.d = 4 # creates a new attribute
837 >>> print(ad['d'])
838 4
839 """
841 def __init__(self, *args, label: str = None, **kwargs):
842 super().__init__(*args, **kwargs)
843 self.__dict__["_label"] = label
845 __setattr__ = dict.__setitem__
846 __delattr__ = dict.__delitem__
848 @property
849 def label(self):
850 return self.__dict__["_label"]
852 def __getattr__(self, key):
853 try:
854 return self[key]
855 except KeyError:
856 raise AttributeError(key)
858 def __rich__(self) -> Tree:
859 label = self.__dict__["_label"] or "AttributeDict"
860 tree = Tree(label, guide_style="dim")
861 walk_dict_tree(self, tree, text_style="dark grey")
862 return tree
864 def __repr__(self):
865 # We only want the first 10 key:value pairs
867 count = 10
868 sub_msg = ", ".join(f"{k!r}:{v!r}" for k, v in itertools.islice(self.items(), 0, count))
870 lbl = f", label='{self.__dict__['_label']}'" if self.label else ""
872 # if we left out key:value pairs, print a ', ...' to indicate incompleteness
873 return self.__class__.__name__ + f"({{{sub_msg}{', ...' if len(self) > count else ''}}}{lbl})"
876attrdict = AttributeDict
877"""Shortcut for the AttributeDict class."""
880def walk_dict_tree(dictionary: dict, tree: Tree, text_style: str = "green"):
881 for k, v in dictionary.items():
882 if isinstance(v, dict):
883 branch = tree.add(f"[purple]{k}", style="", guide_style="dim")
884 walk_dict_tree(v, branch, text_style=text_style)
885 else:
886 text = Text.assemble((str(k), "medium_purple1"), ": ", (str(v), text_style))
887 tree.add(text)
890def recursive_dict_update(this: dict, other: dict) -> dict:
891 """
892 Recursively update a dictionary `this` with the content of another dictionary `other`.
894 Any key in `this` dictionary will be recursively updated with the value of the same key in the
895 `other` dictionary.
897 Please note that the update will be in-place, i.e. the `this` dictionaory will be
898 changed/updated.
899 ```python
900 >>> global_settings = {"A": "GA", "B": "GB", "C": "GC"}
901 >>> local_settings = {"B": "LB", "D": "LD"}
902 >>> {**global_settings, **local_settings}
903 {'A': 'GA', 'B': 'LB', 'C': 'GC', 'D': 'LD'}
905 >>> global_settings = {"A": "GA", "B": "GB", "C": "GC", "R": {"X": "GX", "Y": "GY"}}
906 >>> local_settings = {"B": "LB", "D": "LD", "R": {"Y": "LY"}}
907 >>> recursive_dict_update(global_settings, local_settings)
908 {'A': 'GA', 'B': 'LB', 'C': 'GC', 'R': {'X': 'GX', 'Y': 'LY'}, 'D': 'LD'}
910 >>> global_settings = {"A": {"B": {"C": {"D": 42}}}}
911 >>> local_settings = {"A": {"B": {"C": 13, "D": 73}}}
912 >>> recursive_dict_update(global_settings, local_settings)
913 {'A': {'B': {'C': 13, 'D': 73}}}
914 ```
916 Args:
917 this (dict): The origin dictionary
918 other (dict): Changes that shall be applied to `this`
920 Returns:
921 The original `this` dictionary with the recursive updates.
922 """
924 if not isinstance(this, dict) or not isinstance(other, dict): 924 ↛ 925line 924 didn't jump to line 925 because the condition on line 924 was never true
925 raise ValueError("Expected arguments of type dict.")
927 for key, value in other.items():
928 if isinstance(value, dict) and isinstance(this.get(key), dict):
929 this[key] = recursive_dict_update(this[key], other[key])
930 else:
931 this[key] = other[key]
933 return this
936def flatten_dict(source_dict: dict) -> dict:
937 """
938 Flatten the given dictionary concatenating the keys with a colon '`:`'.
940 Args:
941 source_dict: the original dictionary that will be flattened
943 Returns:
944 A new flattened dictionary.
946 Example:
947 ```python
948 >>> d = {"A": 1, "B": {"E": {"F": 2}}, "C": {"D": 3}}
949 >>> flatten_dict(d)
950 {'A': 1, 'B:E:F': 2, 'C:D': 3}
952 >>> d = {"A": 'a', "B": {"C": {"D": 'd', "E": 'e'}, "F": 'f'}}
953 >>> flatten_dict(d)
954 {'A': 'a', 'B:C:D': 'd', 'B:C:E': 'e', 'B:F': 'f'}
955 ```
956 """
958 def expand(key, value):
959 if isinstance(value, dict):
960 return [(key + ":" + k, v) for k, v in flatten_dict(value).items()]
961 else:
962 return [(key, value)]
964 items = [item for k, v in source_dict.items() for item in expand(k, v)]
966 return dict(items)
969def get_system_stats() -> dict:
970 """
971 Gather system information about the CPUs and memory usage and return a dictionary with the
972 following information:
974 * cpu_load: load average over a period of 1, 5,and 15 minutes given in in percentage
975 (i.e. related to the number of CPU cores that are installed on your system) [percentage]
976 * cpu_count: physical and logical CPU count, i.e. the number of CPU cores (incl. hyper-threads)
977 * total_ram: total physical ram available [bytes]
978 * avail_ram: the memory that can be given instantly to processes without the system going
979 into swap. This is calculated by summing different memory values depending on the platform
980 [bytes]
981 * boot_time: the system boot time expressed in seconds since the epoch [s]
982 * since: boot time of the system, aka Up time [str]
984 Returns:
985 a dictionary with CPU and memory statistics.
986 """
987 statistics = {}
989 # Get Physical and Logical CPU Count
991 physical_and_logical_cpu_count = psutil.cpu_count()
992 statistics["cpu_count"] = physical_and_logical_cpu_count
994 # Load average
995 # This is the average system load calculated over a given period of time of 1, 5 and 15 minutes.
996 #
997 # The numbers returned by psutil.getloadavg() only make sense if
998 # related to the number of CPU cores installed on the system.
999 #
1000 # Here we are converting the load average into percentage.
1001 # The higher the percentage the higher the load.
1003 cpu_load = [x / physical_and_logical_cpu_count * 100 for x in psutil.getloadavg()]
1004 statistics["cpu_load"] = cpu_load
1006 # Memory usage
1008 vmem = psutil.virtual_memory()
1010 statistics["total_ram"] = vmem.total
1011 statistics["avail_ram"] = vmem.available
1013 # boot_time = seconds since the epoch timezone
1014 # the Unix epoch is 00:00:00 UTC on 1 January 1970.
1016 boot_time = psutil.boot_time()
1017 statistics["boot_time"] = boot_time
1018 statistics["since"] = datetime.datetime.fromtimestamp(boot_time, tz=datetime.timezone.utc).strftime(
1019 "%Y-%m-%d %H:%M:%S"
1020 )
1022 return statistics
1025def get_system_name() -> str:
1026 """Returns the name of the system in lower case.
1028 Returns:
1029 name: 'linux', 'darwin', 'windows', ...
1030 """
1031 return platform.system().lower()
1034def get_os_name() -> str:
1035 """Returns the name of the OS in lower case.
1037 If no name could be determined, 'unknown' is returned.
1039 Returns:
1040 os: 'macos', 'centos'
1041 """
1042 sys_name = get_system_name()
1043 if sys_name == "darwin":
1044 return "macos"
1045 if sys_name == "linux":
1046 return distro.id().lower()
1047 if sys_name == "windows":
1048 return "windows"
1049 return "unknown"
1052def get_os_version() -> str:
1053 """Return the version of the OS.
1055 If no version could be determined, 'unknown' is returned.
1057 Returns:
1058 version: as '10.15' or '8.0' or 'unknown'
1059 """
1061 # Don't use `distro.version()` to get the macOS version. That function will return the version
1062 # of the Darwin kernel.
1064 os_name = get_os_name()
1065 sys_name = get_system_name()
1066 if os_name == "unknown":
1067 return "unknown"
1068 if os_name == "macos":
1069 version, _, _ = platform.mac_ver()
1070 return ".".join(version.split(".")[:2])
1071 if sys_name == "linux":
1072 return distro.version()
1074 # FIXME: add other OS here for their version number
1076 return "unknown"
1079def wait_until(
1080 condition: Callable, *args: list, interval: float = 0.1, timeout: float = 1.0, verbose: bool = False, **kwargs: dict
1081) -> bool:
1082 """
1083 Sleep until the given condition is fulfilled. The arguments are passed into the condition
1084 callable which is called in a while loop until the condition is met or the timeout is reached.
1086 Note that the condition can be a function, method or callable class object.
1087 An example of the latter is:
1089 ```python
1090 class SleepUntilCount:
1091 def __init__(self, end):
1092 self._end = end
1093 self._count = 0
1095 def __call__(self, *args, **kwargs):
1096 self._count += 1
1097 if self._count >= self._end:
1098 return True
1099 else:
1100 return False
1101 ```
1103 Args:
1104 condition: a callable that returns True when the condition is met, False otherwise
1105 interval: the sleep interval between condition checks [s, default=0.1]
1106 timeout: the period after which the function returns, even when the condition is
1107 not met [s, default=1]
1108 verbose: log debugging messages if True
1109 *args: any arguments that will be passed into the condition function
1110 **kwargs: any keyword arguments that will be passed into the condition function
1112 Returns:
1113 True when function timed out, False otherwise.
1114 """
1116 if inspect.isfunction(condition) or inspect.ismethod(condition):
1117 func_name = condition.__name__
1118 else:
1119 func_name = condition.__class__.__name__
1121 caller = get_caller_info(level=2)
1123 start = time.time()
1125 while not condition(*args, **kwargs):
1126 if time.time() - start > timeout:
1127 logger.warning(
1128 f"Timeout after {timeout} sec, from {caller.filename} at {caller.lineno}, {func_name}{args} not met."
1129 )
1130 return True
1131 time.sleep(interval)
1133 if verbose:
1134 logger.debug(f"wait_until finished successfully, {func_name}{args}{kwargs} is met.")
1136 return False
1139def waiting_for(
1140 condition: Callable, *args: list, interval: float = 0.1, timeout: float = 1.0, verbose: bool = False, **kwargs: dict
1141) -> float:
1142 """
1143 Sleep until the given condition is fulfilled. The arguments are passed into the condition
1144 callable which is called in a while loop until the condition is met or the timeout is reached.
1146 Note that the condition can be a function, method or callable class object.
1147 An example of the latter is:
1149 ```python
1150 class SleepUntilCount:
1151 def __init__(self, end):
1152 self._end = end
1153 self._count = 0
1155 def __call__(self, *args, **kwargs):
1156 self._count += 1
1157 if self._count >= self._end:
1158 return True
1159 else:
1160 return False
1161 ```
1163 Args:
1164 condition: a callable that returns True when the condition is met, False otherwise
1165 interval: the sleep interval between condition checks [s, default=0.1]
1166 timeout: the period after which the function returns, even when the condition is
1167 not met [s, default=1]
1168 verbose: log debugging messages if True
1169 *args: any arguments that will be passed into the condition function
1170 **kwargs: any keyword arguments that will be passed into the condition function
1172 Returns:
1173 The duration until the condition was met.
1175 Raises:
1176 TimeoutError: when the condition was not fulfilled within the timeout period.
1177 """
1179 if inspect.isfunction(condition) or inspect.ismethod(condition):
1180 func_name = condition.__name__
1181 else:
1182 func_name = condition.__class__.__name__
1184 caller = get_caller_info(level=2)
1186 start = time.time()
1188 while not condition(*args, **kwargs):
1189 if time.time() - start > timeout:
1190 raise TimeoutError(
1191 f"Timeout after {timeout} sec, from {caller.filename} at {caller.lineno}, {func_name}{args} not met."
1192 )
1193 time.sleep(interval)
1195 duration = time.time() - start
1197 if verbose:
1198 logger.debug(f"waiting_for finished successfully after {duration:.3f}s, {func_name}{args}{kwargs} is met.")
1200 return duration
1203def has_internet(host: str = "8.8.8.8", port: int = 53, timeout: float = 3.0):
1204 """Returns True if we have internet connection.
1206 Args:
1207 host: hostname or IP address [default: 8.8.8.8 (google-public-dns-a.google.com)]
1208 port: 53 [service: tcp]
1209 timeout: the time to block before failing on a connection
1211 Note:
1212 This might give the following error codes:
1214 * [Errno 51] Network is unreachable
1215 * [Errno 61] Connection refused (because the port is blocked?)
1216 * timed out
1218 Source: https://stackoverflow.com/a/33117579
1219 """
1220 try:
1221 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1222 s.settimeout(timeout)
1223 s.connect((host, port))
1224 return True
1225 except socket.error as ex:
1226 logging.info(f"No Internet: Unable to open socket to {host}:{port} [{ex}]")
1227 return False
1228 finally:
1229 if s is not None:
1230 s.close()
1233def do_every(
1234 period: float,
1235 func: Callable,
1236 *args: tuple[int, ...],
1237 count: int = None,
1238 setup_func: Callable = None,
1239 teardown_func: Callable = None,
1240 stop_event: threading.Event = None,
1241) -> None:
1242 """
1243 This method executes a function periodically, taking into account
1244 that the function that is executed will take time also and using a
1245 simple `sleep()` will cause a drift. This method will not drift.
1247 You can use this function in combination with the threading module
1248 to execute the function in the background, but be careful as the
1249 function `func` might not be thread safe.
1251 ```
1252 timer_thread = threading.Thread(target=do_every, args=(10, func))
1253 timer_thread.daemon = True
1254 timer_thread.start()
1255 ```
1257 The `setup_func` and `teardown` functions will be called before and after
1258 the loop that repeats the `func` function. This can be used e.g. for setting
1259 up and closing sockets.
1261 Apart from the `count`, the loop can also be stopped by passing a threading
1262 event and setting the `stop_event` when you want to terminate the thread.
1264 ```
1265 self._stop_event = threading.Event()
1267 timer_thread = threading.Thread(
1268 target=do_every,
1269 args=(interval, send_heartbeat),
1270 kwargs={
1271 'stop_event': self._stop_event,
1272 'setup_func': self._connect_hb_socket,
1273 'teardown_func': self._disconnect_hb_socket
1274 }
1275 )
1276 timer_thread.daemon = True
1277 timer_thread.start()
1279 ...
1281 self._stop_event.set()
1282 ```
1284 Args:
1285 period: a time interval between successive executions [seconds]
1286 func: the function to be executed
1287 *args: optional arguments to be passed to the function
1288 count: if you do not need an endless loop, provide the number of
1289 iterations, if count=0 the function will not be executed.
1290 setup_func: a function that will be called before going into the loop
1291 teardown_func: a function that will be called when the loop ended
1292 stop_event: use a threading event to stop the loop
1293 """
1295 # Code from SO:https://stackoverflow.com/a/28034554/4609203
1296 # The max in the yield line serves to protect sleep from negative numbers in case the
1297 # function being called takes longer than the period specified. In that case it would
1298 # execute immediately and make up the lost time in the timing of the next execution.
1300 def g_tick():
1301 next_time = time.time()
1302 while True:
1303 next_time += period
1304 yield max(next_time - time.time(), 0)
1306 g = g_tick()
1307 iteration = 0
1309 if stop_event is None:
1310 stop_event = threading.Event()
1312 if setup_func:
1313 setup_func()
1315 while not stop_event.is_set():
1316 if count is not None and iteration >= count:
1317 break
1318 # Wait for the timeout or until the stop_event is set
1319 # The wait functions returns True only when the event is set and returns False on a timeout
1320 if stop_event.wait(timeout=next(g)):
1321 break
1322 func(*args)
1323 iteration += 1
1325 if teardown_func:
1326 teardown_func()
1329@contextlib.contextmanager
1330def chdir(dirname=None):
1331 """
1332 Context manager to temporarily change directory.
1334 Args:
1335 dirname (str or Path): temporary folder name to switch to within the context
1337 Example:
1338 ```python
1339 with chdir('/tmp'):
1340 ... # do stuff in this writable /tmp folder
1341 ```
1342 """
1343 current_dir = os.getcwd()
1344 try:
1345 if dirname is not None:
1346 os.chdir(dirname)
1347 yield
1348 finally:
1349 os.chdir(current_dir)
1352@contextlib.contextmanager
1353def env_var(**kwargs: dict[str, str]):
1354 """
1355 Context manager to run some code that need alternate settings for environment variables.
1357 Args:
1358 **kwargs: dictionary with environment variables that are needed
1360 Example:
1361 ```python
1362 with env_var(PLATO_DATA_STORAGE_LOCATION="/Users/rik/data"):
1363 # do stuff that needs these alternate setting
1364 ...
1365 ```
1366 """
1367 saved_env = {}
1369 for k, v in kwargs.items():
1370 saved_env[k] = os.environ.get(k)
1371 if v is None:
1372 if k in os.environ:
1373 del os.environ[k]
1374 else:
1375 os.environ[k] = v
1377 yield
1379 for k, v in saved_env.items():
1380 if v is None:
1381 if k in os.environ:
1382 del os.environ[k]
1383 else:
1384 os.environ[k] = v
1387def filter_by_attr(elements: Iterable, **attrs: dict[str, Any]) -> List:
1388 """
1389 A helper that returns the elements from the iterable that meet all the traits passed in `attrs`.
1391 The attributes are compared to their value with the `operator.eq` function. However,
1392 when the given value for an attribute is a tuple, the first element in the tuple is
1393 considered a comparison function and the second value the actual value. The attribute
1394 is then compared to the value using this function.
1396 ```python
1397 result = filter_by_attr(setups, camera__model="EM", site_id=(is_in, ("CSL", "INTA")))
1398 ```
1399 The function `is_in` is defined as follows:
1400 ```python
1401 def is_in(a, b):
1402 return a in b
1403 ```
1404 but you can of course also use a lambda function: `lambda a, b: a in b`.
1406 One function is treated special, it is the built-in function `hasattr`. Using this function,
1407 the value can be `True` or `False`. Use this to return all elements in the iterable
1408 that have the attribute, or not. The following example returns all Setups where the
1409 `gse.ogse.fwc_factor` is not defined:
1410 ```python
1411 result = filter_by_attr(setups, camera__model="EM", gse__ogse__fwc_factor=(hasattr, False)))
1412 ```
1414 When multiple attributes are specified, they are checked using logical AND, not logical OR.
1415 Meaning they have to meet every attribute passed in and not one of them.
1417 To have a nested attribute search (i.e. search by `gse.hexapod.ID`) then
1418 pass in `gse__hexapod__ID` as the keyword argument.
1420 If nothing is found that matches the attributes passed, then an empty list is returned.
1422 When an attribute is not part of the iterated object, that attribute is silently ignored.
1424 Args:
1425 elements: An iterable to search through.
1426 attrs: Keyword arguments that denote attributes to search with.
1427 """
1429 # This code is based on and originates from the get(iterable, **attr) function in the
1430 # discord/utils.py package (https://github.com/Rapptz/discord.py). After my own version,
1431 # Ruud van der Ham, improved the code drastically to the version it is now.
1433 def check(attr_, func, value_, el):
1434 try:
1435 a = operator.attrgetter(attr_)(el)
1436 return value_ if func is hasattr else func(a, value_)
1437 except AttributeError:
1438 return not value_ if func is hasattr else False
1440 attr_func_values = []
1441 for attr, value in attrs.items():
1442 if not (isinstance(value, (tuple, list)) and len(value) == 2 and callable(value[0])):
1443 value = (operator.eq, value)
1444 attr_func_values.append((attr.replace("__", "."), *value))
1446 return [el for el in elements if all(check(attr, func, value, el) for attr, func, value in attr_func_values)]
1449def replace_environment_variable(input_string: str):
1450 """
1451 Returns the `input_string` with all occurrences of ENV['var'].
1453 ```python
1454 >>> replace_environment_variable("ENV['HOME']/data/CSL")
1455 '/Users/rik/data/CSL'
1456 ```
1458 Args:
1459 input_string (str): the string to replace
1460 Returns:
1461 The input string with the ENV['var'] replaced, or None when the environment variable
1462 doesn't exists.
1463 """
1465 match = re.search(r"(.*)ENV\[['\"](\w+)['\"]\](.*)", input_string)
1466 if not match:
1467 return input_string
1468 pre_match = match.group(1)
1469 var = match.group(2)
1470 post_match = match.group(3)
1472 result = os.getenv(var, None)
1474 return pre_match + result + post_match if result else None
1477def read_last_line(filename: str | Path, max_line_length=5000):
1478 """Returns the last line of a (text) file.
1480 The argument `max_line_length` should be at least the length of the last line in the file,
1481 because this value is used to backtrack from the end of the file as an optimization.
1483 Args:
1484 filename (Path | str): the filename as a string or Path
1485 max_line_length (int): the maximum length of the lines in the file
1486 Returns:
1487 The last line in the file (whitespace stripped from the right). An empty string is returned
1488 when the file is empty, `None` is returned when the file doesn't exist.
1489 """
1490 filename = Path(filename)
1492 if not filename.exists():
1493 return None
1495 with filename.open("rb") as file:
1496 file.seek(0, 2) # 2 is relative to end of file
1497 size = file.tell()
1498 if size:
1499 file.seek(max(0, size - max_line_length))
1500 return file.readlines()[-1].decode("utf-8").rstrip("\n")
1501 else:
1502 return ""
1505def read_last_lines(filename: str | Path, num_lines: int) -> List[str]:
1506 """
1507 Return the last lines of a text file.
1509 Args:
1510 filename: Filename.
1511 num_lines: Number of lines at the back of the file that should be read and returned.
1513 Returns:
1514 Last lines of a text file as a list of strings. An empty list is returned
1515 when the file doesn't exist.
1517 Raises:
1518 AssertionError: when the requested num_lines is zero (0) or a negative number.
1519 """
1521 # See: https://www.geeksforgeeks.org/python-reading-last-n-lines-of-a-file/
1522 # (Method 3: Through exponential search)
1524 filename = Path(filename)
1526 sanity_check(num_lines >= 0, "the number of lines to read shall be a positive number or zero.")
1528 if not filename.exists():
1529 return []
1531 # Declaring variable to implement exponential search
1533 pos = num_lines + 1
1535 # List to store last N lines
1537 lines = []
1539 with open(filename) as f:
1540 size = f.seek(0, SEEK_END)
1541 while len(lines) <= num_lines:
1542 try:
1543 f.seek(size - pos, SEEK_SET)
1544 # ValueError: e.g. negative seek position
1545 except (IOError, ValueError):
1546 f.seek(0)
1547 break
1549 finally:
1550 lines = list(f)
1551 lines = [x.rstrip() for x in lines]
1553 # Increasing value of variable exponentially
1555 pos *= 2
1557 return lines[-num_lines:]
1560def is_namespace(module: str | ModuleType) -> bool:
1561 """
1562 Checks if a module represents a namespace package.
1564 Args:
1565 module: The module to be checked.
1567 Returns:
1568 True if the argument is a namespace package, False otherwise.
1570 Note:
1571 A namespace package is a special kind of package that spans multiple
1572 directories or locations, but doesn't contain an `__init__.py` file
1573 in any of its directories.
1575 Technically, a namespace package is defined as a module that has a
1576 `__path__` attribute and no `__file__` attribute.
1578 A namespace package allows for package portions to be distributed
1579 independently.
1581 """
1583 if isinstance(module, str): 1583 ↛ 1584line 1583 didn't jump to line 1584 because the condition on line 1583 was never true
1584 try:
1585 module = importlib.import_module(module)
1586 except (TypeError, ModuleNotFoundError):
1587 return False
1589 if hasattr(module, "__path__") and getattr(module, "__file__", None) is None: 1589 ↛ 1590line 1589 didn't jump to line 1590 because the condition on line 1589 was never true
1590 return True
1591 else:
1592 return False
1595def is_module(module: str | ModuleType) -> bool:
1596 """
1597 Returns True if the argument is a module or represents a module, False otherwise.
1599 Args:
1600 module: a module or module name.
1602 Returns:
1603 True if the argument is a module, False otherwise.
1604 """
1605 if isinstance(module, ModuleType):
1606 return True
1607 elif isinstance(module, str):
1608 try:
1609 module = importlib.import_module(module)
1610 except (TypeError, ModuleNotFoundError):
1611 return False
1612 else:
1613 return True
1614 else:
1615 return False
1618def get_package_description(package_name) -> str:
1619 """
1620 Returns the description of the package as specified in the projects metadata Summary.
1622 Example:
1623 ```python
1624 >>> get_package_description('cgse-common')
1625 'Software framework to support hardware testing'
1626 ```
1627 """
1628 try:
1629 # Get the metadata for the package
1630 metadata = importlib.metadata.metadata(package_name)
1631 # Extract the description
1632 description = metadata.get("Summary", "Description not found")
1633 return description
1634 except importlib.metadata.PackageNotFoundError:
1635 return "Package not found"
1638def get_package_location(module: str) -> List[Path]:
1639 """
1640 Retrieves the file system locations associated with a Python package.
1642 This function takes a module, module name, or fully qualified module path,
1643 and returns a list of Path objects representing the file system locations
1644 associated with the package. If the module is a namespace package, it returns
1645 the paths of all namespaces; otherwise, it returns the location of the module.
1647 Args:
1648 module (Union[FunctionType, ModuleType, str]): The module or module name to
1649 retrieve locations for.
1651 Returns:
1652 List[Path]: A list of Path objects representing the file system locations.
1654 Note:
1655 If the module is not found or is not a valid module, an empty list is returned.
1657 """
1659 if isinstance(module, FunctionType):
1660 module_name = module.__module__
1661 elif isinstance(module, ModuleType):
1662 module_name = module.__name__
1663 elif isinstance(module, str):
1664 module_name = module
1665 try:
1666 module = importlib.import_module(module)
1667 except TypeError:
1668 warnings.warn(f"The module is not found or is not valid: {module_name}.")
1669 return []
1670 else:
1671 return []
1673 if is_namespace(module):
1674 return [Path(location) for location in module.__path__]
1675 else:
1676 location = get_module_location(module)
1677 return [] if location is None else [location]
1680def get_module_location(arg: Any) -> Path | None:
1681 """
1682 Returns the location of the module as a Path object.
1684 The function can be given a string, which should then be a module name, or a function or module.
1685 For the latter two, the module name will be determined.
1687 Args:
1688 arg: can be one of the following: function, module, string
1690 Returns:
1691 The location of the module as a Path object or None when the location can not be determined or
1692 an invalid argument was provided.
1694 Example:
1695 ```python
1696 >>> get_module_location('egse')
1697 Path('/path/to/egse')
1699 >>> get_module_location(egse.system)
1700 Path('/path/to/egse/system')
1701 ```
1703 Note:
1704 If the module is not found or is not a valid module, None is returned.
1706 Warning:
1707 If the module is a namespace, None will be returned. Use the function
1708 [is_namespace()](system.md#egse.system.is_namespace) to determine if the 'module'
1709 is a namespace.
1711 """
1712 if isinstance(arg, FunctionType): 1712 ↛ 1714line 1712 didn't jump to line 1714 because the condition on line 1712 was never true
1713 # print(f"func: {arg = }, {arg.__module__ = }")
1714 module_name = arg.__module__
1715 elif isinstance(arg, ModuleType): 1715 ↛ 1717line 1715 didn't jump to line 1717 because the condition on line 1715 was never true
1716 # print(f"mod: {arg = }, {arg.__file__ = }")
1717 module_name = arg.__name__
1718 elif isinstance(arg, str): 1718 ↛ 1722line 1718 didn't jump to line 1722 because the condition on line 1718 was always true
1719 # print(f"str: {arg = }")
1720 module_name = arg
1721 else:
1722 return None
1724 # print(f"{module_name = }")
1726 try:
1727 module = importlib.import_module(module_name)
1728 except TypeError:
1729 return None
1731 if is_namespace(module): 1731 ↛ 1733line 1731 didn't jump to line 1733 because the condition on line 1731 was never true
1732 # print(f"{module = }")
1733 return None
1735 location = Path(module.__file__)
1737 if location.is_dir(): 1737 ↛ 1738line 1737 didn't jump to line 1738 because the condition on line 1737 was never true
1738 return location.resolve()
1739 elif location.is_file(): 1739 ↛ 1743line 1739 didn't jump to line 1743 because the condition on line 1739 was always true
1740 return location.parent.resolve()
1741 else:
1742 # print(f"Unknown {location = }")
1743 return None
1746def get_full_classname(obj: object) -> str:
1747 """
1748 Returns the fully qualified class name for the given object.
1750 Args:
1751 obj (object): The object for which to retrieve the fully qualified class name.
1753 Returns:
1754 str: The fully qualified class name, including the module.
1756 Example:
1757 ```python
1758 >>> get_full_classname("example")
1759 'builtins.str'
1761 >>> get_full_classname(42)
1762 'builtins.int'
1763 ```
1765 Note:
1766 The function considers various scenarios, such as objects being classes,
1767 built-ins, or literals like int, float, or complex numbers.
1769 """
1771 if type(obj) is type or obj.__class__.__module__ == str.__module__:
1772 try:
1773 module = obj.__module__
1774 name = obj.__qualname__
1775 except (TypeError, AttributeError):
1776 module = type(obj).__module__
1777 name = obj.__class__.__qualname__
1778 else:
1779 module = obj.__class__.__module__
1780 name = obj.__class__.__qualname__
1782 return module + "." + name
1785def find_class(class_name: str) -> Type:
1786 """Find and returns a class based on the fully qualified name.
1788 A class name can be preceded with the string `class//`. This is used in YAML
1789 files where the class is then instantiated on load by the [Setup](setup.md#egse.setup.Setup).
1791 Args:
1792 class_name (str): a fully qualified name for the class
1794 Returns:
1795 The class object corresponding to the fully qualified class name.
1797 Raises:
1798 AttributeError: when the class is not found in the module.
1799 ValueError: when the class_name can not be parsed.
1800 ModuleNotFoundError: if the module could not be found.
1801 """
1802 if class_name.startswith("class//"):
1803 class_name = class_name[7:]
1805 module_name, class_name = class_name.rsplit(".", 1)
1806 module = importlib.import_module(module_name)
1807 return getattr(module, class_name)
1810def type_name(var):
1811 """Returns the name of the type of var."""
1812 return type(var).__name__
1815def check_argument_type(obj: object, name: str, target_class: Union[type, Tuple[type]], allow_none: bool = False):
1816 """
1817 Check that the given object is of a specific (sub)type of the given target_class.
1818 The `target_class` can be a tuple of types.
1820 Args:
1821 obj: any object
1822 name: the name of the object
1823 target_class: the required type of the object (can be a tuple of types)
1824 allow_none: True if the object can be None
1826 Raises:
1827 TypeError: when not of the required type or None when not allowed.
1828 """
1829 if obj is None and allow_none:
1830 return
1831 if obj is None:
1832 raise TypeError(f"The argument '{name}' cannot be None.")
1833 if not isinstance(obj, target_class):
1834 raise TypeError(f"The argument '{name}' must be of type {target_class}, but is {type(obj)}")
1837def check_str_for_slash(arg: str):
1838 """Check if there is a slash in the given string, and raise a ValueError if so.
1840 Raises:
1841 ValueError: if the string contains a slash '`/`'.
1842 """
1844 if "/" in arg:
1845 ValueError(f"The given argument can not contain slashes, {arg=}.")
1848def check_is_a_string(var: Any, allow_none=False):
1849 """
1850 Checks if the given variable is a string and raises a TypeError if the check fails.
1852 Args:
1853 var: The variable to be checked.
1854 allow_none (bool, optional): If True, allows the variable to be None without raising an error.
1855 Defaults to False.
1857 Raises:
1858 TypeError: If the variable is not a string or is None (when allow_none is False).
1860 Example:
1861 ```python
1862 check_is_a_string("example")
1863 ```
1865 Note:
1866 This function is designed to validate that the input variable is a string.
1867 If `allow_none` is set to True, it allows the variable to be None without raising an error.
1869 """
1871 if var is None and allow_none:
1872 return
1873 if var is None and not allow_none:
1874 raise TypeError("The given variable cannot be None.")
1875 if not isinstance(var, str):
1876 raise TypeError(f"var must be a string, however {type(var)=}")
1879def sanity_check(flag: bool, msg: str):
1880 """
1881 Checks a boolean flag and raises an AssertionError with the provided message if the check fails.
1883 This function serves as a replacement for the 'assert' statement in production code.
1884 Using this ensures that your checks are not removed during optimizations.
1886 Args:
1887 flag (bool): The boolean flag to be checked.
1888 msg (str): The message to be included in the AssertionError if the check fails.
1890 Raises:
1891 AssertionError: If the flag is False.
1893 Example:
1894 ```python
1895 >>> sanity_check(x > 0, "x must be greater than 0")
1896 ```
1898 Note:
1899 This function is designed for production code to perform runtime checks
1900 that won't be removed during optimizations.
1902 """
1904 if not flag:
1905 raise AssertionError(msg)
1908class NotSpecified:
1909 """
1910 Class for NOT_SPECIFIED constant.
1911 Is used so that a parameter can have a default value other than None.
1913 Evaluate to False when converted to boolean.
1914 """
1916 def __nonzero__(self):
1917 """Always returns False. Called when to converting to bool in Python 2."""
1918 return False
1920 def __bool__(self):
1921 """Always returns False. Called when to converting to bool in Python 3."""
1922 return False
1925NOT_SPECIFIED = NotSpecified()
1926"""The constant that defines a not-specified value. Intended use is as a sentinel object."""
1928# Do not try to catch SIGKILL (9) that will just terminate your script without any warning
1930SIGNAL_NAME = {
1931 1: "SIGHUP",
1932 2: "SIGINT",
1933 3: "SIGQUIT",
1934 6: "SIGABRT",
1935 15: "SIGTERM",
1936 30: "SIGUSR1",
1937 31: "SIGUSR2",
1938}
1939"""The signals that can be caught with the SignalCatcher."""
1942class SignalCatcher:
1943 """
1944 This class registers handler to signals. When a signal is caught, the handler is
1945 executed and a flag for termination or user action is set to True. Check for this
1946 flag in your application loop.
1948 - Termination signals: 1=HUP, 2=INT, 3=QUIT, 6=ABORT, 15=TERM
1949 - User signals: 30=USR1, 31=USR2
1950 """
1952 def __init__(self):
1953 self.term_signal_received = False
1954 self.user_signal_received = False
1955 self.term_signals = [1, 2, 3, 6, 15]
1956 self.user_signals = [30, 31]
1957 for signal_number in self.term_signals:
1958 signal.signal(signal_number, self.handler)
1959 for signal_number in self.user_signals:
1960 signal.signal(signal_number, self.handler)
1962 self._signal_number = None
1963 self._signal_name = None
1965 @property
1966 def signal_number(self):
1967 """The value of the signal that was caught."""
1968 return self._signal_number
1970 @property
1971 def signal_name(self):
1972 """The name of the signal that was caught."""
1973 return self._signal_name
1975 def handler(self, signal_number, frame):
1976 """Handle the known signals by setting the appropriate flag."""
1977 logger.warning(f"Received signal {SIGNAL_NAME[signal_number]} [{signal_number}].")
1978 if signal_number in self.term_signals:
1979 self.term_signal_received = True
1980 if signal_number in self.user_signals:
1981 self.user_signal_received = True
1982 self._signal_number = signal_number
1983 self._signal_name = SIGNAL_NAME[signal_number]
1985 def clear(self, term: bool = False):
1986 """
1987 Call this method to clear the user signal after handling.
1988 Termination signals are not cleared by default since the application is supposed to terminate.
1989 Pass in a `term=True` to also clear the TERM signals, e.g. when you want to ignore some
1990 TERM signals.
1991 """
1992 self.user_signal_received = False
1993 if term:
1994 self.term_signal_received = False
1995 self._signal_number = None
1996 self._signal_name = None
1999def is_in(a, b):
2000 """Returns result of `a in b`."""
2001 return a in b
2004def is_not_in(a, b):
2005 """Returns result of `a not in b`."""
2006 return a not in b
2009def is_in_ipython():
2010 """Returns True if the code is running in IPython."""
2011 return hasattr(builtins, "__IPYTHON__")
2014_function_timing = {}
2017def execution_time(func):
2018 """
2019 A decorator to save the execution time of the function. Use this decorator
2020 if you want —by default and always— have an idea of the average execution time
2021 of the given function.
2023 Use this in conjunction with the [get_average_execution_time()](system.md#egse.system.get_average_execution_time)
2024 function to retrieve the average execution time for the given function.
2025 """
2027 @functools.wraps(func)
2028 def wrapper(*args, **kwargs):
2029 return save_average_execution_time(func, *args, **kwargs)
2031 return wrapper
2034def save_average_execution_time(func: Callable, *args, **kwargs):
2035 """
2036 Executes the function 'func' with the given arguments and saves the execution time. All positional
2037 arguments (in args) and keyword arguments (in kwargs) are passed into the function. The execution
2038 time is saved in a deque of maximum 100 elements. When more times are added, the oldest times are
2039 discarded. This function is used in conjunction with the
2040 [get_average_execution_time()](system.md#egse.system.get_average_execution_time) function.
2041 """
2043 with Timer(log_level=logging.NOTSET) as timer:
2044 response = func(*args, **kwargs)
2046 if func not in _function_timing:
2047 _function_timing[func] = collections.deque(maxlen=100)
2049 _function_timing[func].append(timer.get_elapsed())
2051 return response
2054def get_average_execution_time(func: Callable) -> float:
2055 """
2056 Returns the average execution time of the given function. The function 'func' shall be previously executed using
2057 the [save_average_execution_time()](system.md#egse.system.save_average_execution_time) function which remembers the
2058 last
2059 100 execution times of the function.
2060 You can also decorate your function with [@execution_time](system.md#egse.system.execution_time) to permanently
2061 monitor it.
2062 The average time is a moving average over the last 100 times. If the function was never called before, 0.0 is
2063 returned.
2065 This function can be used when setting a frequency to execute a certain function. When the average execution time
2066 of the function is longer than the execution interval, the frequency shall be decreased or the process will get
2067 stalled.
2068 """
2070 # If the function was previously wrapped with the `@execution_time` wrapper, we need to get
2071 # to the original function object because that's the one that is saved.
2073 with contextlib.suppress(AttributeError):
2074 func = func.__wrapped__
2076 try:
2077 d = _function_timing[func]
2078 return sum(d) / len(d)
2079 except KeyError:
2080 return 0.0
2083def get_average_execution_times() -> dict:
2084 """
2085 Returns a dictionary with `key = <function name>` and `value = <average execution time>`, for all function that
2086 have been monitored in this process.
2087 """
2088 return {func.__name__: get_average_execution_time(func) for func in _function_timing}
2091def clear_average_execution_times():
2092 """Clear out all function timing for this process."""
2093 _function_timing.clear()
2096def get_system_architecture() -> str:
2097 """
2098 Returns the machine type. This is a string describing the processor architecture,
2099 like 'i386' or 'arm64', but the exact string is not defined. An empty string can be
2100 returned when the type cannot be determined.
2101 """
2102 return platform.machine()
2105def time_in_ms() -> int:
2106 """
2107 Returns the current time in milliseconds since the Epoch.
2109 Note:
2110 if you are looking for a high performance timer, you should really be using `perf_counter()`
2111 instead of this function.
2112 """
2113 return int(round(time.time() * 1000))
2116class Sentinel:
2117 """
2118 This Sentinel can be used as an alternative to None or other meaningful values in e.g. a function argument.
2120 Usually, a sensible default would be to use None, but if None is a valid input parameter, you can use a Sentinel
2121 object and check in the function if the argument value is a Sentinel object.
2123 Example:
2124 ```python
2125 def get_info(server_socket, timeout: int = Sentinel()):
2126 if isinstance(timeout, Sentinel):
2127 raise ValueError("You should enter a valid timeout or None")
2128 ```
2129 """
2131 def __repr__(self):
2132 return "A default Sentinel object."
2135def touch(path: Path | str):
2136 """
2137 Unix-like 'touch', i.e. create a file if it doesn't exist and set the modification time to the current time.
2139 Args:
2140 path: full path to the file, can start with `~` which is automatically expanded.
2141 """
2143 path = Path(path).expanduser().resolve()
2144 basedir = path.parent
2145 if not basedir.exists():
2146 basedir.mkdir(parents=True, exist_ok=True)
2148 with path.open("a"):
2149 os.utime(path)
2152def capture_rich_output(obj: Any, width: int = 120) -> str:
2153 """
2154 Capture the output of a Rich console print of the given object. If the object is a known Rich renderable or if
2155 the object implements the `__rich__()` method, the output string will contain escape sequences to format the
2156 output when printed to a terminal.
2158 This method is usually used to represent Rich output in a log file, e.g. to print a table in the log file.
2160 Args:
2161 obj: any object
2162 width: the console width to use, None for full width
2164 Returns:
2165 The output of the capture, a string that possibly contains escape sequences as a result of rendering rich text.
2166 """
2167 console = Console(width=width)
2169 with console.capture() as capture:
2170 console.print(obj)
2172 captured_output = capture.get()
2174 return captured_output
2177def log_rich_output(logger_: logging.Logger, level: int, obj: Any):
2178 console = Console(width=None)
2180 with console.capture() as capture:
2181 console.print() # start on a fresh line when logging
2182 console.print(obj)
2184 captured_output = capture.get()
2186 logger_.log(level, captured_output)
2189def is_package_installed(package_name):
2190 """Check if a package is installed."""
2191 return importlib.util.find_spec(package_name) is not None
2194def get_logging_level(level: str | int):
2195 """
2196 Convert a logging level to its integer representation.
2198 This function normalizes various logging level inputs (string names,
2199 integer values, or custom level strings) into their corresponding
2200 integer logging levels.
2202 Args:
2203 level (str | int): The logging level to convert. Can be:
2204 - Standard logging level name (e.g., 'DEBUG', 'INFO', 'WARNING')
2205 - Integer logging level (e.g., 10, 20, 30)
2206 - Custom level string (e.g., 'Level 25')
2208 Returns:
2209 int: The integer representation of the logging level.
2210 - Standard levels: DEBUG=10, INFO=20, WARNING=30, ERROR=40, CRITICAL=50
2211 - Custom levels: Extracted integer value or dynamically resolved value
2212 """
2214 log_level = logging.getLevelName(level)
2216 if isinstance(log_level, str):
2217 match = re.search(r"\d+", log_level)
2218 if match:
2219 int_level = level = int(match.group())
2220 else:
2221 int_level = getattr(logging, log_level)
2222 else:
2223 int_level = log_level
2225 return int_level
2228def camel_to_kebab(camel_str: str) -> str:
2229 """Convert a string in CamelCase to kebab-case."""
2231 # Handle sequences of uppercase letters followed by lowercase
2232 s1 = re.sub("([A-Z]+)([A-Z][a-z])", r"\1-\2", camel_str)
2234 # Handle lowercase/digit followed by uppercase
2235 s2 = re.sub("([a-z0-9])([A-Z])", r"\1-\2", s1)
2236 return s2.lower()
2239def camel_to_snake(camel_str: str) -> str:
2240 """Convert a string in CamelCase to snake_case."""
2242 # Handle sequences of uppercase letters followed by lowercase
2243 s1 = re.sub("([A-Z]+)([A-Z][a-z])", r"\1_\2", camel_str)
2245 # Handle lowercase/digit followed by uppercase
2246 s2 = re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1)
2247 return s2.lower()
2250def kebab_to_title(kebab_str: str) -> str:
2251 """Convert kebab-case to Title Case (each word capitalized)"""
2252 return kebab_str.replace("-", " ").title()
2255def snake_to_title(snake_str: str) -> str:
2256 """Convert snake_case to Title Case (each word capitalized)"""
2257 return snake_str.replace("_", " ").title()
2260def caffeinate(pid: int = None):
2261 """Prevent your macOS system from entering idle sleep while a process is running.
2263 This function uses the macOS 'caffeinate' utility to prevent the system from
2264 going to sleep due to inactivity. It's particularly useful for long-running
2265 background processes that may lose network connections or be interrupted
2266 when the system sleeps.
2268 The function only operates on macOS systems and silently does nothing on
2269 other operating systems.
2271 Args:
2272 pid (int, optional): Process ID to monitor. If provided, caffeinate will
2273 keep the system awake as long as the specified process is running.
2274 If None or 0, defaults to the current process ID (os.getpid()).
2276 Returns:
2277 None
2279 Raises:
2280 FileNotFoundError: If 'caffeinate' command is not found in PATH (shouldn't
2281 happen on standard macOS installations).
2282 OSError: If subprocess.Popen fails to start the caffeinate process.
2284 Example:
2285 >>> # Keep system awake while current process runs
2286 >>> caffeinate()
2288 >>> # Keep system awake while specific process runs
2289 >>> caffeinate(1234)
2291 Note:
2292 - Uses 'caffeinate -i -w <pid>' which prevents idle sleep (-i) and monitors
2293 a specific process (-w)
2294 - The caffeinate process will automatically terminate when the monitored
2295 process exits
2296 - On non-macOS systems, this function does nothing
2297 - Logs a warning message when caffeinate is started
2299 See Also:
2300 macOS caffeinate(8) man page for more details on the underlying utility.
2301 """
2302 if not pid:
2303 pid = os.getpid()
2305 if get_os_name() == "macos":
2306 logger.warning(f"Running 'caffeinate -i -w {pid}' on macOS to prevent the system from idle sleeping.")
2307 subprocess.Popen([shutil.which("caffeinate"), "-i", "-w", str(pid)])
2310ignore_m_warning("egse.system")
2312if __name__ == "__main__": 2312 ↛ 2313line 2312 didn't jump to line 2313 because the condition on line 2312 was never true
2313 print(f"Host IP: {get_host_ip()}")
2314 print(f"System name: {get_system_name()}")
2315 print(f"OS name: {get_os_name()}")
2316 print(f"OS version: {get_os_version()}")
2317 print(f"Architecture: {get_system_architecture()}")
2318 print(f"Python version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")
2319 print("Running in IPython") if is_in_ipython() else None