Coverage for /Users/rik/github/cgse/libs/cgse-common/src/egse/settings.py: 66%
130 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-09-15 11:57 +0200
1"""
2The Settings class handles user and configuration settings that are provided in
3a [`YAML`](http://yaml.org) file.
5The idea is that settings are grouped by components or any arbitrary grouping that makes sense for
6the application or for the user. Settings are also modular and provided by each package by means
7of entry-points. The Settings class can read from different YAML files.
9By default, settings are loaded from a file called `settings.yaml`, but this can be changed in the entry-point
10definition.
12The yaml configuration files are provided as entry points by the packages that specified an entry-point group
13'_cgse.settings_' in the `pyproject.toml`. The Settings dictionary (attrdict) is constructed from the configuration
14YAML files from each of the packages. Settings can be overwritten by the next package configuration file. So,
15make sure the group names in each package configuration file are unique.
17The YAML file is read and the configuration parameters for the given group are
18available as instance variables of the returned class.
20The intended use is as follows:
22```python
23from egse.settings import Settings
25dsi_settings = Settings.load("DSI")
27if 0x000C <= dsi_settings.RMAP_BASE_ADDRESS <= 0x00FF:
28 ... # do something here
29else:
30 raise RMAPError("Attempt to access outside the RMAP memory map.")
31```
33The above code reads the settings from the default YAML file for a group called `DSI`.
34The settings will then be available as variables of the returned class, in this case
35`dsi_settings`. The returned class is and behaves also like a dictionary, so you can
36check if a configuration parameter is defined like this:
38```python
39if "DSI_FEE_IP_ADDRESS" not in dsi_settings:
40 # define the IP address of the DSI
41```
42The YAML section for the above code looks like this:
44```text
45DSI:
47 # DSI Specific Settings
49 DSI_FEE_IP_ADDRESS 10.33.178.144 # IP address of the DSI EtherSpaceLink interface
50 LINK_SPEED: 100 # SpW link speed used for both up- and downlink
52 # RMAP Specific Settings
54 RMAP_BASE_ADDRESS: 0x00000000 # The start of the RMAP memory map managed by the FEE
55 RMAP_MEMORY_SIZE: 4096 # The size of the RMAP memory map managed by the FEE
56```
58When you want to read settings from another YAML file, specify the `filename=` keyword.
59If that file is located at a specific location, also use the `location=` keyword.
61 my_settings = Settings.load(filename="user.yaml", location="/Users/JohnDoe")
63The above code will read the YAML file from the given location and not from the entry-points.
65---
67"""
69from __future__ import annotations
71import logging
72import re
73from pathlib import Path
74from typing import Any
76import yaml # This module is provided by the pip package PyYaml - pip install pyyaml
78from egse.env import get_local_settings_env_name
79from egse.env import get_local_settings_path
80from egse.log import logger
81from egse.system import attrdict
82from egse.system import recursive_dict_update
84_HERE = Path(__file__).resolve().parent
87class SettingsError(Exception):
88 """A settings-specific error."""
90 pass
93# Fix the problem: YAML loads 5e-6 as string and not a number
94# https://stackoverflow.com/questions/30458977/yaml-loads-5e-6-as-string-and-not-a-number
96SAFE_LOADER = yaml.SafeLoader
97SAFE_LOADER.add_implicit_resolver(
98 "tag:yaml.org,2002:float",
99 re.compile(
100 """^(?:
101 [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
102 |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
103 |\\.[0-9_]+(?:[eE][-+][0-9]+)?
104 |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
105 |[-+]?\\.(?:inf|Inf|INF)
106 |\\.(?:nan|NaN|NAN))$""",
107 re.X,
108 ),
109 list("-+0123456789."),
110)
113def load_settings_file(path: Path, filename: str, force: bool = False) -> attrdict:
114 """
115 Loads the YAML configuration file that is located at `path / filename`.
117 Args:
118 path (PATH): the folder where the YAML file is located
119 filename (str): the name of the YAML configuration file
120 force (bool): force reloading, i.e. don't use the cached information
122 Raises:
123 SettingsError: when the configuration file doesn't exist or cannot be found or \
124 when there was an error reading the configuration file.
126 Returns:
127 A dictionary (attrdict) with all the settings from the given file.
129 Note:
130 in case of an empty configuration file, and empty dictionary \
131 is returned and a warning message is issued.
132 """
133 try:
134 yaml_document = read_configuration_file(path / filename, force=force)
135 settings = attrdict({name: value for name, value in yaml_document.items()})
136 except FileNotFoundError as exc:
137 raise SettingsError(f"The Settings YAML file '{filename}' is not found at {path!s}. ") from exc
139 if not settings: 139 ↛ 140line 139 didn't jump to line 140 because the condition on line 139 was never true
140 logger.warning(
141 f"The Settings YAML file '{filename}' at {path!s} is empty. "
142 f"No local settings were loaded, an empty dictionary is returned."
143 )
145 return settings
148def load_global_settings(entry_point: str = "cgse.settings", force: bool = False) -> attrdict:
149 """
150 Loads the settings that are defined by the given entry_point. The entry-points are defined in the
151 `pyproject.toml` files of the packages that export their global settings.
153 Args:
154 entry_point (str): the name of the entry-point group [default: 'cgse.settings']
155 force (bool): force reloading the settings, i.e. ignore the cache
157 Returns:
158 A dictionary (attrdict) containing a collection of all the settings exported by the packages \
159 through the given entry-point.
161 """
162 from egse.plugin import get_file_infos
164 ep_settings = get_file_infos(entry_point)
166 global_settings = attrdict(label="Settings")
168 for ep_name, (path, filename) in ep_settings.items():
169 settings = load_settings_file(path, filename, force)
170 recursive_dict_update(global_settings, settings)
172 return global_settings
175def load_local_settings(force: bool = False) -> attrdict:
176 """
177 Loads the local settings file that is defined from the environment variable *PROJECT*_LOCAL_SETTINGS (where
178 *PROJECT* is the name of your project, defined in the environment variable of the same name).
180 This function might return an empty dictionary when
182 - the local settings YAML file is empty
183 - the local settings environment variable is not defined.
185 in both cases a warning message is logged.
187 Raises:
188 SettingsError: when the local settings YAML file is not found. Check the *PROJECT*_LOCAL_SETTINGS \
189 environment variable.
191 Returns:
192 A dictionary (attrdict) with all local settings.
194 """
195 local_settings = attrdict()
197 local_settings_path = get_local_settings_path()
199 if local_settings_path: 199 ↛ 203line 199 didn't jump to line 203 because the condition on line 199 was always true
200 path = Path(local_settings_path)
201 local_settings = load_settings_file(path.parent, path.name, force)
203 return local_settings
206def read_configuration_file(filename: Path, *, force=False) -> dict:
207 """
208 Read the YAML input configuration file. The configuration file is only read
209 once and memoized as load optimization.
211 Args:
212 filename (Path): the fully qualified filename of the YAML file
213 force (bool): force reloading the file, even when it was memoized
215 Raises:
216 SettingsError: when there was an error reading the YAML file.
218 Returns:
219 a dictionary containing all the configuration settings from the YAML file.
220 """
221 filename = str(filename)
223 if force or not Settings.is_memoized(filename):
224 logger.debug(f"Parsing YAML configuration file {filename}.")
226 with open(filename, "r") as stream:
227 try:
228 yaml_document = yaml.load(stream, Loader=SAFE_LOADER)
229 except yaml.YAMLError as exc:
230 logger.error(exc)
231 raise SettingsError(f"Error loading YAML document {filename}") from exc
233 Settings.add_memoized(filename, yaml_document)
235 return Settings.get_memoized(filename) or {}
238class Settings:
239 """
240 The Settings class provides a load() method that loads configuration settings for a group
241 into a dynamically created class as instance variables.
242 """
244 __memoized_yaml = {} # Memoized settings yaml files
245 __profile = False # Used for profiling methods and functions
247 LOG_FORMAT_DEFAULT = "%(levelname)s:%(module)s:%(lineno)d:%(message)s"
248 LOG_FORMAT_FULL = "%(asctime)23s:%(levelname)8s:%(lineno)5d:%(name)-20s: %(message)s"
249 LOG_FORMAT_THREAD = "%(asctime)23s:%(levelname)7s:%(lineno)5d:%(name)-20s(%(threadName)-15s): %(message)s"
250 LOG_FORMAT_PROCESS = (
251 "%(asctime)23s:%(levelname)7s:%(lineno)5d:%(name)20s.%(funcName)-31s(%(processName)-20s): %(message)s"
252 )
253 LOG_FORMAT_DATE = "%d/%m/%Y %H:%M:%S"
255 @classmethod
256 def get_memoized_locations(cls) -> list:
257 return list(cls.__memoized_yaml.keys())
259 @classmethod
260 def is_memoized(cls, filename: str) -> bool:
261 return filename in cls.__memoized_yaml
263 @classmethod
264 def add_memoized(cls, filename: str, yaml_document: Any):
265 cls.__memoized_yaml[filename] = yaml_document
267 @classmethod
268 def get_memoized(cls, filename: str):
269 return cls.__memoized_yaml.get(filename)
271 @classmethod
272 def clear_memoized(cls):
273 cls.__memoized_yaml.clear()
275 @classmethod
276 def set_profiling(cls, flag):
277 cls.__profile = flag
279 @classmethod
280 def profiling(cls):
281 return cls.__profile
283 @staticmethod
284 def _load_all(
285 entry_point: str = "cgse.settings", add_local_settings: bool = False, force: bool = False
286 ) -> attrdict:
287 """
288 Loads all settings from all package with the entry point 'cgse.settings'
289 """
290 global_settings = load_global_settings(entry_point, force)
292 # Load the LOCAL settings YAML file
294 if add_local_settings:
295 local_settings = load_local_settings(force)
296 recursive_dict_update(global_settings, local_settings)
298 return global_settings
300 @staticmethod
301 def _load_group(
302 group_name: str, entry_point: str = "cgse.settings", add_local_settings: bool = False, force: bool = False
303 ) -> attrdict:
304 global_settings = load_global_settings(entry_point, force)
306 group_settings = attrdict(label=group_name)
308 if group_name in global_settings: 308 ↛ 313line 308 didn't jump to line 313 because the condition on line 308 was always true
309 group_settings = attrdict(
310 {name: value for name, value in global_settings[group_name].items()}, label=group_name
311 )
313 if add_local_settings: 313 ↛ 318line 313 didn't jump to line 318 because the condition on line 313 was always true
314 local_settings = load_local_settings(force)
315 if group_name in local_settings: 315 ↛ 316line 315 didn't jump to line 316 because the condition on line 315 was never true
316 recursive_dict_update(group_settings, local_settings[group_name])
318 if not group_settings: 318 ↛ 319line 318 didn't jump to line 319 because the condition on line 318 was never true
319 raise SettingsError(f"Group name '{group_name}' is not defined in the global nor in the local settings.")
321 return group_settings
323 @staticmethod
324 def _load_one(location: str, filename: str, force=False) -> attrdict:
325 return load_settings_file(Path(location).expanduser(), filename, force)
327 @classmethod
328 def load(
329 cls, group_name=None, filename="settings.yaml", location=None, *, add_local_settings=True, force=False
330 ) -> attrdict:
331 """
332 Load the settings for the given group. When no group is provided, the
333 complete configuration is returned.
335 The Settings are loaded from entry-points that are defined in each of the
336 packages that provide a Settings file.
338 If a location is explicitly provided, the Settings will be loaded from that
339 location, using the given filename or the default (which is settings.yaml).
341 Args:
342 group_name (str): the name of one of the main groups from the YAML file
343 filename (str): the name of the YAML file to read [default=settings.yaml]
344 location (str, Path): the path to the location of the YAML file
345 force (bool): force reloading the file
346 add_local_settings (bool): update the Settings with site specific local settings
348 Returns:
349 a dynamically created class with the configuration parameters as instance variables.
351 Raises:
352 SettingsError: when the group is not defined in the YAML file.
353 """
354 if group_name: 354 ↛ 356line 354 didn't jump to line 356 because the condition on line 354 was always true
355 return cls._load_group(group_name, add_local_settings=add_local_settings, force=force)
356 elif location:
357 return cls._load_one(location=location, filename=filename, force=force)
358 else:
359 return cls._load_all(add_local_settings=add_local_settings, force=force)
361 @classmethod
362 def to_string(cls):
363 """
364 Returns a simple string representation of the cached configuration of this Settings class.
365 """
366 memoized = cls.__memoized_yaml
368 msg = ""
369 for key in memoized.keys():
370 msg += f"YAML file: {key}\n"
371 for field in memoized[key].keys():
372 length = 60
373 line = str(memoized[key][field])
374 trunc = line[:length]
375 if len(line) > length:
376 trunc += " ..."
377 msg += f" {field}: {trunc}\n"
379 return msg.rstrip()
382def main(args: list | None = None): # pragma: no cover
383 # We provide convenience to inspect the settings by calling this module directly from Python.
384 #
385 # python -m egse.settings
386 #
387 # Use the '--help' option to see what your choices are.
389 logging.basicConfig(level=20)
391 import argparse
393 parser = argparse.ArgumentParser(
394 description=(
395 f"Print out the default Settings, updated with local settings if the "
396 f"{get_local_settings_env_name()} environment variable is set."
397 ),
398 )
399 parser.add_argument("--local", action="store_true", help="print only the local settings.")
400 parser.add_argument(
401 "--global", action="store_true", help="print only the global settings, don't include local settings."
402 )
403 parser.add_argument("--group", help="print only settings for this group")
404 args = parser.parse_args(args or [])
406 # The following import will activate the pretty printing of the AttributeDict
407 # through the __rich__ method.
409 from rich import print
411 if args.local:
412 location = get_local_settings_path()
413 if location:
414 location = str(Path(location).expanduser().resolve())
415 settings = Settings.load(filename=location)
416 print(settings)
417 print(f"Loaded from [purple]{location}.")
418 else:
419 print("[red]No local settings defined.")
420 else:
421 # if the global option is given we don't want to include local settings
422 add_local_settings = False if vars(args)["global"] else True
424 if args.group:
425 settings = Settings.load(args.group, add_local_settings=add_local_settings)
426 else:
427 settings = Settings.load(add_local_settings=add_local_settings)
428 print(settings)
429 print("[blue]Memoized locations:")
430 locations = Settings.get_memoized_locations()
431 print([str(loc) for loc in locations])
434def get_site_id() -> str:
435 site = Settings.load("SITE")
436 return site.ID
439# ignore_m_warning('egse.settings')
441if __name__ == "__main__": 441 ↛ 442line 441 didn't jump to line 442 because the condition on line 441 was never true
442 main()