Coverage for fastblocks/actions/sync/settings.py: 20%
359 statements
« prev ^ index » next coverage.py v7.10.7, created at 2025-10-09 00:47 -0700
« prev ^ index » next coverage.py v7.10.7, created at 2025-10-09 00:47 -0700
1"""Settings file synchronization between filesystem and cloud storage.
3Settings sync is intentionally limited to filesystem and cloud storage only.
4Unlike templates, settings are not cached for security and consistency reasons.
5"""
7import typing as t
8from pathlib import Path
10import yaml
11from acb.debug import debug
12from anyio import Path as AsyncPath
14from .strategies import (
15 ConflictStrategy,
16 SyncDirection,
17 SyncResult,
18 SyncStrategy,
19 create_backup,
20 get_file_info,
21 resolve_conflict,
22 should_sync,
23)
26class SettingsSyncResult(SyncResult):
27 def __init__(
28 self,
29 *,
30 config_reloaded: list[str] | None = None,
31 adapters_affected: list[str] | None = None,
32 **kwargs: t.Any,
33 ) -> None:
34 super().__init__(**kwargs)
35 self.config_reloaded = config_reloaded if config_reloaded is not None else []
36 self.adapters_affected = (
37 adapters_affected if adapters_affected is not None else []
38 )
41async def sync_settings(
42 *,
43 settings_path: AsyncPath | None = None,
44 adapter_names: list[str] | None = None,
45 strategy: SyncStrategy | None = None,
46 storage_bucket: str | None = None,
47 reload_config: bool = True,
48) -> SettingsSyncResult:
49 config = _prepare_settings_sync_config(settings_path, strategy)
50 result = SettingsSyncResult()
52 if storage_bucket is None:
53 storage_bucket = await _get_default_settings_bucket()
55 storage = await _initialize_storage_only(result)
56 if not storage:
57 return result
59 settings_files = await _discover_settings_files(
60 config["settings_path"],
61 adapter_names,
62 )
63 if not settings_files:
64 debug("No settings files found to sync")
65 return result
67 debug(f"Found {len(settings_files)} settings files to sync")
69 await _sync_settings_files(
70 settings_files,
71 storage,
72 config["strategy"],
73 storage_bucket,
74 result,
75 )
77 await _handle_config_reload(reload_config, result)
79 debug(
80 f"Settings sync completed: {len(result.synced_items)} synced, {len(result.conflicts)} conflicts",
81 )
83 return result
86def _prepare_settings_sync_config(
87 settings_path: AsyncPath | None,
88 strategy: SyncStrategy | None,
89) -> dict[str, t.Any]:
90 return {
91 "settings_path": settings_path or AsyncPath("settings"),
92 "strategy": strategy or SyncStrategy(),
93 }
96async def _initialize_storage_only(result: SettingsSyncResult) -> t.Any | None:
97 try:
98 from acb.depends import depends
100 storage = depends.get("storage")
101 if not storage:
102 result.errors.append(Exception("Storage adapter not available"))
103 return None
105 return storage
106 except Exception as e:
107 result.errors.append(e)
108 return None
111async def _get_default_settings_bucket() -> str:
112 try:
113 storage_config_path = AsyncPath("settings/storage.yml")
114 if await storage_config_path.exists():
115 content = await storage_config_path.read_text()
116 config = yaml.safe_load(content)
117 if isinstance(config, dict):
118 bucket_name = t.cast(
119 str, config.get("buckets", {}).get("settings", "settings")
120 )
121 else:
122 bucket_name = "settings"
123 debug(f"Using settings bucket from config: {bucket_name}")
124 return bucket_name
125 except Exception as e:
126 debug(f"Could not load storage config, using default: {e}")
127 debug("Using fallback settings bucket: settings")
128 return "settings"
131async def _sync_settings_files(
132 settings_files: list[dict[str, t.Any]],
133 storage: t.Any,
134 strategy: SyncStrategy,
135 storage_bucket: str,
136 result: SettingsSyncResult,
137) -> None:
138 for settings_info in settings_files:
139 try:
140 file_result = await _sync_single_settings_file(
141 settings_info,
142 storage,
143 strategy,
144 storage_bucket,
145 )
146 _accumulate_settings_sync_results(file_result, result)
148 except Exception as e:
149 result.errors.append(e)
150 debug(f"Error syncing settings {settings_info['relative_path']}: {e}")
153def _accumulate_settings_sync_results(
154 file_result: dict[str, t.Any],
155 result: SettingsSyncResult,
156) -> None:
157 if file_result.get("synced"):
158 result.synced_items.extend(file_result["synced"])
159 result.adapters_affected.extend(file_result.get("adapters_affected", []))
160 if file_result.get("conflicts"):
161 result.conflicts.extend(file_result["conflicts"])
162 if file_result.get("errors"):
163 result.errors.extend(file_result["errors"])
164 if file_result.get("skipped"):
165 result.skipped.extend(file_result["skipped"])
166 if file_result.get("backed_up"):
167 result.backed_up.extend(file_result["backed_up"])
170async def _handle_config_reload(
171 reload_config: bool,
172 result: SettingsSyncResult,
173) -> None:
174 if reload_config and result.synced_items:
175 try:
176 await _reload_configuration(result.adapters_affected)
177 result.config_reloaded = result.adapters_affected.copy()
178 except Exception as e:
179 result.errors.append(e)
180 debug(f"Error reloading configuration: {e}")
183async def _discover_settings_files(
184 settings_path: AsyncPath,
185 adapter_names: list[str] | None = None,
186) -> list[dict[str, t.Any]]:
187 settings_files: list[dict[str, t.Any]] = []
189 if not await settings_path.exists():
190 debug(f"Settings path does not exist: {settings_path}")
191 return settings_files
193 for pattern in ("*.yml", "*.yaml"):
194 await _discover_files_with_pattern(
195 settings_path,
196 pattern,
197 adapter_names,
198 settings_files,
199 )
201 return settings_files
204async def _discover_files_with_pattern(
205 settings_path: AsyncPath,
206 pattern: str,
207 adapter_names: list[str] | None,
208 settings_files: list[dict[str, t.Any]],
209) -> None:
210 async for file_path in settings_path.rglob(pattern):
211 if await file_path.is_file():
212 await _process_settings_file(
213 file_path,
214 settings_path,
215 adapter_names,
216 settings_files,
217 )
220async def _process_settings_file(
221 file_path: AsyncPath,
222 settings_path: AsyncPath,
223 adapter_names: list[str] | None,
224 settings_files: list[dict[str, t.Any]],
225) -> None:
226 adapter_name = file_path.stem
228 if adapter_names and adapter_name not in adapter_names:
229 return
231 try:
232 rel_path = file_path.relative_to(settings_path)
233 settings_files.append(
234 {
235 "local_path": file_path,
236 "relative_path": rel_path,
237 "storage_path": str(rel_path),
238 "adapter_name": adapter_name,
239 },
240 )
241 except ValueError:
242 debug(f"Could not get relative path for {file_path}")
245async def _sync_single_settings_file(
246 settings_info: dict[str, t.Any],
247 storage: t.Any,
248 strategy: SyncStrategy,
249 storage_bucket: str,
250) -> dict[str, t.Any]:
251 local_path = settings_info["local_path"]
252 storage_path = settings_info["storage_path"]
253 adapter_name = settings_info["adapter_name"]
255 result = _create_sync_result()
257 try:
258 local_info, remote_info = await _get_file_infos(
259 local_path,
260 storage,
261 storage_bucket,
262 storage_path,
263 )
265 if not await _should_sync_file(
266 local_info,
267 remote_info,
268 strategy,
269 storage_path,
270 result,
271 ):
272 return result
274 if not await _validate_local_yaml(local_info, storage_path, result):
275 return result
277 await _execute_sync_operation(
278 local_path,
279 storage,
280 storage_bucket,
281 storage_path,
282 local_info,
283 remote_info,
284 strategy,
285 result,
286 )
288 if result["synced"]:
289 result["adapters_affected"].append(adapter_name)
291 except Exception as e:
292 result["errors"].append(e)
293 debug(f"Error in _sync_single_settings_file for {storage_path}: {e}")
295 return result
298def _create_sync_result() -> dict[str, t.Any]:
299 return {
300 "synced": [],
301 "conflicts": [],
302 "errors": [],
303 "skipped": [],
304 "backed_up": [],
305 "adapters_affected": [],
306 }
309async def _get_file_infos(
310 local_path: t.Any,
311 storage: t.Any,
312 storage_bucket: str,
313 storage_path: str,
314) -> tuple[dict[str, t.Any], dict[str, t.Any]]:
315 local_info = await get_file_info(Path(local_path))
316 remote_info = await _get_storage_file_info(storage, storage_bucket, storage_path)
317 return local_info, remote_info
320async def _should_sync_file(
321 local_info: dict[str, t.Any],
322 remote_info: dict[str, t.Any],
323 strategy: SyncStrategy,
324 storage_path: str,
325 result: dict[str, t.Any],
326) -> bool:
327 sync_needed, reason = should_sync(local_info, remote_info, strategy.direction)
328 if not sync_needed:
329 result["skipped"].append(f"{storage_path} ({reason})")
330 return False
332 debug(f"Syncing settings {storage_path}: {reason}")
333 return True
336async def _validate_local_yaml(
337 local_info: dict[str, t.Any],
338 storage_path: str,
339 result: dict[str, t.Any],
340) -> bool:
341 if local_info["exists"]:
342 try:
343 await _validate_yaml_content(local_info["content"])
344 except Exception as e:
345 result["errors"].append(f"Invalid YAML in {storage_path}: {e}")
346 return False
347 return True
350async def _execute_sync_operation(
351 local_path: t.Any,
352 storage: t.Any,
353 storage_bucket: str,
354 storage_path: str,
355 local_info: dict[str, t.Any],
356 remote_info: dict[str, t.Any],
357 strategy: SyncStrategy,
358 result: dict[str, t.Any],
359) -> None:
360 if _should_pull_settings(strategy, local_info, remote_info):
361 await _pull_settings(
362 local_path,
363 storage,
364 storage_bucket,
365 storage_path,
366 strategy,
367 result,
368 )
369 elif _should_push_settings(strategy, local_info, remote_info):
370 await _push_settings(
371 local_path,
372 storage,
373 storage_bucket,
374 storage_path,
375 strategy,
376 result,
377 )
378 elif _has_bidirectional_conflict(strategy, local_info, remote_info):
379 await _handle_settings_conflict(
380 local_path,
381 storage,
382 storage_bucket,
383 storage_path,
384 local_info,
385 remote_info,
386 strategy,
387 result,
388 )
391def _should_pull_settings(
392 strategy: SyncStrategy,
393 local_info: dict[str, t.Any],
394 remote_info: dict[str, t.Any],
395) -> bool:
396 return strategy.direction == SyncDirection.PULL or (
397 strategy.direction == SyncDirection.BIDIRECTIONAL
398 and remote_info["exists"]
399 and (not local_info["exists"] or remote_info["mtime"] > local_info["mtime"])
400 )
403def _should_push_settings(
404 strategy: SyncStrategy,
405 local_info: dict[str, t.Any],
406 remote_info: dict[str, t.Any],
407) -> bool:
408 return strategy.direction == SyncDirection.PUSH or (
409 strategy.direction == SyncDirection.BIDIRECTIONAL
410 and local_info["exists"]
411 and (not remote_info["exists"] or local_info["mtime"] > remote_info["mtime"])
412 )
415def _has_bidirectional_conflict(
416 strategy: SyncStrategy,
417 local_info: dict[str, t.Any],
418 remote_info: dict[str, t.Any],
419) -> bool:
420 return (
421 strategy.direction == SyncDirection.BIDIRECTIONAL
422 and local_info["exists"]
423 and remote_info["exists"]
424 )
427async def _get_storage_file_info(
428 storage: t.Any,
429 bucket: str,
430 file_path: str,
431) -> dict[str, t.Any]:
432 try:
433 bucket_obj = getattr(storage, bucket, None)
435 if not bucket_obj:
436 await storage._create_bucket(bucket)
437 bucket_obj = getattr(storage, bucket)
439 exists = await bucket_obj.exists(file_path)
441 if not exists:
442 return {
443 "exists": False,
444 "size": 0,
445 "mtime": 0,
446 "content_hash": None,
447 }
449 content = await bucket_obj.read(file_path)
450 metadata = await bucket_obj.stat(file_path)
452 import hashlib
454 content_hash = hashlib.blake2b(content).hexdigest()
456 return {
457 "exists": True,
458 "size": len(content),
459 "mtime": metadata.get("mtime", 0),
460 "content_hash": content_hash,
461 "content": content,
462 }
464 except Exception as e:
465 debug(f"Error getting storage file info for {file_path}: {e}")
466 return {
467 "exists": False,
468 "size": 0,
469 "mtime": 0,
470 "content_hash": None,
471 "error": str(e),
472 }
475async def _validate_yaml_content(content: bytes) -> None:
476 try:
477 import yaml
479 yaml.safe_load(content.decode())
480 except Exception as e:
481 msg = f"Invalid YAML content: {e}"
482 raise ValueError(msg)
485async def _pull_settings(
486 local_path: AsyncPath,
487 storage: t.Any,
488 bucket: str,
489 storage_path: str,
490 strategy: SyncStrategy,
491 result: dict[str, t.Any],
492) -> None:
493 try:
494 bucket_obj = getattr(storage, bucket)
496 if strategy.dry_run:
497 debug(f"DRY RUN: Would pull {storage_path} to {local_path}")
498 result["synced"].append(f"PULL(dry-run): {storage_path}")
499 return
501 if await local_path.exists() and strategy.backup_on_conflict:
502 backup_path = await create_backup(Path(local_path))
503 result["backed_up"].append(str(backup_path))
505 content = await bucket_obj.read(storage_path)
507 await _validate_yaml_content(content)
509 await local_path.parent.mkdir(parents=True, exist_ok=True)
511 await local_path.write_bytes(content)
513 result["synced"].append(f"PULL: {storage_path}")
514 debug(f"Pulled settings from storage: {storage_path}")
516 except Exception as e:
517 result["errors"].append(e)
518 debug(f"Error pulling settings {storage_path}: {e}")
521async def _push_settings(
522 local_path: AsyncPath,
523 storage: t.Any,
524 bucket: str,
525 storage_path: str,
526 strategy: SyncStrategy,
527 result: dict[str, t.Any],
528) -> None:
529 try:
530 bucket_obj = getattr(storage, bucket)
532 if strategy.dry_run:
533 debug(f"DRY RUN: Would push {local_path} to {storage_path}")
534 result["synced"].append(f"PUSH(dry-run): {storage_path}")
535 return
537 content = await local_path.read_bytes()
538 await _validate_yaml_content(content)
540 await bucket_obj.write(storage_path, content)
542 result["synced"].append(f"PUSH: {storage_path}")
543 debug(f"Pushed settings to storage: {storage_path}")
545 except Exception as e:
546 result["errors"].append(e)
547 debug(f"Error pushing settings {storage_path}: {e}")
550async def _handle_settings_conflict(
551 local_path: AsyncPath,
552 storage: t.Any,
553 bucket: str,
554 storage_path: str,
555 local_info: dict[str, t.Any],
556 remote_info: dict[str, t.Any],
557 strategy: SyncStrategy,
558 result: dict[str, t.Any],
559) -> None:
560 try:
561 if strategy.conflict_strategy == ConflictStrategy.MANUAL:
562 result["conflicts"].append(
563 {
564 "path": storage_path,
565 "local_mtime": local_info["mtime"],
566 "remote_mtime": remote_info["mtime"],
567 "reason": "manual_resolution_required",
568 },
569 )
570 return
572 try:
573 await _validate_yaml_content(local_info["content"])
574 await _validate_yaml_content(remote_info["content"])
575 except Exception as e:
576 result["errors"].append(f"Invalid YAML during conflict resolution: {e}")
577 return
579 resolved_content, resolution_reason = await resolve_conflict(
580 Path(local_path),
581 remote_info["content"],
582 local_info["content"],
583 strategy.conflict_strategy,
584 local_info["mtime"],
585 remote_info["mtime"],
586 )
588 if strategy.dry_run:
589 debug(
590 f"DRY RUN: Would resolve conflict for {storage_path}: {resolution_reason}",
591 )
592 result["synced"].append(
593 f"CONFLICT(dry-run): {storage_path} - {resolution_reason}",
594 )
595 return
597 if (
598 strategy.backup_on_conflict
599 or strategy.conflict_strategy == ConflictStrategy.BACKUP_BOTH
600 ):
601 backup_path = await create_backup(Path(local_path), "conflict")
602 result["backed_up"].append(str(backup_path))
604 if resolved_content == remote_info["content"]:
605 await local_path.write_bytes(resolved_content)
606 result["synced"].append(
607 f"CONFLICT->REMOTE: {storage_path} - {resolution_reason}",
608 )
609 else:
610 bucket_obj = getattr(storage, bucket)
611 await bucket_obj.write(storage_path, resolved_content)
612 result["synced"].append(
613 f"CONFLICT->LOCAL: {storage_path} - {resolution_reason}",
614 )
616 debug(f"Resolved settings conflict: {storage_path} - {resolution_reason}")
618 except Exception as e:
619 result["errors"].append(e)
620 result["conflicts"].append(
621 {
622 "path": storage_path,
623 "error": str(e),
624 "reason": "resolution_failed",
625 },
626 )
629async def _reload_configuration(adapter_names: list[str]) -> None:
630 try:
631 from acb.config import reload_config # type: ignore[attr-defined]
632 from acb.depends import depends
634 config = await reload_config()
635 depends.set("config", config)
636 debug(f"Reloaded configuration for adapters: {adapter_names}")
637 except Exception as e:
638 debug(f"Error reloading configuration: {e}")
639 raise
642async def backup_settings(
643 settings_path: AsyncPath | None = None,
644 backup_suffix: str | None = None,
645) -> dict[str, t.Any]:
646 settings_path = settings_path or AsyncPath("settings")
647 backup_suffix = backup_suffix or _generate_backup_suffix()
649 result = _create_backup_result()
651 try:
652 if not await settings_path.exists():
653 result["errors"].append(f"Settings path does not exist: {settings_path}")
654 return result
656 await _backup_files_with_patterns(settings_path, backup_suffix, result)
658 except Exception as e:
659 result["errors"].append(str(e))
660 debug(f"Error in backup_settings: {e}")
662 return result
665def _generate_backup_suffix() -> str:
666 import time
668 timestamp = int(time.time())
669 return f"backup_{timestamp}"
672def _create_backup_result() -> dict[str, t.Any]:
673 return {
674 "backed_up": [],
675 "errors": [],
676 "skipped": [],
677 }
680async def _backup_files_with_patterns(
681 settings_path: AsyncPath,
682 backup_suffix: str,
683 result: dict[str, t.Any],
684) -> None:
685 patterns = ["*.yml", "*.yaml"]
687 for pattern in patterns:
688 await _backup_files_with_pattern(settings_path, pattern, backup_suffix, result)
691async def _backup_files_with_pattern(
692 settings_path: AsyncPath,
693 pattern: str,
694 backup_suffix: str,
695 result: dict[str, t.Any],
696) -> None:
697 async for file_path in settings_path.rglob(pattern):
698 if await file_path.is_file():
699 await _backup_single_file(file_path, backup_suffix, result)
702async def _backup_single_file(
703 file_path: AsyncPath,
704 backup_suffix: str,
705 result: dict[str, t.Any],
706) -> None:
707 try:
708 backup_path = await create_backup(Path(file_path), backup_suffix)
709 result["backed_up"].append(str(backup_path))
710 except Exception as e:
711 result["errors"].append(f"{file_path}: {e}")
714async def get_settings_sync_status(
715 settings_path: AsyncPath | None = None,
716 storage_bucket: str = "settings",
717) -> dict[str, t.Any]:
718 if settings_path is None:
719 settings_path = AsyncPath("settings")
721 status: dict[str, t.Any] = {
722 "total_settings": 0,
723 "in_sync": 0,
724 "out_of_sync": 0,
725 "local_only": 0,
726 "remote_only": 0,
727 "conflicts": 0,
728 "details": [],
729 }
731 try:
732 storage = await _get_storage_adapter()
733 if not storage:
734 status["error"] = "Storage adapter not available"
735 return status
737 settings_files = await _discover_settings_files(settings_path)
738 status["total_settings"] = len(settings_files)
740 await _process_settings_files(settings_files, storage, storage_bucket, status)
742 status["out_of_sync"] = (
743 status["conflicts"] + status["local_only"] + status["remote_only"]
744 )
746 except Exception as e:
747 status["error"] = str(e)
748 debug(f"Error getting settings sync status: {e}")
750 return status
753async def _get_storage_adapter() -> t.Any:
754 """Get the storage adapter."""
755 from acb.depends import depends
757 return depends.get("storage")
760async def _process_settings_files(
761 settings_files: list[dict[str, t.Any]],
762 storage: t.Any,
763 storage_bucket: str,
764 status: dict[str, t.Any],
765) -> None:
766 """Process all settings files and update status."""
767 for settings_info in settings_files:
768 local_info = await get_file_info(Path(settings_info["local_path"]))
769 remote_info = await _get_storage_file_info(
770 storage,
771 storage_bucket,
772 settings_info["storage_path"],
773 )
775 file_status = _create_file_status(settings_info, local_info, remote_info)
776 _update_status_counters(local_info, remote_info, file_status, status)
777 status["details"].append(file_status)
780def _create_file_status(
781 settings_info: dict[str, t.Any],
782 local_info: dict[str, t.Any],
783 remote_info: dict[str, t.Any],
784) -> dict[str, t.Any]:
785 """Create file status dictionary."""
786 file_status: dict[str, t.Any] = {
787 "path": settings_info["storage_path"],
788 "adapter": settings_info["adapter_name"],
789 "local_exists": local_info["exists"],
790 "remote_exists": remote_info["exists"],
791 }
793 # Determine sync status
794 if local_info["exists"] and remote_info["exists"]:
795 if local_info["content_hash"] == remote_info["content_hash"]:
796 file_status["status"] = "in_sync"
797 else:
798 file_status["status"] = "conflict"
799 file_status["local_mtime"] = local_info["mtime"]
800 file_status["remote_mtime"] = remote_info["mtime"]
801 elif local_info["exists"]:
802 file_status["status"] = "local_only"
803 elif remote_info["exists"]:
804 file_status["status"] = "remote_only"
805 else:
806 file_status["status"] = "missing"
808 return file_status
811def _update_status_counters(
812 local_info: dict[str, t.Any],
813 remote_info: dict[str, t.Any],
814 file_status: dict[str, t.Any],
815 status: dict[str, t.Any],
816) -> None:
817 """Update status counters based on file status."""
818 if local_info["exists"] and remote_info["exists"]:
819 if local_info["content_hash"] == remote_info["content_hash"]:
820 status["in_sync"] += 1
821 else:
822 status["conflicts"] += 1
823 elif local_info["exists"]:
824 status["local_only"] += 1
825 elif remote_info["exists"]:
826 status["remote_only"] += 1
829async def validate_all_settings(
830 settings_path: AsyncPath | None = None,
831) -> dict[str, t.Any]:
832 if settings_path is None:
833 settings_path = AsyncPath("settings")
835 result: dict[str, t.Any] = {
836 "valid": [],
837 "invalid": [],
838 "missing": [],
839 "total_checked": 0,
840 }
842 try:
843 settings_files = await _discover_settings_files(settings_path)
844 result["total_checked"] = len(settings_files)
846 for settings_info in settings_files:
847 file_path = settings_info["local_path"]
849 if not await file_path.exists():
850 result["missing"].append(str(file_path))
851 continue
853 try:
854 content = await file_path.read_bytes()
855 await _validate_yaml_content(content)
856 result["valid"].append(str(file_path))
857 except Exception as e:
858 result["invalid"].append(
859 {
860 "path": str(file_path),
861 "error": str(e),
862 },
863 )
865 except Exception as e:
866 result["error"] = str(e)
867 debug(f"Error validating settings: {e}")
869 return result