Coverage for fastblocks/actions/sync/templates.py: 34%
357 statements
« prev ^ index » next coverage.py v7.10.7, created at 2025-10-09 00:47 -0700
« prev ^ index » next coverage.py v7.10.7, created at 2025-10-09 00:47 -0700
1"""Template synchronization between filesystem, storage, and cache layers."""
3import typing as t
4from pathlib import Path
6import yaml
7from acb.debug import debug
8from anyio import Path as AsyncPath
10from .strategies import (
11 ConflictStrategy,
12 SyncDirection,
13 SyncResult,
14 SyncStrategy,
15 create_backup,
16 get_file_info,
17 resolve_conflict,
18 should_sync,
19)
22class TemplateSyncResult(SyncResult):
23 def __init__(
24 self,
25 *,
26 cache_invalidated: list[str] | None = None,
27 bytecode_cleared: list[str] | None = None,
28 **kwargs: t.Any,
29 ) -> None:
30 super().__init__(**kwargs)
31 self.cache_invalidated = (
32 cache_invalidated if cache_invalidated is not None else []
33 )
34 self.bytecode_cleared = bytecode_cleared if bytecode_cleared is not None else []
35 self._direction: str | None = None
36 self._conflict_strategy: str | None = None
37 self._filters: dict[str, t.Any] | None = None
38 self._dry_run: bool = False
40 @property
41 def synchronized_files(self) -> list[str]:
42 return self.synced_items
44 @property
45 def sync_status(self) -> str:
46 if self.has_errors:
47 return "error"
48 elif self.has_conflicts:
49 return "conflict"
50 elif self.synced_items:
51 return "success"
52 return "no_changes"
54 @property
55 def conflicts_resolved(self) -> list[dict[str, t.Any]]:
56 return self.conflicts
58 @property
59 def direction(self) -> str | None:
60 return getattr(self, "_direction", None)
62 @property
63 def conflict_strategy(self) -> str | None:
64 return getattr(self, "_conflict_strategy", None)
66 @property
67 def conflicts_requiring_resolution(self) -> list[dict[str, t.Any]]:
68 return self.conflicts
70 @property
71 def filtered_files(self) -> list[str]:
72 return []
74 @property
75 def dry_run(self) -> bool:
76 return getattr(self, "_dry_run", False)
78 @property
79 def would_sync_files(self) -> list[str]:
80 if self.dry_run:
81 return [item for item in self.synced_items if "dry-run" in item]
82 return []
84 @property
85 def would_resolve_conflicts(self) -> list[dict[str, t.Any]]:
86 if self.dry_run:
87 return [conf for conf in self.conflicts if "dry-run" in str(conf)]
88 return []
91async def sync_templates(
92 *,
93 template_paths: list[AsyncPath] | None = None,
94 patterns: list[str] | None = None,
95 strategy: SyncStrategy | None = None,
96 storage_bucket: str | None = None,
97 direction: str | None = None,
98 conflict_strategy: str | None = None,
99 filters: dict[str, t.Any] | None = None,
100 dry_run: bool = False,
101) -> TemplateSyncResult:
102 if filters and "include_patterns" in filters:
103 patterns = filters["include_patterns"]
105 config = _prepare_sync_config(
106 template_paths, patterns, strategy, direction, conflict_strategy, dry_run
107 )
108 result = TemplateSyncResult()
109 result._direction = config.get("direction")
110 result._conflict_strategy = conflict_strategy
111 result._filters = filters
112 result._dry_run = dry_run
114 if storage_bucket is None:
115 storage_bucket = await _get_default_templates_bucket()
117 try:
118 adapters = await _initialize_adapters(result)
119 except Exception as e:
120 result.errors.append(e)
121 debug(f"Error initializing adapters: {e}")
122 return result
124 if not adapters:
125 return result
127 template_files = await _discover_template_files(
128 config["template_paths"],
129 config["patterns"],
130 )
131 debug(f"Found {len(template_files)} template files to sync")
133 await _sync_template_files(
134 template_files,
135 adapters,
136 config["strategy"],
137 storage_bucket,
138 result,
139 )
141 debug(
142 f"Template sync completed: {len(result.synced_items)} synced, {len(result.conflicts)} conflicts",
143 )
144 return result
147def _prepare_sync_config(
148 template_paths: list[AsyncPath] | None,
149 patterns: list[str] | None,
150 strategy: SyncStrategy | None,
151 direction: str | None = None,
152 conflict_strategy: str | None = None,
153 dry_run: bool = False,
154) -> dict[str, t.Any]:
155 direction_mapping = {
156 "cloud_to_local": SyncDirection.PULL,
157 "local_to_cloud": SyncDirection.PUSH,
158 "bidirectional": SyncDirection.BIDIRECTIONAL,
159 }
161 conflict_mapping = {
162 "local_wins": ConflictStrategy.LOCAL_WINS,
163 "remote_wins": ConflictStrategy.REMOTE_WINS,
164 "cloud_wins": ConflictStrategy.REMOTE_WINS,
165 "newest_wins": ConflictStrategy.NEWEST_WINS,
166 "manual": ConflictStrategy.MANUAL,
167 "backup_both": ConflictStrategy.BACKUP_BOTH,
168 }
170 if strategy is None:
171 sync_direction = SyncDirection.BIDIRECTIONAL
172 if direction and direction in direction_mapping:
173 sync_direction = direction_mapping[direction]
175 sync_conflict = ConflictStrategy.NEWEST_WINS
176 if conflict_strategy and conflict_strategy in conflict_mapping:
177 sync_conflict = conflict_mapping[conflict_strategy]
179 strategy = SyncStrategy(
180 direction=sync_direction, conflict_strategy=sync_conflict, dry_run=dry_run
181 )
183 return {
184 "template_paths": template_paths or [AsyncPath("templates")],
185 "patterns": patterns or ["*.html", "*.jinja2", "*.j2", "*.txt"],
186 "strategy": strategy,
187 "direction": direction,
188 }
191async def _initialize_adapters(result: TemplateSyncResult) -> dict[str, t.Any] | None:
192 try:
193 from acb.depends import depends
195 storage = depends.get("storage")
196 cache = depends.get("cache")
197 if not storage:
198 result.errors.append(Exception("Storage adapter not available"))
199 return None
201 return {"storage": storage, "cache": cache}
202 except Exception as e:
203 result.errors.append(e)
204 return None
207async def _get_default_templates_bucket() -> str:
208 try:
209 storage_config_path = AsyncPath("settings/storage.yml")
210 if await storage_config_path.exists():
211 content = await storage_config_path.read_text()
212 config = yaml.safe_load(content)
213 if isinstance(config, dict):
214 bucket_name = t.cast(
215 str, config.get("buckets", {}).get("templates", "templates")
216 )
217 else:
218 bucket_name = "templates"
219 debug(f"Using templates bucket from config: {bucket_name}")
220 return bucket_name
221 except Exception as e:
222 debug(f"Could not load storage config, using default: {e}")
223 debug("Using fallback templates bucket: templates")
224 return "templates"
227async def _discover_template_files(
228 template_paths: list[AsyncPath],
229 patterns: list[str],
230) -> list[dict[str, t.Any]]:
231 template_files = []
233 for base_path in template_paths:
234 if not await base_path.exists():
235 debug(f"Template path does not exist: {base_path}")
236 continue
238 files = await _scan_path_for_templates(base_path, patterns)
239 template_files.extend(files)
241 return template_files
244async def _scan_path_for_templates(
245 base_path: AsyncPath,
246 patterns: list[str],
247) -> list[dict[str, t.Any]]:
248 files = []
250 for pattern in patterns:
251 async for file_path in base_path.rglob(pattern):
252 if await file_path.is_file():
253 try:
254 rel_path = file_path.relative_to(base_path)
255 files.append(
256 {
257 "local_path": file_path,
258 "relative_path": rel_path,
259 "storage_path": str(rel_path),
260 },
261 )
262 except ValueError:
263 debug(f"Could not get relative path for {file_path}")
265 return files
268async def _sync_template_files(
269 template_files: list[dict[str, t.Any]],
270 adapters: dict[str, t.Any],
271 strategy: SyncStrategy,
272 storage_bucket: str,
273 result: TemplateSyncResult,
274) -> None:
275 for template_info in template_files:
276 try:
277 file_result = await _sync_single_template(
278 template_info,
279 adapters["storage"],
280 adapters["cache"],
281 strategy,
282 storage_bucket,
283 )
284 _accumulate_sync_results(file_result, result)
286 except Exception as e:
287 result.errors.append(e)
288 debug(f"Error syncing template {template_info['relative_path']}: {e}")
291def _accumulate_sync_results(
292 file_result: dict[str, t.Any],
293 result: TemplateSyncResult,
294) -> None:
295 for key in (
296 "synced",
297 "conflicts",
298 "errors",
299 "skipped",
300 "backed_up",
301 "cache_invalidated",
302 "bytecode_cleared",
303 ):
304 if file_result.get(key):
305 getattr(result, f"{key}_items" if key == "synced" else key).extend(
306 file_result[key],
307 )
310async def _sync_single_template(
311 template_info: dict[str, t.Any],
312 storage: t.Any,
313 cache: t.Any,
314 strategy: SyncStrategy,
315 storage_bucket: str,
316) -> dict[str, t.Any]:
317 local_path = template_info["local_path"]
318 storage_path = template_info["storage_path"]
319 relative_path = template_info["relative_path"]
321 result: dict[str, t.Any] = {
322 "synced": [],
323 "conflicts": [],
324 "errors": [],
325 "skipped": [],
326 "backed_up": [],
327 "cache_invalidated": [],
328 "bytecode_cleared": [],
329 }
331 try:
332 local_info = await get_file_info(Path(local_path))
333 remote_info = await _get_storage_file_info(
334 storage,
335 storage_bucket,
336 storage_path,
337 )
339 sync_needed, reason = should_sync(local_info, remote_info, strategy.direction)
341 if not sync_needed:
342 result["skipped"].append(f"{relative_path} ({reason})")
343 return result
345 debug(f"Syncing template {relative_path}: {reason}")
347 # Handle sync based on direction
348 await _handle_sync_direction(
349 strategy,
350 local_info,
351 remote_info,
352 local_path,
353 storage,
354 storage_bucket,
355 storage_path,
356 result,
357 )
359 # Invalidate cache if needed
360 if result["synced"]:
361 await _invalidate_template_cache(cache, str(relative_path), result)
363 except Exception as e:
364 result["errors"].append(e)
365 debug(f"Error in _sync_single_template for {relative_path}: {e}")
367 return result
370async def _handle_sync_direction(
371 strategy: SyncStrategy,
372 local_info: dict[str, t.Any],
373 remote_info: dict[str, t.Any],
374 local_path: t.Any,
375 storage: t.Any,
376 storage_bucket: str,
377 storage_path: str,
378 result: dict[str, t.Any],
379) -> None:
380 """Handle sync based on direction."""
381 if _should_pull_template(strategy, local_info, remote_info):
382 await _pull_template(
383 local_path,
384 storage,
385 storage_bucket,
386 storage_path,
387 strategy,
388 result,
389 )
390 elif _should_push_template(strategy, local_info, remote_info):
391 await _push_template(
392 local_path,
393 storage,
394 storage_bucket,
395 storage_path,
396 strategy,
397 result,
398 )
399 elif _has_bidirectional_conflict(strategy, local_info, remote_info):
400 await _handle_template_conflict(
401 local_path,
402 storage,
403 storage_bucket,
404 storage_path,
405 local_info,
406 remote_info,
407 strategy,
408 result,
409 )
412def _should_pull_template(
413 strategy: SyncStrategy, local_info: dict[str, t.Any], remote_info: dict[str, t.Any]
414) -> bool:
415 """Check if template should be pulled."""
416 return strategy.direction == SyncDirection.PULL or (
417 strategy.direction == SyncDirection.BIDIRECTIONAL
418 and remote_info["exists"]
419 and (not local_info["exists"] or remote_info["mtime"] > local_info["mtime"])
420 )
423def _should_push_template(
424 strategy: SyncStrategy, local_info: dict[str, t.Any], remote_info: dict[str, t.Any]
425) -> bool:
426 """Check if template should be pushed."""
427 return strategy.direction == SyncDirection.PUSH or (
428 strategy.direction == SyncDirection.BIDIRECTIONAL
429 and local_info["exists"]
430 and (not remote_info["exists"] or local_info["mtime"] > remote_info["mtime"])
431 )
434def _has_bidirectional_conflict(
435 strategy: SyncStrategy, local_info: dict[str, t.Any], remote_info: dict[str, t.Any]
436) -> bool:
437 """Check if there's a bidirectional conflict."""
438 return (
439 strategy.direction == SyncDirection.BIDIRECTIONAL
440 and local_info["exists"]
441 and remote_info["exists"]
442 )
445async def _get_storage_file_info(
446 storage: t.Any,
447 bucket: str,
448 file_path: str,
449) -> dict[str, t.Any]:
450 try:
451 bucket_obj = getattr(storage, bucket)
453 exists = await bucket_obj.exists(file_path)
455 if not exists:
456 return {
457 "exists": False,
458 "size": 0,
459 "mtime": 0,
460 "content_hash": None,
461 }
463 content = await bucket_obj.read(file_path)
464 metadata = await bucket_obj.stat(file_path)
466 import hashlib
468 content_hash = hashlib.blake2b(content).hexdigest()
470 return {
471 "exists": True,
472 "size": len(content),
473 "mtime": metadata.get("mtime", 0),
474 "content_hash": content_hash,
475 "content": content,
476 }
478 except Exception as e:
479 debug(f"Error getting storage file info for {file_path}: {e}")
480 return {
481 "exists": False,
482 "size": 0,
483 "mtime": 0,
484 "content_hash": None,
485 "error": str(e),
486 }
489async def _pull_template(
490 local_path: AsyncPath,
491 storage: t.Any,
492 bucket: str,
493 storage_path: str,
494 strategy: SyncStrategy,
495 result: dict[str, t.Any],
496) -> None:
497 try:
498 bucket_obj = getattr(storage, bucket)
500 if strategy.dry_run:
501 debug(f"DRY RUN: Would pull {storage_path} to {local_path}")
502 result["synced"].append(f"PULL(dry-run): {storage_path}")
503 return
505 if await local_path.exists() and strategy.backup_on_conflict:
506 backup_path = await create_backup(Path(local_path))
507 result["backed_up"].append(str(backup_path))
509 content = await bucket_obj.read(storage_path)
511 await local_path.parent.mkdir(parents=True, exist_ok=True)
513 await local_path.write_bytes(content)
515 result["synced"].append(f"PULL: {storage_path}")
516 debug(f"Pulled template from storage: {storage_path}")
518 except Exception as e:
519 result["errors"].append(e)
520 debug(f"Error pulling template {storage_path}: {e}")
523async def _push_template(
524 local_path: AsyncPath,
525 storage: t.Any,
526 bucket: str,
527 storage_path: str,
528 strategy: SyncStrategy,
529 result: dict[str, t.Any],
530) -> None:
531 try:
532 bucket_obj = getattr(storage, bucket)
534 if strategy.dry_run:
535 debug(f"DRY RUN: Would push {local_path} to {storage_path}")
536 result["synced"].append(f"PUSH(dry-run): {storage_path}")
537 return
539 content = await local_path.read_bytes()
541 await bucket_obj.write(storage_path, content)
543 result["synced"].append(f"PUSH: {storage_path}")
544 debug(f"Pushed template to storage: {storage_path}")
546 except Exception as e:
547 result["errors"].append(e)
548 debug(f"Error pushing template {storage_path}: {e}")
551async def _handle_template_conflict(
552 local_path: AsyncPath,
553 storage: t.Any,
554 bucket: str,
555 storage_path: str,
556 local_info: dict[str, t.Any],
557 remote_info: dict[str, t.Any],
558 strategy: SyncStrategy,
559 result: dict[str, t.Any],
560) -> None:
561 try:
562 if strategy.conflict_strategy == ConflictStrategy.MANUAL:
563 result["conflicts"].append(
564 {
565 "path": storage_path,
566 "local_mtime": local_info["mtime"],
567 "remote_mtime": remote_info["mtime"],
568 "reason": "manual_resolution_required",
569 },
570 )
571 return
573 resolved_content, resolution_reason = await resolve_conflict(
574 Path(local_path),
575 remote_info["content"],
576 local_info["content"],
577 strategy.conflict_strategy,
578 local_info["mtime"],
579 remote_info["mtime"],
580 )
582 if strategy.dry_run:
583 debug(
584 f"DRY RUN: Would resolve conflict for {storage_path}: {resolution_reason}",
585 )
586 result["synced"].append(
587 f"CONFLICT(dry-run): {storage_path} - {resolution_reason}",
588 )
589 return
591 if (
592 strategy.backup_on_conflict
593 or strategy.conflict_strategy == ConflictStrategy.BACKUP_BOTH
594 ):
595 backup_path = await create_backup(Path(local_path), "conflict")
596 result["backed_up"].append(str(backup_path))
598 if resolved_content == remote_info["content"]:
599 await local_path.write_bytes(resolved_content)
600 result["synced"].append(
601 f"CONFLICT->REMOTE: {storage_path} - {resolution_reason}",
602 )
603 else:
604 bucket_obj = getattr(storage, bucket)
605 await bucket_obj.write(storage_path, resolved_content)
606 result["synced"].append(
607 f"CONFLICT->LOCAL: {storage_path} - {resolution_reason}",
608 )
610 debug(f"Resolved template conflict: {storage_path} - {resolution_reason}")
612 except Exception as e:
613 result["errors"].append(e)
614 result["conflicts"].append(
615 {
616 "path": storage_path,
617 "error": str(e),
618 "reason": "resolution_failed",
619 },
620 )
623async def _invalidate_template_cache(
624 cache: t.Any,
625 template_path: str,
626 result: dict[str, t.Any],
627) -> None:
628 if not cache:
629 return
631 try:
632 template_key = f"template:{template_path}"
633 await cache.delete(template_key)
634 result["cache_invalidated"].append(template_key)
636 bytecode_key = f"bccache:{template_path}"
637 await cache.delete(bytecode_key)
638 result["bytecode_cleared"].append(bytecode_key)
640 await cache.delete_pattern(f"template:*:{template_path}")
641 await cache.delete_pattern(f"bccache:*:{template_path}")
643 debug(f"Invalidated cache for template: {template_path}")
645 except Exception as e:
646 debug(f"Error invalidating cache for {template_path}: {e}")
649async def warm_template_cache(
650 template_paths: list[str] | None = None,
651 cache_namespace: str = "templates",
652) -> dict[str, t.Any]:
653 result: dict[str, t.Any] = {
654 "warmed": [],
655 "errors": [],
656 "skipped": [],
657 }
659 if not template_paths:
660 template_paths = [
661 "base.html",
662 "index.html",
663 "layout.html",
664 "404.html",
665 "500.html",
666 ]
668 try:
669 from acb.depends import depends
671 cache = depends.get("cache")
672 storage = depends.get("storage")
674 if not cache or not storage:
675 result["errors"].append(Exception("Cache or storage not available"))
676 return result
678 for template_path in template_paths:
679 try:
680 cache_key = f"{cache_namespace}:{template_path}"
681 if await cache.exists(cache_key):
682 result["skipped"].append(template_path)
683 continue
685 content = await storage.templates.read(template_path)
687 await cache.set(cache_key, content, ttl=86400)
688 result["warmed"].append(template_path)
690 debug(f"Warmed cache for template: {template_path}")
692 except Exception as e:
693 result["errors"].append(f"{template_path}: {e}")
694 debug(f"Error warming cache for {template_path}: {e}")
696 except Exception as e:
697 result["errors"].append(str(e))
698 debug(f"Error in warm_template_cache: {e}")
700 return result
703async def get_template_sync_status(
704 template_paths: list[AsyncPath] | None = None,
705 storage_bucket: str = "templates",
706) -> dict[str, t.Any]:
707 if template_paths is None:
708 template_paths = [AsyncPath("templates")]
710 status: dict[str, t.Any] = {
711 "total_templates": 0,
712 "in_sync": 0,
713 "out_of_sync": 0,
714 "local_only": 0,
715 "remote_only": 0,
716 "conflicts": 0,
717 "details": [],
718 }
720 try:
721 from acb.depends import depends
723 storage = depends.get("storage")
725 if not storage:
726 status["error"] = "Storage adapter not available"
727 return status
729 template_files = await _discover_template_files_for_status(template_paths)
730 status["total_templates"] = len(template_files)
732 await _process_template_files_for_status(
733 template_files,
734 storage,
735 storage_bucket,
736 status,
737 )
739 except Exception as e:
740 status["error"] = str(e)
741 debug(f"Error getting template sync status: {e}")
743 return status
746async def _discover_template_files_for_status(
747 template_paths: list[AsyncPath],
748) -> list[dict[str, t.Any]]:
749 template_files = []
750 for base_path in template_paths:
751 if await base_path.exists():
752 async for file_path in base_path.rglob("*.html"):
753 if await file_path.is_file():
754 rel_path = file_path.relative_to(base_path)
755 template_files.append(
756 {
757 "local_path": file_path,
758 "storage_path": str(rel_path),
759 },
760 )
761 return template_files
764async def _process_template_files_for_status(
765 template_files: list[dict[str, t.Any]],
766 storage: t.Any,
767 storage_bucket: str,
768 status: dict[str, t.Any],
769) -> None:
770 for template_info in template_files:
771 local_info = await get_file_info(Path(template_info["local_path"]))
772 remote_info = await _get_storage_file_info(
773 storage,
774 storage_bucket,
775 template_info["storage_path"],
776 )
778 file_status = _create_file_status_info(template_info, local_info, remote_info)
779 _update_status_counters(local_info, remote_info, file_status, status)
781 details_list = status["details"]
782 assert isinstance(details_list, list)
783 details_list.append(file_status)
785 _calculate_out_of_sync_total(status)
788def _create_file_status_info(
789 template_info: dict[str, t.Any],
790 local_info: dict[str, t.Any],
791 remote_info: dict[str, t.Any],
792) -> dict[str, t.Any]:
793 return {
794 "path": template_info["storage_path"],
795 "local_exists": local_info["exists"],
796 "remote_exists": remote_info["exists"],
797 }
800def _update_status_counters(
801 local_info: dict[str, t.Any],
802 remote_info: dict[str, t.Any],
803 file_status: dict[str, t.Any],
804 status: dict[str, t.Any],
805) -> None:
806 if local_info["exists"] and remote_info["exists"]:
807 if local_info["content_hash"] == remote_info["content_hash"]:
808 file_status["status"] = "in_sync"
809 status["in_sync"] = status["in_sync"] + 1
810 else:
811 file_status["status"] = "conflict"
812 file_status["local_mtime"] = local_info["mtime"]
813 file_status["remote_mtime"] = remote_info["mtime"]
814 status["conflicts"] = status["conflicts"] + 1
815 elif local_info["exists"]:
816 file_status["status"] = "local_only"
817 status["local_only"] = status["local_only"] + 1
818 elif remote_info["exists"]:
819 file_status["status"] = "remote_only"
820 status["remote_only"] = status["remote_only"] + 1
821 else:
822 file_status["status"] = "missing"
825def _calculate_out_of_sync_total(status: dict[str, t.Any]) -> None:
826 conflicts = status["conflicts"]
827 local_only = status["local_only"]
828 remote_only = status["remote_only"]
829 assert isinstance(conflicts, int)
830 assert isinstance(local_only, int)
831 assert isinstance(remote_only, int)
832 status["out_of_sync"] = conflicts + local_only + remote_only