Coverage for fastblocks/actions/sync/templates.py: 36%

1"""Template synchronization between filesystem, storage, and cache layers."""

3import typing as t

4from pathlib import Path

6import yaml

7from acb.debug import debug

8from anyio import Path as AsyncPath

10from .strategies import (

11 ConflictStrategy,

12 SyncDirection,

13 SyncResult,

14 SyncStrategy,

15 create_backup,

16 get_file_info,

17 resolve_conflict,

18 should_sync,

19)

22class TemplateSyncResult(SyncResult):

23 def __init__(

24 self,

25 *,

26 cache_invalidated: list[str] | None = None,

27 bytecode_cleared: list[str] | None = None,

28 **kwargs: t.Any,

29 ) -> None:

30 super().__init__(**kwargs)

31 self.cache_invalidated = (

32 cache_invalidated if cache_invalidated is not None else []

33 )

34 self.bytecode_cleared = bytecode_cleared if bytecode_cleared is not None else []

35 self._direction: str | None = None

36 self._conflict_strategy: str | None = None

37 self._filters: dict[str, t.Any] | None = None

38 self._dry_run: bool = False

40 @property

41 def synchronized_files(self) -> list[str]:

42 return self.synced_items

44 @property

45 def sync_status(self) -> str:

46 if self.has_errors:

47 return "error"

48 elif self.has_conflicts:

49 return "conflict"

50 elif self.synced_items:

51 return "success"

52 return "no_changes"

54 @property

55 def conflicts_resolved(self) -> list[dict[str, t.Any]]:

56 return self.conflicts

58 @property

59 def direction(self) -> str | None:

60 return getattr(self, "_direction", None)

62 @property

63 def conflict_strategy(self) -> str | None:

64 return getattr(self, "_conflict_strategy", None)

66 @property

67 def conflicts_requiring_resolution(self) -> list[dict[str, t.Any]]:

68 return self.conflicts

70 @property

71 def filtered_files(self) -> list[str]:

72 return []

74 @property

75 def dry_run(self) -> bool:

76 return getattr(self, "_dry_run", False)

78 @property

79 def would_sync_files(self) -> list[str]:

80 if self.dry_run:

81 return [item for item in self.synced_items if "dry-run" in item]

82 return []

84 @property

85 def would_resolve_conflicts(self) -> list[dict[str, t.Any]]:

86 if self.dry_run:

87 return [conf for conf in self.conflicts if "dry-run" in str(conf)]

88 return []

91async def sync_templates(

92 *,

93 template_paths: list[AsyncPath] | None = None,

94 patterns: list[str] | None = None,

95 strategy: SyncStrategy | None = None,

96 storage_bucket: str | None = None,

97 direction: str | None = None,

98 conflict_strategy: str | None = None,

99 filters: dict[str, t.Any] | None = None,

100 dry_run: bool = False,

101) -> TemplateSyncResult:

102 if filters and "include_patterns" in filters:

103 patterns = filters["include_patterns"]

104

105 config = _prepare_sync_config(

106 template_paths, patterns, strategy, direction, conflict_strategy, dry_run

107 )

108 result = TemplateSyncResult()

109 result._direction = config.get("direction")

110 result._conflict_strategy = conflict_strategy

111 result._filters = filters

112 result._dry_run = dry_run

113

114 if storage_bucket is None:

115 storage_bucket = await _get_default_templates_bucket()

116

117 try:

118 adapters = await _initialize_adapters(result)

119 except Exception as e:

120 result.errors.append(e)

121 debug(f"Error initializing adapters: {e}")

122 return result

123

124 if not adapters:

125 return result

126

127 template_files = await _discover_template_files(

128 config["template_paths"],

129 config["patterns"],

130 )

131 debug(f"Found {len(template_files)} template files to sync")

132

133 await _sync_template_files(

134 template_files,

135 adapters,

136 config["strategy"],

137 storage_bucket,

138 result,

139 )

140

141 debug(

142 f"Template sync completed: {len(result.synced_items)} synced, {len(result.conflicts)} conflicts",

143 )

144 return result

145

146

147def _prepare_sync_config(

148 template_paths: list[AsyncPath] | None,

149 patterns: list[str] | None,

150 strategy: SyncStrategy | None,

151 direction: str | None = None,

152 conflict_strategy: str | None = None,

153 dry_run: bool = False,

154) -> dict[str, t.Any]:

155 direction_mapping = {

156 "cloud_to_local": SyncDirection.PULL,

157 "local_to_cloud": SyncDirection.PUSH,

158 "bidirectional": SyncDirection.BIDIRECTIONAL,

159 }

160

161 conflict_mapping = {

162 "local_wins": ConflictStrategy.LOCAL_WINS,

163 "remote_wins": ConflictStrategy.REMOTE_WINS,

164 "cloud_wins": ConflictStrategy.REMOTE_WINS,

165 "newest_wins": ConflictStrategy.NEWEST_WINS,

166 "manual": ConflictStrategy.MANUAL,

167 "backup_both": ConflictStrategy.BACKUP_BOTH,

168 }

169

170 if strategy is None:

171 sync_direction = SyncDirection.BIDIRECTIONAL

172 if direction and direction in direction_mapping:

173 sync_direction = direction_mapping[direction]

174

175 sync_conflict = ConflictStrategy.NEWEST_WINS

176 if conflict_strategy and conflict_strategy in conflict_mapping:

177 sync_conflict = conflict_mapping[conflict_strategy]

178

179 strategy = SyncStrategy(

180 direction=sync_direction, conflict_strategy=sync_conflict, dry_run=dry_run

181 )

182

183 return {

184 "template_paths": template_paths or [AsyncPath("templates")],

185 "patterns": patterns or ["*.html", "*.jinja2", "*.j2", "*.txt"],

186 "strategy": strategy,

187 "direction": direction,

188 }

189

190

191async def _initialize_adapters(result: TemplateSyncResult) -> dict[str, t.Any] | None:

192 try:

193 from acb.depends import depends

194

195 storage = depends.get("storage")

196 cache = depends.get("cache")

197 if not storage:

198 result.errors.append(Exception("Storage adapter not available"))

199 return None

200

201 return {"storage": storage, "cache": cache}

202 except Exception as e:

203 result.errors.append(e)

204 return None

205

206

207async def _get_default_templates_bucket() -> str:

208 try:

209 storage_config_path = AsyncPath("settings/storage.yml")

210 if await storage_config_path.exists():

211 content = await storage_config_path.read_text()

212 config = yaml.safe_load(content)

213 if isinstance(config, dict):

214 bucket_name = config.get("buckets", {}).get("templates", "templates")

215 else:

216 bucket_name = "templates"

217 debug(f"Using templates bucket from config: {bucket_name}")

218 return bucket_name

219 except Exception as e:

220 debug(f"Could not load storage config, using default: {e}")

221 debug("Using fallback templates bucket: templates")

222 return "templates"

223

224

225async def _discover_template_files(

226 template_paths: list[AsyncPath],

227 patterns: list[str],

228) -> list[dict[str, t.Any]]:

229 template_files = []

230

231 for base_path in template_paths:

232 if not await base_path.exists():

233 debug(f"Template path does not exist: {base_path}")

234 continue

235

236 files = await _scan_path_for_templates(base_path, patterns)

237 template_files.extend(files)

238

239 return template_files

240

241

242async def _scan_path_for_templates(

243 base_path: AsyncPath,

244 patterns: list[str],

245) -> list[dict[str, t.Any]]:

246 files = []

247

248 for pattern in patterns:

249 async for file_path in base_path.rglob(pattern):

250 if await file_path.is_file():

251 try:

252 rel_path = file_path.relative_to(base_path)

253 files.append(

254 {

255 "local_path": file_path,

256 "relative_path": rel_path,

257 "storage_path": str(rel_path),

258 },

259 )

260 except ValueError:

261 debug(f"Could not get relative path for {file_path}")

262

263 return files

264

265

266async def _sync_template_files(

267 template_files: list[dict[str, t.Any]],

268 adapters: dict[str, t.Any],

269 strategy: SyncStrategy,

270 storage_bucket: str,

271 result: TemplateSyncResult,

272) -> None:

273 for template_info in template_files:

274 try:

275 file_result = await _sync_single_template(

276 template_info,

277 adapters["storage"],

278 adapters["cache"],

279 strategy,

280 storage_bucket,

281 )

282 _accumulate_sync_results(file_result, result)

283

284 except Exception as e:

285 result.errors.append(e)

286 debug(f"Error syncing template {template_info['relative_path']}: {e}")

287

288

289def _accumulate_sync_results(

290 file_result: dict[str, t.Any],

291 result: TemplateSyncResult,

292) -> None:

293 for key in (

294 "synced",

295 "conflicts",

296 "errors",

297 "skipped",

298 "backed_up",

299 "cache_invalidated",

300 "bytecode_cleared",

301 ):

302 if file_result.get(key):

303 getattr(result, f"{key}_items" if key == "synced" else key).extend(

304 file_result[key],

305 )

306

307

308async def _sync_single_template(

309 template_info: dict[str, t.Any],

310 storage: t.Any,

311 cache: t.Any,

312 strategy: SyncStrategy,

313 storage_bucket: str,

314) -> dict[str, t.Any]:

315 local_path = template_info["local_path"]

316 storage_path = template_info["storage_path"]

317 relative_path = template_info["relative_path"]

318

319 result: dict[str, t.Any] = {

320 "synced": [],

321 "conflicts": [],

322 "errors": [],

323 "skipped": [],

324 "backed_up": [],

325 "cache_invalidated": [],

326 "bytecode_cleared": [],

327 }

328

329 try:

330 local_info = await get_file_info(Path(local_path))

331

332 remote_info = await _get_storage_file_info(

333 storage,

334 storage_bucket,

335 storage_path,

336 )

337

338 sync_needed, reason = should_sync(local_info, remote_info, strategy.direction)

339

340 if not sync_needed:

341 result["skipped"].append(f"{relative_path} ({reason})")

342 return result

343

344 debug(f"Syncing template {relative_path}: {reason}")

345

346 if strategy.direction == SyncDirection.PULL or (

347 strategy.direction == SyncDirection.BIDIRECTIONAL

348 and remote_info["exists"]

349 and (not local_info["exists"] or remote_info["mtime"] > local_info["mtime"])

350 ):

351 await _pull_template(

352 local_path,

353 storage,

354 storage_bucket,

355 storage_path,

356 strategy,

357 result,

358 )

359

360 elif strategy.direction == SyncDirection.PUSH or (

361 strategy.direction == SyncDirection.BIDIRECTIONAL

362 and local_info["exists"]

363 and (

364 not remote_info["exists"] or local_info["mtime"] > remote_info["mtime"]

365 )

366 ):

367 await _push_template(

368 local_path,

369 storage,

370 storage_bucket,

371 storage_path,

372 strategy,

373 result,

374 )

375

376 elif (

377 strategy.direction == SyncDirection.BIDIRECTIONAL

378 and local_info["exists"]

379 and remote_info["exists"]

380 ):

381 await _handle_template_conflict(

382 local_path,

383 storage,

384 storage_bucket,

385 storage_path,

386 local_info,

387 remote_info,

388 strategy,

389 result,

390 )

391

392 if result["synced"]:

393 await _invalidate_template_cache(cache, str(relative_path), result)

394

395 except Exception as e:

396 result["errors"].append(e)

397 debug(f"Error in _sync_single_template for {relative_path}: {e}")

398

399 return result

400

401

402async def _get_storage_file_info(

403 storage: t.Any,

404 bucket: str,

405 file_path: str,

406) -> dict[str, t.Any]:

407 try:

408 bucket_obj = getattr(storage, bucket)

409

410 exists = await bucket_obj.exists(file_path)

411

412 if not exists:

413 return {

414 "exists": False,

415 "size": 0,

416 "mtime": 0,

417 "content_hash": None,

418 }

419

420 content = await bucket_obj.read(file_path)

421 metadata = await bucket_obj.stat(file_path)

422

423 import hashlib

424

425 content_hash = hashlib.blake2b(content).hexdigest()

426

427 return {

428 "exists": True,

429 "size": len(content),

430 "mtime": metadata.get("mtime", 0),

431 "content_hash": content_hash,

432 "content": content,

433 }

434

435 except Exception as e:

436 debug(f"Error getting storage file info for {file_path}: {e}")

437 return {

438 "exists": False,

439 "size": 0,

440 "mtime": 0,

441 "content_hash": None,

442 "error": str(e),

443 }

444

445

446async def _pull_template(

447 local_path: AsyncPath,

448 storage: t.Any,

449 bucket: str,

450 storage_path: str,

451 strategy: SyncStrategy,

452 result: dict[str, t.Any],

453) -> None:

454 try:

455 bucket_obj = getattr(storage, bucket)

456

457 if strategy.dry_run:

458 debug(f"DRY RUN: Would pull {storage_path} to {local_path}")

459 result["synced"].append(f"PULL(dry-run): {storage_path}")

460 return

461

462 if await local_path.exists() and strategy.backup_on_conflict:

463 backup_path = await create_backup(Path(local_path))

464 result["backed_up"].append(str(backup_path))

465

466 content = await bucket_obj.read(storage_path)

467

468 await local_path.parent.mkdir(parents=True, exist_ok=True)

469

470 await local_path.write_bytes(content)

471

472 result["synced"].append(f"PULL: {storage_path}")

473 debug(f"Pulled template from storage: {storage_path}")

474

475 except Exception as e:

476 result["errors"].append(e)

477 debug(f"Error pulling template {storage_path}: {e}")

478

479

480async def _push_template(

481 local_path: AsyncPath,

482 storage: t.Any,

483 bucket: str,

484 storage_path: str,

485 strategy: SyncStrategy,

486 result: dict[str, t.Any],

487) -> None:

488 try:

489 bucket_obj = getattr(storage, bucket)

490

491 if strategy.dry_run:

492 debug(f"DRY RUN: Would push {local_path} to {storage_path}")

493 result["synced"].append(f"PUSH(dry-run): {storage_path}")

494 return

495

496 content = await local_path.read_bytes()

497

498 await bucket_obj.write(storage_path, content)

499

500 result["synced"].append(f"PUSH: {storage_path}")

501 debug(f"Pushed template to storage: {storage_path}")

502

503 except Exception as e:

504 result["errors"].append(e)

505 debug(f"Error pushing template {storage_path}: {e}")

506

507

508async def _handle_template_conflict(

509 local_path: AsyncPath,

510 storage: t.Any,

511 bucket: str,

512 storage_path: str,

513 local_info: dict[str, t.Any],

514 remote_info: dict[str, t.Any],

515 strategy: SyncStrategy,

516 result: dict[str, t.Any],

517) -> None:

518 try:

519 if strategy.conflict_strategy == ConflictStrategy.MANUAL:

520 result["conflicts"].append(

521 {

522 "path": storage_path,

523 "local_mtime": local_info["mtime"],

524 "remote_mtime": remote_info["mtime"],

525 "reason": "manual_resolution_required",

526 },

527 )

528 return

529

530 resolved_content, resolution_reason = await resolve_conflict(

531 Path(local_path),

532 remote_info["content"],

533 local_info["content"],

534 strategy.conflict_strategy,

535 local_info["mtime"],

536 remote_info["mtime"],

537 )

538

539 if strategy.dry_run:

540 debug(

541 f"DRY RUN: Would resolve conflict for {storage_path}: {resolution_reason}",

542 )

543 result["synced"].append(

544 f"CONFLICT(dry-run): {storage_path} - {resolution_reason}",

545 )

546 return

547

548 if (

549 strategy.backup_on_conflict

550 or strategy.conflict_strategy == ConflictStrategy.BACKUP_BOTH

551 ):

552 backup_path = await create_backup(Path(local_path), "conflict")

553 result["backed_up"].append(str(backup_path))

554

555 if resolved_content == remote_info["content"]:

556 await local_path.write_bytes(resolved_content)

557 result["synced"].append(

558 f"CONFLICT->REMOTE: {storage_path} - {resolution_reason}",

559 )

560 else:

561 bucket_obj = getattr(storage, bucket)

562 await bucket_obj.write(storage_path, resolved_content)

563 result["synced"].append(

564 f"CONFLICT->LOCAL: {storage_path} - {resolution_reason}",

565 )

566

567 debug(f"Resolved template conflict: {storage_path} - {resolution_reason}")

568

569 except Exception as e:

570 result["errors"].append(e)

571 result["conflicts"].append(

572 {

573 "path": storage_path,

574 "error": str(e),

575 "reason": "resolution_failed",

576 },

577 )

578

579

580async def _invalidate_template_cache(

581 cache: t.Any,

582 template_path: str,

583 result: dict[str, t.Any],

584) -> None:

585 if not cache:

586 return

587

588 try:

589 template_key = f"template:{template_path}"

590 await cache.delete(template_key)

591 result["cache_invalidated"].append(template_key)

592

593 bytecode_key = f"bccache:{template_path}"

594 await cache.delete(bytecode_key)

595 result["bytecode_cleared"].append(bytecode_key)

596

597 await cache.delete_pattern(f"template:*:{template_path}")

598 await cache.delete_pattern(f"bccache:*:{template_path}")

599

600 debug(f"Invalidated cache for template: {template_path}")

601

602 except Exception as e:

603 debug(f"Error invalidating cache for {template_path}: {e}")

604

605

606async def warm_template_cache(

607 template_paths: list[str] | None = None,

608 cache_namespace: str = "templates",

609) -> dict[str, t.Any]:

610 result: dict[str, t.Any] = {

611 "warmed": [],

612 "errors": [],

613 "skipped": [],

614 }

615

616 if not template_paths:

617 template_paths = [

618 "base.html",

619 "index.html",

620 "layout.html",

621 "404.html",

622 "500.html",

623 ]

624

625 try:

626 from acb.depends import depends

627

628 cache = depends.get("cache")

629 storage = depends.get("storage")

630

631 if not cache or not storage:

632 result["errors"].append(Exception("Cache or storage not available"))

633 return result

634

635 for template_path in template_paths:

636 try:

637 cache_key = f"{cache_namespace}:{template_path}"

638 if await cache.exists(cache_key):

639 result["skipped"].append(template_path)

640 continue

641

642 content = await storage.templates.read(template_path)

643

644 await cache.set(cache_key, content, ttl=86400)

645 result["warmed"].append(template_path)

646

647 debug(f"Warmed cache for template: {template_path}")

648

649 except Exception as e:

650 result["errors"].append(f"{template_path}: {e}")

651 debug(f"Error warming cache for {template_path}: {e}")

652

653 except Exception as e:

654 result["errors"].append(str(e))

655 debug(f"Error in warm_template_cache: {e}")

656

657 return result

658

659

660async def get_template_sync_status(

661 template_paths: list[AsyncPath] | None = None,

662 storage_bucket: str = "templates",

663) -> dict[str, t.Any]:

664 if template_paths is None:

665 template_paths = [AsyncPath("templates")]

666

667 status: dict[str, t.Any] = {

668 "total_templates": 0,

669 "in_sync": 0,

670 "out_of_sync": 0,

671 "local_only": 0,

672 "remote_only": 0,

673 "conflicts": 0,

674 "details": [],

675 }

676

677 try:

678 from acb.depends import depends

679

680 storage = depends.get("storage")

681

682 if not storage:

683 status["error"] = "Storage adapter not available"

684 return status

685

686 template_files = await _discover_template_files_for_status(template_paths)

687 status["total_templates"] = len(template_files)

688

689 await _process_template_files_for_status(

690 template_files,

691 storage,

692 storage_bucket,

693 status,

694 )

695

696 except Exception as e:

697 status["error"] = str(e)

698 debug(f"Error getting template sync status: {e}")

699

700 return status

701

702

703async def _discover_template_files_for_status(

704 template_paths: list[AsyncPath],

705) -> list[dict[str, t.Any]]:

706 template_files = []

707 for base_path in template_paths:

708 if await base_path.exists():

709 async for file_path in base_path.rglob("*.html"):

710 if await file_path.is_file():

711 rel_path = file_path.relative_to(base_path)

712 template_files.append(

713 {

714 "local_path": file_path,

715 "storage_path": str(rel_path),

716 },

717 )

718 return template_files

719

720

721async def _process_template_files_for_status(

722 template_files: list[dict[str, t.Any]],

723 storage: t.Any,

724 storage_bucket: str,

725 status: dict[str, t.Any],

726) -> None:

727 for template_info in template_files:

728 local_info = await get_file_info(Path(template_info["local_path"]))

729 remote_info = await _get_storage_file_info(

730 storage,

731 storage_bucket,

732 template_info["storage_path"],

733 )

734

735 file_status = _create_file_status_info(template_info, local_info, remote_info)

736 _update_status_counters(local_info, remote_info, file_status, status)

737

738 details_list = status["details"]

739 assert isinstance(details_list, list)

740 details_list.append(file_status)

741

742 _calculate_out_of_sync_total(status)

743

744

745def _create_file_status_info(

746 template_info: dict[str, t.Any],

747 local_info: dict[str, t.Any],

748 remote_info: dict[str, t.Any],

749) -> dict[str, t.Any]:

750 return {

751 "path": template_info["storage_path"],

752 "local_exists": local_info["exists"],

753 "remote_exists": remote_info["exists"],

754 }

755

756

757def _update_status_counters(

758 local_info: dict[str, t.Any],

759 remote_info: dict[str, t.Any],

760 file_status: dict[str, t.Any],

761 status: dict[str, t.Any],

762) -> None:

763 if local_info["exists"] and remote_info["exists"]:

764 if local_info["content_hash"] == remote_info["content_hash"]:

765 file_status["status"] = "in_sync"

766 status["in_sync"] = status["in_sync"] + 1

767 else:

768 file_status["status"] = "conflict"

769 file_status["local_mtime"] = local_info["mtime"]

770 file_status["remote_mtime"] = remote_info["mtime"]

771 status["conflicts"] = status["conflicts"] + 1

772 elif local_info["exists"]:

773 file_status["status"] = "local_only"

774 status["local_only"] = status["local_only"] + 1

775 elif remote_info["exists"]:

776 file_status["status"] = "remote_only"

777 status["remote_only"] = status["remote_only"] + 1

778 else:

779 file_status["status"] = "missing"

780

781

782def _calculate_out_of_sync_total(status: dict[str, t.Any]) -> None:

783 conflicts = status["conflicts"]

784 local_only = status["local_only"]

785 remote_only = status["remote_only"]

786 assert isinstance(conflicts, int)

787 assert isinstance(local_only, int)

788 assert isinstance(remote_only, int)

789 status["out_of_sync"] = conflicts + local_only + remote_only