Coverage for fastblocks/actions/sync/settings.py: 21%

1"""Settings file synchronization between filesystem and cloud storage.

3Settings sync is intentionally limited to filesystem and cloud storage only.

4Unlike templates, settings are not cached for security and consistency reasons.

5"""

7import typing as t

8from pathlib import Path

10import yaml

11from acb.debug import debug

12from anyio import Path as AsyncPath

14from .strategies import (

15 ConflictStrategy,

16 SyncDirection,

17 SyncResult,

18 SyncStrategy,

19 create_backup,

20 get_file_info,

21 resolve_conflict,

22 should_sync,

23)

26class SettingsSyncResult(SyncResult):

27 def __init__(

28 self,

29 *,

30 config_reloaded: list[str] | None = None,

31 adapters_affected: list[str] | None = None,

32 **kwargs: t.Any,

33 ) -> None:

34 super().__init__(**kwargs)

35 self.config_reloaded = config_reloaded if config_reloaded is not None else []

36 self.adapters_affected = (

37 adapters_affected if adapters_affected is not None else []

38 )

41async def sync_settings(

42 *,

43 settings_path: AsyncPath | None = None,

44 adapter_names: list[str] | None = None,

45 strategy: SyncStrategy | None = None,

46 storage_bucket: str | None = None,

47 reload_config: bool = True,

48) -> SettingsSyncResult:

49 config = _prepare_settings_sync_config(settings_path, strategy)

50 result = SettingsSyncResult()

52 if storage_bucket is None:

53 storage_bucket = await _get_default_settings_bucket()

55 storage = await _initialize_storage_only(result)

56 if not storage:

57 return result

59 settings_files = await _discover_settings_files(

60 config["settings_path"],

61 adapter_names,

62 )

63 if not settings_files:

64 debug("No settings files found to sync")

65 return result

67 debug(f"Found {len(settings_files)} settings files to sync")

69 await _sync_settings_files(

70 settings_files,

71 storage,

72 config["strategy"],

73 storage_bucket,

74 result,

75 )

77 await _handle_config_reload(reload_config, result)

79 debug(

80 f"Settings sync completed: {len(result.synced_items)} synced, {len(result.conflicts)} conflicts",

81 )

83 return result

86def _prepare_settings_sync_config(

87 settings_path: AsyncPath | None,

88 strategy: SyncStrategy | None,

89) -> dict[str, t.Any]:

90 return {

91 "settings_path": settings_path or AsyncPath("settings"),

92 "strategy": strategy or SyncStrategy(),

93 }

96async def _initialize_storage_only(result: SettingsSyncResult) -> t.Any | None:

97 try:

98 from acb.depends import depends

100 storage = depends.get("storage")

101 if not storage:

102 result.errors.append(Exception("Storage adapter not available"))

103 return None

104

105 return storage

106 except Exception as e:

107 result.errors.append(e)

108 return None

109

110

111async def _get_default_settings_bucket() -> str:

112 try:

113 storage_config_path = AsyncPath("settings/storage.yml")

114 if await storage_config_path.exists():

115 content = await storage_config_path.read_text()

116 config = yaml.safe_load(content)

117 if isinstance(config, dict):

118 bucket_name = config.get("buckets", {}).get("settings", "settings")

119 else:

120 bucket_name = "settings"

121 debug(f"Using settings bucket from config: {bucket_name}")

122 return bucket_name

123 except Exception as e:

124 debug(f"Could not load storage config, using default: {e}")

125 debug("Using fallback settings bucket: settings")

126 return "settings"

127

128

129async def _sync_settings_files(

130 settings_files: list[dict[str, t.Any]],

131 storage: t.Any,

132 strategy: SyncStrategy,

133 storage_bucket: str,

134 result: SettingsSyncResult,

135) -> None:

136 for settings_info in settings_files:

137 try:

138 file_result = await _sync_single_settings_file(

139 settings_info,

140 storage,

141 strategy,

142 storage_bucket,

143 )

144 _accumulate_settings_sync_results(file_result, result)

145

146 except Exception as e:

147 result.errors.append(e)

148 debug(f"Error syncing settings {settings_info['relative_path']}: {e}")

149

150

151def _accumulate_settings_sync_results(

152 file_result: dict[str, t.Any],

153 result: SettingsSyncResult,

154) -> None:

155 if file_result.get("synced"):

156 result.synced_items.extend(file_result["synced"])

157 result.adapters_affected.extend(file_result.get("adapters_affected", []))

158 if file_result.get("conflicts"):

159 result.conflicts.extend(file_result["conflicts"])

160 if file_result.get("errors"):

161 result.errors.extend(file_result["errors"])

162 if file_result.get("skipped"):

163 result.skipped.extend(file_result["skipped"])

164 if file_result.get("backed_up"):

165 result.backed_up.extend(file_result["backed_up"])

166

167

168async def _handle_config_reload(

169 reload_config: bool,

170 result: SettingsSyncResult,

171) -> None:

172 if reload_config and result.synced_items:

173 try:

174 await _reload_configuration(result.adapters_affected)

175 result.config_reloaded = result.adapters_affected.copy()

176 except Exception as e:

177 result.errors.append(e)

178 debug(f"Error reloading configuration: {e}")

179

180

181async def _discover_settings_files(

182 settings_path: AsyncPath,

183 adapter_names: list[str] | None = None,

184) -> list[dict[str, t.Any]]:

185 settings_files: list[dict[str, t.Any]] = []

186

187 if not await settings_path.exists():

188 debug(f"Settings path does not exist: {settings_path}")

189 return settings_files

190

191 for pattern in ("*.yml", "*.yaml"):

192 await _discover_files_with_pattern(

193 settings_path,

194 pattern,

195 adapter_names,

196 settings_files,

197 )

198

199 return settings_files

200

201

202async def _discover_files_with_pattern(

203 settings_path: AsyncPath,

204 pattern: str,

205 adapter_names: list[str] | None,

206 settings_files: list[dict[str, t.Any]],

207) -> None:

208 async for file_path in settings_path.rglob(pattern):

209 if await file_path.is_file():

210 await _process_settings_file(

211 file_path,

212 settings_path,

213 adapter_names,

214 settings_files,

215 )

216

217

218async def _process_settings_file(

219 file_path: AsyncPath,

220 settings_path: AsyncPath,

221 adapter_names: list[str] | None,

222 settings_files: list[dict[str, t.Any]],

223) -> None:

224 adapter_name = file_path.stem

225

226 if adapter_names and adapter_name not in adapter_names:

227 return

228

229 try:

230 rel_path = file_path.relative_to(settings_path)

231 settings_files.append(

232 {

233 "local_path": file_path,

234 "relative_path": rel_path,

235 "storage_path": str(rel_path),

236 "adapter_name": adapter_name,

237 },

238 )

239 except ValueError:

240 debug(f"Could not get relative path for {file_path}")

241

242

243async def _sync_single_settings_file(

244 settings_info: dict[str, t.Any],

245 storage: t.Any,

246 strategy: SyncStrategy,

247 storage_bucket: str,

248) -> dict[str, t.Any]:

249 local_path = settings_info["local_path"]

250 storage_path = settings_info["storage_path"]

251 adapter_name = settings_info["adapter_name"]

252

253 result = _create_sync_result()

254

255 try:

256 local_info, remote_info = await _get_file_infos(

257 local_path,

258 storage,

259 storage_bucket,

260 storage_path,

261 )

262

263 if not await _should_sync_file(

264 local_info,

265 remote_info,

266 strategy,

267 storage_path,

268 result,

269 ):

270 return result

271

272 if not await _validate_local_yaml(local_info, storage_path, result):

273 return result

274

275 await _execute_sync_operation(

276 local_path,

277 storage,

278 storage_bucket,

279 storage_path,

280 local_info,

281 remote_info,

282 strategy,

283 result,

284 )

285

286 if result["synced"]:

287 result["adapters_affected"].append(adapter_name)

288

289 except Exception as e:

290 result["errors"].append(e)

291 debug(f"Error in _sync_single_settings_file for {storage_path}: {e}")

292

293 return result

294

295

296def _create_sync_result() -> dict[str, t.Any]:

297 return {

298 "synced": [],

299 "conflicts": [],

300 "errors": [],

301 "skipped": [],

302 "backed_up": [],

303 "adapters_affected": [],

304 }

305

306

307async def _get_file_infos(

308 local_path: t.Any,

309 storage: t.Any,

310 storage_bucket: str,

311 storage_path: str,

312) -> tuple[dict[str, t.Any], dict[str, t.Any]]:

313 local_info = await get_file_info(Path(local_path))

314 remote_info = await _get_storage_file_info(storage, storage_bucket, storage_path)

315 return local_info, remote_info

316

317

318async def _should_sync_file(

319 local_info: dict[str, t.Any],

320 remote_info: dict[str, t.Any],

321 strategy: SyncStrategy,

322 storage_path: str,

323 result: dict[str, t.Any],

324) -> bool:

325 sync_needed, reason = should_sync(local_info, remote_info, strategy.direction)

326 if not sync_needed:

327 result["skipped"].append(f"{storage_path} ({reason})")

328 return False

329

330 debug(f"Syncing settings {storage_path}: {reason}")

331 return True

332

333

334async def _validate_local_yaml(

335 local_info: dict[str, t.Any],

336 storage_path: str,

337 result: dict[str, t.Any],

338) -> bool:

339 if local_info["exists"]:

340 try:

341 await _validate_yaml_content(local_info["content"])

342 except Exception as e:

343 result["errors"].append(f"Invalid YAML in {storage_path}: {e}")

344 return False

345 return True

346

347

348async def _execute_sync_operation(

349 local_path: t.Any,

350 storage: t.Any,

351 storage_bucket: str,

352 storage_path: str,

353 local_info: dict[str, t.Any],

354 remote_info: dict[str, t.Any],

355 strategy: SyncStrategy,

356 result: dict[str, t.Any],

357) -> None:

358 if _should_pull_settings(strategy, local_info, remote_info):

359 await _pull_settings(

360 local_path,

361 storage,

362 storage_bucket,

363 storage_path,

364 strategy,

365 result,

366 )

367 elif _should_push_settings(strategy, local_info, remote_info):

368 await _push_settings(

369 local_path,

370 storage,

371 storage_bucket,

372 storage_path,

373 strategy,

374 result,

375 )

376 elif _has_bidirectional_conflict(strategy, local_info, remote_info):

377 await _handle_settings_conflict(

378 local_path,

379 storage,

380 storage_bucket,

381 storage_path,

382 local_info,

383 remote_info,

384 strategy,

385 result,

386 )

387

388

389def _should_pull_settings(

390 strategy: SyncStrategy,

391 local_info: dict[str, t.Any],

392 remote_info: dict[str, t.Any],

393) -> bool:

394 return strategy.direction == SyncDirection.PULL or (

395 strategy.direction == SyncDirection.BIDIRECTIONAL

396 and remote_info["exists"]

397 and (not local_info["exists"] or remote_info["mtime"] > local_info["mtime"])

398 )

399

400

401def _should_push_settings(

402 strategy: SyncStrategy,

403 local_info: dict[str, t.Any],

404 remote_info: dict[str, t.Any],

405) -> bool:

406 return strategy.direction == SyncDirection.PUSH or (

407 strategy.direction == SyncDirection.BIDIRECTIONAL

408 and local_info["exists"]

409 and (not remote_info["exists"] or local_info["mtime"] > remote_info["mtime"])

410 )

411

412

413def _has_bidirectional_conflict(

414 strategy: SyncStrategy,

415 local_info: dict[str, t.Any],

416 remote_info: dict[str, t.Any],

417) -> bool:

418 return (

419 strategy.direction == SyncDirection.BIDIRECTIONAL

420 and local_info["exists"]

421 and remote_info["exists"]

422 )

423

424

425async def _get_storage_file_info(

426 storage: t.Any,

427 bucket: str,

428 file_path: str,

429) -> dict[str, t.Any]:

430 try:

431 bucket_obj = getattr(storage, bucket, None)

432

433 if not bucket_obj:

434 await storage._create_bucket(bucket)

435 bucket_obj = getattr(storage, bucket)

436

437 exists = await bucket_obj.exists(file_path)

438

439 if not exists:

440 return {

441 "exists": False,

442 "size": 0,

443 "mtime": 0,

444 "content_hash": None,

445 }

446

447 content = await bucket_obj.read(file_path)

448 metadata = await bucket_obj.stat(file_path)

449

450 import hashlib

451

452 content_hash = hashlib.blake2b(content).hexdigest()

453

454 return {

455 "exists": True,

456 "size": len(content),

457 "mtime": metadata.get("mtime", 0),

458 "content_hash": content_hash,

459 "content": content,

460 }

461

462 except Exception as e:

463 debug(f"Error getting storage file info for {file_path}: {e}")

464 return {

465 "exists": False,

466 "size": 0,

467 "mtime": 0,

468 "content_hash": None,

469 "error": str(e),

470 }

471

472

473async def _validate_yaml_content(content: bytes) -> None:

474 try:

475 import yaml

476

477 yaml.safe_load(content.decode())

478 except Exception as e:

479 msg = f"Invalid YAML content: {e}"

480 raise ValueError(msg)

481

482

483async def _pull_settings(

484 local_path: AsyncPath,

485 storage: t.Any,

486 bucket: str,

487 storage_path: str,

488 strategy: SyncStrategy,

489 result: dict[str, t.Any],

490) -> None:

491 try:

492 bucket_obj = getattr(storage, bucket)

493

494 if strategy.dry_run:

495 debug(f"DRY RUN: Would pull {storage_path} to {local_path}")

496 result["synced"].append(f"PULL(dry-run): {storage_path}")

497 return

498

499 if await local_path.exists() and strategy.backup_on_conflict:

500 backup_path = await create_backup(Path(local_path))

501 result["backed_up"].append(str(backup_path))

502

503 content = await bucket_obj.read(storage_path)

504

505 await _validate_yaml_content(content)

506

507 await local_path.parent.mkdir(parents=True, exist_ok=True)

508

509 await local_path.write_bytes(content)

510

511 result["synced"].append(f"PULL: {storage_path}")

512 debug(f"Pulled settings from storage: {storage_path}")

513

514 except Exception as e:

515 result["errors"].append(e)

516 debug(f"Error pulling settings {storage_path}: {e}")

517

518

519async def _push_settings(

520 local_path: AsyncPath,

521 storage: t.Any,

522 bucket: str,

523 storage_path: str,

524 strategy: SyncStrategy,

525 result: dict[str, t.Any],

526) -> None:

527 try:

528 bucket_obj = getattr(storage, bucket)

529

530 if strategy.dry_run:

531 debug(f"DRY RUN: Would push {local_path} to {storage_path}")

532 result["synced"].append(f"PUSH(dry-run): {storage_path}")

533 return

534

535 content = await local_path.read_bytes()

536 await _validate_yaml_content(content)

537

538 await bucket_obj.write(storage_path, content)

539

540 result["synced"].append(f"PUSH: {storage_path}")

541 debug(f"Pushed settings to storage: {storage_path}")

542

543 except Exception as e:

544 result["errors"].append(e)

545 debug(f"Error pushing settings {storage_path}: {e}")

546

547

548async def _handle_settings_conflict(

549 local_path: AsyncPath,

550 storage: t.Any,

551 bucket: str,

552 storage_path: str,

553 local_info: dict[str, t.Any],

554 remote_info: dict[str, t.Any],

555 strategy: SyncStrategy,

556 result: dict[str, t.Any],

557) -> None:

558 try:

559 if strategy.conflict_strategy == ConflictStrategy.MANUAL:

560 result["conflicts"].append(

561 {

562 "path": storage_path,

563 "local_mtime": local_info["mtime"],

564 "remote_mtime": remote_info["mtime"],

565 "reason": "manual_resolution_required",

566 },

567 )

568 return

569

570 try:

571 await _validate_yaml_content(local_info["content"])

572 await _validate_yaml_content(remote_info["content"])

573 except Exception as e:

574 result["errors"].append(f"Invalid YAML during conflict resolution: {e}")

575 return

576

577 resolved_content, resolution_reason = await resolve_conflict(

578 Path(local_path),

579 remote_info["content"],

580 local_info["content"],

581 strategy.conflict_strategy,

582 local_info["mtime"],

583 remote_info["mtime"],

584 )

585

586 if strategy.dry_run:

587 debug(

588 f"DRY RUN: Would resolve conflict for {storage_path}: {resolution_reason}",

589 )

590 result["synced"].append(

591 f"CONFLICT(dry-run): {storage_path} - {resolution_reason}",

592 )

593 return

594

595 if (

596 strategy.backup_on_conflict

597 or strategy.conflict_strategy == ConflictStrategy.BACKUP_BOTH

598 ):

599 backup_path = await create_backup(Path(local_path), "conflict")

600 result["backed_up"].append(str(backup_path))

601

602 if resolved_content == remote_info["content"]:

603 await local_path.write_bytes(resolved_content)

604 result["synced"].append(

605 f"CONFLICT->REMOTE: {storage_path} - {resolution_reason}",

606 )

607 else:

608 bucket_obj = getattr(storage, bucket)

609 await bucket_obj.write(storage_path, resolved_content)

610 result["synced"].append(

611 f"CONFLICT->LOCAL: {storage_path} - {resolution_reason}",

612 )

613

614 debug(f"Resolved settings conflict: {storage_path} - {resolution_reason}")

615

616 except Exception as e:

617 result["errors"].append(e)

618 result["conflicts"].append(

619 {

620 "path": storage_path,

621 "error": str(e),

622 "reason": "resolution_failed",

623 },

624 )

625

626

627async def _reload_configuration(adapter_names: list[str]) -> None:

628 try:

629 from acb.config import reload_config # type: ignore[attr-defined]

630 from acb.depends import depends

631

632 config = await reload_config()

633 depends.set("config", config)

634 debug(f"Reloaded configuration for adapters: {adapter_names}")

635 except Exception as e:

636 debug(f"Error reloading configuration: {e}")

637 raise

638

639

640async def backup_settings(

641 settings_path: AsyncPath | None = None,

642 backup_suffix: str | None = None,

643) -> dict[str, t.Any]:

644 settings_path = settings_path or AsyncPath("settings")

645 backup_suffix = backup_suffix or _generate_backup_suffix()

646

647 result = _create_backup_result()

648

649 try:

650 if not await settings_path.exists():

651 result["errors"].append(f"Settings path does not exist: {settings_path}")

652 return result

653

654 await _backup_files_with_patterns(settings_path, backup_suffix, result)

655

656 except Exception as e:

657 result["errors"].append(str(e))

658 debug(f"Error in backup_settings: {e}")

659

660 return result

661

662

663def _generate_backup_suffix() -> str:

664 import time

665

666 timestamp = int(time.time())

667 return f"backup_{timestamp}"

668

669

670def _create_backup_result() -> dict[str, t.Any]:

671 return {

672 "backed_up": [],

673 "errors": [],

674 "skipped": [],

675 }

676

677

678async def _backup_files_with_patterns(

679 settings_path: AsyncPath,

680 backup_suffix: str,

681 result: dict[str, t.Any],

682) -> None:

683 patterns = ["*.yml", "*.yaml"]

684

685 for pattern in patterns:

686 await _backup_files_with_pattern(settings_path, pattern, backup_suffix, result)

687

688

689async def _backup_files_with_pattern(

690 settings_path: AsyncPath,

691 pattern: str,

692 backup_suffix: str,

693 result: dict[str, t.Any],

694) -> None:

695 async for file_path in settings_path.rglob(pattern):

696 if await file_path.is_file():

697 await _backup_single_file(file_path, backup_suffix, result)

698

699

700async def _backup_single_file(

701 file_path: AsyncPath,

702 backup_suffix: str,

703 result: dict[str, t.Any],

704) -> None:

705 try:

706 backup_path = await create_backup(Path(file_path), backup_suffix)

707 result["backed_up"].append(str(backup_path))

708 except Exception as e:

709 result["errors"].append(f"{file_path}: {e}")

710

711

712async def get_settings_sync_status(

713 settings_path: AsyncPath | None = None,

714 storage_bucket: str = "settings",

715) -> dict[str, t.Any]:

716 if settings_path is None:

717 settings_path = AsyncPath("settings")

718

719 status: dict[str, t.Any] = {

720 "total_settings": 0,

721 "in_sync": 0,

722 "out_of_sync": 0,

723 "local_only": 0,

724 "remote_only": 0,

725 "conflicts": 0,

726 "details": [],

727 }

728

729 try:

730 from acb.depends import depends

731

732 storage = depends.get("storage")

733

734 if not storage:

735 status["error"] = "Storage adapter not available"

736 return status

737

738 settings_files = await _discover_settings_files(settings_path)

739 status["total_settings"] = len(settings_files)

740

741 for settings_info in settings_files:

742 local_info = await get_file_info(Path(settings_info["local_path"]))

743 remote_info = await _get_storage_file_info(

744 storage,

745 storage_bucket,

746 settings_info["storage_path"],

747 )

748

749 file_status: dict[str, t.Any] = {

750 "path": settings_info["storage_path"],

751 "adapter": settings_info["adapter_name"],

752 "local_exists": local_info["exists"],

753 "remote_exists": remote_info["exists"],

754 }

755

756 if local_info["exists"] and remote_info["exists"]:

757 if local_info["content_hash"] == remote_info["content_hash"]:

758 file_status["status"] = "in_sync"

759 status["in_sync"] += 1

760 else:

761 file_status["status"] = "conflict"

762 file_status["local_mtime"] = local_info["mtime"]

763 file_status["remote_mtime"] = remote_info["mtime"]

764 status["conflicts"] += 1

765 elif local_info["exists"]:

766 file_status["status"] = "local_only"

767 status["local_only"] += 1

768 elif remote_info["exists"]:

769 file_status["status"] = "remote_only"

770 status["remote_only"] += 1

771 else:

772 file_status["status"] = "missing"

773

774 status["details"].append(file_status)

775

776 status["out_of_sync"] = (

777 status["conflicts"] + status["local_only"] + status["remote_only"]

778 )

779

780 except Exception as e:

781 status["error"] = str(e)

782 debug(f"Error getting settings sync status: {e}")

783

784 return status

785

786

787async def validate_all_settings(

788 settings_path: AsyncPath | None = None,

789) -> dict[str, t.Any]:

790 if settings_path is None:

791 settings_path = AsyncPath("settings")

792

793 result: dict[str, t.Any] = {

794 "valid": [],

795 "invalid": [],

796 "missing": [],

797 "total_checked": 0,

798 }

799

800 try:

801 settings_files = await _discover_settings_files(settings_path)

802 result["total_checked"] = len(settings_files)

803

804 for settings_info in settings_files:

805 file_path = settings_info["local_path"]

806

807 if not await file_path.exists():

808 result["missing"].append(str(file_path))

809 continue

810

811 try:

812 content = await file_path.read_bytes()

813 await _validate_yaml_content(content)

814 result["valid"].append(str(file_path))

815 except Exception as e:

816 result["invalid"].append(

817 {

818 "path": str(file_path),

819 "error": str(e),

820 },

821 )

822

823 except Exception as e:

824 result["error"] = str(e)

825 debug(f"Error validating settings: {e}")

826

827 return result