Coverage for app/services/backup_scheduler.py: 100%

1"""Built-in daily backup scheduling and retention. 

2 

3OPENHANGAR_BACKUP_TIME (HH:MM UTC, empty = disabled) runs the existing 

4run_backup() from a daemon thread once a day, guarded by the same advisory 

5lock mechanism as the other schedulers so only one gunicorn worker produces 

6a ZIP per tick. After every *successful* backup, retention prunes per 

7OPENHANGAR_BACKUP_RETENTION: 'simple' (default) keeps the newest 

8OPENHANGAR_BACKUP_KEEP backups; 'gfs' keeps everything for 

9OPENHANGAR_BACKUP_KEEP_DAYS days, then the newest backup per week for 

10OPENHANGAR_BACKUP_KEEP_WEEKS weeks, per month for 

11OPENHANGAR_BACKUP_KEEP_MONTHS months, then per year forever. A failed 

12backup never triggers pruning, so a broken pipeline cannot silently erase 

13the archives that still exist. 

14""" 

15 

16import logging 

17import os 

18from datetime import UTC, date, timedelta 

19from typing import Any 

20 

21log = logging.getLogger(__name__) 

22 

23# See the lock id registry in services/advisory_lock.py. 

24BACKUP_LOCK_ID = 7283910460 

25 

26DEFAULT_KEEP = 30 

27 

28 

29def parse_backup_time() -> "tuple[int, int] | None": 

30 """Return (hour, minute) from OPENHANGAR_BACKUP_TIME, or None when unset. 

31 

32 Raises ValueError with a human-readable message if the value is invalid. 

33 """ 

34 raw = os.environ.get("OPENHANGAR_BACKUP_TIME", "").strip() 

35 if not raw: 

36 return None 

37 err = ( 

38 f"OPENHANGAR_BACKUP_TIME={raw!r} is invalid — expected HH:MM UTC " 

39 f"(e.g. '03:30'), or empty to disable scheduled backups" 

40 ) 

41 parts = raw.split(":") 

42 if len(parts) != 2: 

43 raise ValueError(err) 

44 try: 

45 hour, minute = int(parts[0]), int(parts[1]) 

46 except ValueError: 

47 raise ValueError(err) 

48 if not (0 <= hour <= 23 and 0 <= minute <= 59): 

49 raise ValueError(err) 

50 return hour, minute 

51 

52 

53def _parse_positive_int(env_name: str, default: int) -> int: 

54 """Positive-integer env var with a default; ValueError on bad values.""" 

55 raw = os.environ.get(env_name, "").strip() 

56 if not raw: 

57 return default 

58 err = f"{env_name}={raw!r} is invalid — expected a positive integer" 

59 try: 

60 value = int(raw) 

61 except ValueError: 

62 raise ValueError(err) 

63 if value < 1: 

64 raise ValueError(err) 

65 return value 

66 

67 

68def parse_backup_keep() -> int: 

69 """Retention count for the 'simple' scheme (OPENHANGAR_BACKUP_KEEP).""" 

70 return _parse_positive_int("OPENHANGAR_BACKUP_KEEP", DEFAULT_KEEP) 

71 

72 

73RETENTION_SIMPLE = "simple" 

74RETENTION_GFS = "gfs" 

75 

76DEFAULT_KEEP_DAYS = 7 

77DEFAULT_KEEP_WEEKS = 4 

78DEFAULT_KEEP_MONTHS = 12 

79 

80 

81def parse_backup_retention() -> str: 

82 """Retention scheme from OPENHANGAR_BACKUP_RETENTION (default 'simple'). 

83 

84 'simple' keeps the newest OPENHANGAR_BACKUP_KEEP backups; 'gfs' keeps 

85 everything for OPENHANGAR_BACKUP_KEEP_DAYS days, then the newest backup 

86 per week for OPENHANGAR_BACKUP_KEEP_WEEKS weeks, per month for 

87 OPENHANGAR_BACKUP_KEEP_MONTHS months, and per year forever. 

88 """ 

89 raw = os.environ.get("OPENHANGAR_BACKUP_RETENTION", "").strip().lower() 

90 if not raw: 

91 return RETENTION_SIMPLE 

92 if raw not in (RETENTION_SIMPLE, RETENTION_GFS): 

93 raise ValueError( 

94 f"OPENHANGAR_BACKUP_RETENTION={raw!r} is invalid — expected " 

95 f"'{RETENTION_SIMPLE}' or '{RETENTION_GFS}'" 

96 ) 

97 return raw 

98 

99 

100def parse_backup_keep_days() -> int: 

101 return _parse_positive_int("OPENHANGAR_BACKUP_KEEP_DAYS", DEFAULT_KEEP_DAYS) 

102 

103 

104def parse_backup_keep_weeks() -> int: 

105 return _parse_positive_int("OPENHANGAR_BACKUP_KEEP_WEEKS", DEFAULT_KEEP_WEEKS) 

106 

107 

108def parse_backup_keep_months() -> int: 

109 return _parse_positive_int("OPENHANGAR_BACKUP_KEEP_MONTHS", DEFAULT_KEEP_MONTHS) 

110 

111 

112def parse_auto_upgrade_enabled() -> bool: 

113 """OPENHANGAR_AUTO_UPGRADE (true/false, default false). 

114 

115 Raises ValueError with a human-readable message if the value is set but 

116 not recognized as a boolean. 

117 """ 

118 raw = os.environ.get("OPENHANGAR_AUTO_UPGRADE", "").strip().lower() 

119 if not raw: 

120 return False 

121 if raw in ("1", "true", "yes"): 

122 return True 

123 if raw in ("0", "false", "no"): 

124 return False 

125 raise ValueError( 

126 f"OPENHANGAR_AUTO_UPGRADE={raw!r} is invalid — expected true/false" 

127 ) 

128 

129 

130def _gfs_keep_ids( 

131 ok_records: "list[Any]", today: "date", days: int, weeks: int, months: int 

132) -> "set[int]": 

133 """Grandfather-father-son keep-set over newest-first successful backups. 

134 

135 Everything younger than `days` days is kept. Beyond that, the newest 

136 backup of each ISO week is kept for the first `weeks` distinct weeks, 

137 then the newest per calendar month for `months` distinct months, then 

138 the newest per calendar year forever. Counting distinct periods (like 

139 restic's --keep-weekly) means gaps in the schedule never shrink the 

140 retained history. 

141 """ 

142 keep: set[int] = set() 

143 weeks_seen: set[tuple[int, int]] = set() 

144 months_seen: set[tuple[int, int]] = set() 

145 years_seen: set[int] = set() 

146 daily_cutoff = today - timedelta(days=days) 

147 for record in ok_records: 

148 created = record.created_at.date() 

149 if created >= daily_cutoff: 

150 keep.add(record.id) 

151 continue 

152 iso = created.isocalendar() 

153 wkey = (iso[0], iso[1]) 

154 if wkey in weeks_seen: 

155 continue # this week is already represented by a newer backup 

156 if len(weeks_seen) < weeks: 

157 weeks_seen.add(wkey) 

158 keep.add(record.id) 

159 continue 

160 mkey = (created.year, created.month) 

161 if mkey in months_seen: 

162 continue 

163 if len(months_seen) < months: 

164 months_seen.add(mkey) 

165 keep.add(record.id) 

166 continue 

167 if created.year not in years_seen: 

168 years_seen.add(created.year) 

169 keep.add(record.id) 

170 return keep 

171 

172 

173def prune_old_backups(keep: "int | None" = None, today: "date | None" = None) -> int: 

174 """Delete successful backups that fall outside the retention scheme. 

175 

176 With an explicit `keep` (or OPENHANGAR_BACKUP_RETENTION=simple, the 

177 default) the newest `keep` backups survive. With the 'gfs' scheme the 

178 day/week/month/year tiers decide. Returns the number of records 

179 removed. A record whose file cannot be deleted is kept so the operator 

180 can still see (and clean up) the stranded archive. Failed records 

181 carry no archive and are left alone. 

182 """ 

183 from models import BackupRecord, db # pyright: ignore[reportMissingImports] 

184 

185 ok_records = ( 

186 BackupRecord.query.filter_by(status="ok") 

187 .order_by(BackupRecord.created_at.desc(), BackupRecord.id.desc()) 

188 .all() 

189 ) 

190 if keep is None and parse_backup_retention() == RETENTION_GFS: 

191 keep_ids = _gfs_keep_ids( 

192 ok_records, 

193 today or date.today(), 

194 parse_backup_keep_days(), 

195 parse_backup_keep_weeks(), 

196 parse_backup_keep_months(), 

197 ) 

198 to_delete = [r for r in ok_records if r.id not in keep_ids] 

199 else: 

200 if keep is None: 

201 keep = parse_backup_keep() 

202 to_delete = ok_records[keep:] 

203 

204 removed = 0 

205 for record in to_delete: 

206 try: 

207 if record.path and os.path.exists(record.path): 

208 os.remove(record.path) 

209 except OSError: 

210 log.warning( 

211 "Backup retention: could not delete %s — keeping its record", 

212 record.path, 

213 ) 

214 continue 

215 db.session.delete(record) 

216 removed += 1 

217 db.session.commit() 

218 return removed 

219 

220 

221def _maybe_trigger_auto_upgrade() -> None: 

222 """Trigger a one-click upgrade if OPENHANGAR_AUTO_UPGRADE is enabled and a 

223 newer version is available. Only ever called right after a *successful* 

224 scheduled backup (see run_scheduled_backup below) — an automatic upgrade 

225 must never run without a fresh backup to fall back on. 

226 """ 

227 if not parse_auto_upgrade_enabled(): 

228 return 

229 upgrade_dir = os.environ.get("OPENHANGAR_UPGRADE_DIR", "").strip() 

230 if not upgrade_dir: 

231 # _validate_config() blocks startup on this combination, so this is 

232 # defensive — it should not be reachable in practice. 

233 log.warning( 

234 "OPENHANGAR_AUTO_UPGRADE is set but OPENHANGAR_UPGRADE_DIR is not — skipping" 

235 ) 

236 return 

237 

238 from config.routes import ( # pyright: ignore[reportMissingImports] 

239 _upgrade_trigger_state, 

240 _write_upgrade_trigger, 

241 ) 

242 

243 state = _upgrade_trigger_state(upgrade_dir) 

244 if state != "idle": 

245 log.info("Auto-upgrade: an upgrade is already %s — skipping", state) 

246 return 

247 

248 # Fresh check right here rather than reusing the cached update_available 

249 # AppSetting flag — that flag is refreshed by a background thread on its 

250 # own schedule (0-6h random delay, then every 24h) and could be stale by 

251 # the time the nightly backup runs. 

252 from services.version_service import ( # pyright: ignore[reportMissingImports] 

253 fetch_latest_version, 

254 is_newer_version, 

255 ) 

256 

257 current = os.environ.get("OPENHANGAR_VERSION", "development") 

258 latest = fetch_latest_version() 

259 if not is_newer_version(current, latest): 

260 log.info( 

261 "Auto-upgrade: no newer version available (current=%s, latest=%s) — nothing to do", 

262 current, 

263 latest, 

264 ) 

265 return 

266 

267 _write_upgrade_trigger(upgrade_dir, "system:auto-upgrade") 

268 log.info("Auto-upgrade: triggered upgrade from %s to %s", current, latest) 

269 

270 

271def run_scheduled_backup(app: "object") -> None: 

272 """One scheduled tick: back up, then prune retention on success only.""" 

273 from models import db # pyright: ignore[reportMissingImports] 

274 

275 from services.advisory_lock import ( 

276 advisory_lock_scope, # pyright: ignore[reportMissingImports] 

277 ) 

278 

279 with app.app_context(): # type: ignore[attr-defined] 

280 try: 

281 with advisory_lock_scope(db, BACKUP_LOCK_ID) as acquired: 

282 if not acquired: 

283 log.info( 

284 "Scheduled backup: another worker holds the lock — skipping" 

285 ) 

286 return 

287 from config.routes import ( 

288 run_backup, # pyright: ignore[reportMissingImports] 

289 ) 

290 

291 try: 

292 record = run_backup() 

293 except RuntimeError: 

294 log.exception("Scheduled backup failed — retention pruning skipped") 

295 return 

296 log.info("Scheduled backup OK: %s", record.filename) 

297 from services.backup_verification import ( 

298 verify_and_alert, # pyright: ignore[reportMissingImports] 

299 ) 

300 

301 verify_and_alert(record) 

302 removed = prune_old_backups() 

303 if removed: 

304 log.info("Backup retention: pruned %d old backup(s)", removed) 

305 _maybe_trigger_auto_upgrade() 

306 except Exception: 

307 log.exception("Error in scheduled backup run") 

308 

309 

310def _backup_daily_loop(app: "object", run_hour: int, run_minute: int) -> None: 

311 import time 

312 from datetime import datetime, timedelta 

313 

314 log.info("Backup scheduled daily at %02d:%02d UTC", run_hour, run_minute) 

315 while True: 

316 now = datetime.now(UTC) 

317 next_run = now.replace( 

318 hour=run_hour, minute=run_minute, second=0, microsecond=0 

319 ) 

320 if next_run <= now: 

321 next_run += timedelta(days=1) 

322 time.sleep((next_run - datetime.now(UTC)).total_seconds()) 

323 run_scheduled_backup(app) 

324 

325 

326def start_backup_scheduler(app: "object") -> None: 

327 """Start the daily backup thread when OPENHANGAR_BACKUP_TIME is set.""" 

328 import threading 

329 

330 schedule = parse_backup_time() 

331 if schedule is None: 

332 log.info("OPENHANGAR_BACKUP_TIME not set — built-in backup scheduling disabled") 

333 return 

334 threading.Thread( 

335 target=_backup_daily_loop, 

336 args=(app, schedule[0], schedule[1]), 

337 daemon=True, 

338 name="backup-scheduler", 

339 ).start()