Coverage for app/services/backup_scheduler.py: 100%

1"""Built-in daily backup scheduling and retention. 

2 

3OPENHANGAR_BACKUP_TIME (HH:MM UTC, empty = disabled) runs the existing 

4run_backup() from a daemon thread once a day, guarded by the same advisory 

5lock mechanism as the other schedulers so only one gunicorn worker produces 

6a ZIP per tick. After every *successful* backup, retention prunes per 

7OPENHANGAR_BACKUP_RETENTION: 'simple' (default) keeps the newest 

8OPENHANGAR_BACKUP_KEEP backups; 'gfs' keeps everything for 

9OPENHANGAR_BACKUP_KEEP_DAYS days, then the newest backup per week for 

10OPENHANGAR_BACKUP_KEEP_WEEKS weeks, per month for 

11OPENHANGAR_BACKUP_KEEP_MONTHS months, then per year forever. A failed 

12backup never triggers pruning, so a broken pipeline cannot silently erase 

13the archives that still exist. 

14""" 

15 

16import logging 

17import os 

18from datetime import UTC, date, timedelta 

19from typing import Any 

20 

21log = logging.getLogger(__name__) 

22 

23# See the lock id registry in services/advisory_lock.py. 

24BACKUP_LOCK_ID = 7283910460 

25 

26DEFAULT_KEEP = 30 

27 

28 

29def parse_backup_time() -> "tuple[int, int] | None": 

30 """Return (hour, minute) from OPENHANGAR_BACKUP_TIME, or None when unset. 

31 

32 Raises ValueError with a human-readable message if the value is invalid. 

33 """ 

34 raw = os.environ.get("OPENHANGAR_BACKUP_TIME", "").strip() 

35 if not raw: 

36 return None 

37 err = ( 

38 f"OPENHANGAR_BACKUP_TIME={raw!r} is invalid — expected HH:MM UTC " 

39 f"(e.g. '03:30'), or empty to disable scheduled backups" 

40 ) 

41 parts = raw.split(":") 

42 if len(parts) != 2: 

43 raise ValueError(err) 

44 try: 

45 hour, minute = int(parts[0]), int(parts[1]) 

46 except ValueError: 

47 raise ValueError(err) 

48 if not (0 <= hour <= 23 and 0 <= minute <= 59): 

49 raise ValueError(err) 

50 return hour, minute 

51 

52 

53def _parse_positive_int(env_name: str, default: int) -> int: 

54 """Positive-integer env var with a default; ValueError on bad values.""" 

55 raw = os.environ.get(env_name, "").strip() 

56 if not raw: 

57 return default 

58 err = f"{env_name}={raw!r} is invalid — expected a positive integer" 

59 try: 

60 value = int(raw) 

61 except ValueError: 

62 raise ValueError(err) 

63 if value < 1: 

64 raise ValueError(err) 

65 return value 

66 

67 

68def parse_backup_keep() -> int: 

69 """Retention count for the 'simple' scheme (OPENHANGAR_BACKUP_KEEP).""" 

70 return _parse_positive_int("OPENHANGAR_BACKUP_KEEP", DEFAULT_KEEP) 

71 

72 

73RETENTION_SIMPLE = "simple" 

74RETENTION_GFS = "gfs" 

75 

76DEFAULT_KEEP_DAYS = 7 

77DEFAULT_KEEP_WEEKS = 4 

78DEFAULT_KEEP_MONTHS = 12 

79 

80 

81def parse_backup_retention() -> str: 

82 """Retention scheme from OPENHANGAR_BACKUP_RETENTION (default 'simple'). 

83 

84 'simple' keeps the newest OPENHANGAR_BACKUP_KEEP backups; 'gfs' keeps 

85 everything for OPENHANGAR_BACKUP_KEEP_DAYS days, then the newest backup 

86 per week for OPENHANGAR_BACKUP_KEEP_WEEKS weeks, per month for 

87 OPENHANGAR_BACKUP_KEEP_MONTHS months, and per year forever. 

88 """ 

89 raw = os.environ.get("OPENHANGAR_BACKUP_RETENTION", "").strip().lower() 

90 if not raw: 

91 return RETENTION_SIMPLE 

92 if raw not in (RETENTION_SIMPLE, RETENTION_GFS): 

93 raise ValueError( 

94 f"OPENHANGAR_BACKUP_RETENTION={raw!r} is invalid — expected " 

95 f"'{RETENTION_SIMPLE}' or '{RETENTION_GFS}'" 

96 ) 

97 return raw 

98 

99 

100def parse_backup_keep_days() -> int: 

101 return _parse_positive_int("OPENHANGAR_BACKUP_KEEP_DAYS", DEFAULT_KEEP_DAYS) 

102 

103 

104def parse_backup_keep_weeks() -> int: 

105 return _parse_positive_int("OPENHANGAR_BACKUP_KEEP_WEEKS", DEFAULT_KEEP_WEEKS) 

106 

107 

108def parse_backup_keep_months() -> int: 

109 return _parse_positive_int("OPENHANGAR_BACKUP_KEEP_MONTHS", DEFAULT_KEEP_MONTHS) 

110 

111 

112def _gfs_keep_ids( 

113 ok_records: "list[Any]", today: "date", days: int, weeks: int, months: int 

114) -> "set[int]": 

115 """Grandfather-father-son keep-set over newest-first successful backups. 

116 

117 Everything younger than `days` days is kept. Beyond that, the newest 

118 backup of each ISO week is kept for the first `weeks` distinct weeks, 

119 then the newest per calendar month for `months` distinct months, then 

120 the newest per calendar year forever. Counting distinct periods (like 

121 restic's --keep-weekly) means gaps in the schedule never shrink the 

122 retained history. 

123 """ 

124 keep: set[int] = set() 

125 weeks_seen: set[tuple[int, int]] = set() 

126 months_seen: set[tuple[int, int]] = set() 

127 years_seen: set[int] = set() 

128 daily_cutoff = today - timedelta(days=days) 

129 for record in ok_records: 

130 created = record.created_at.date() 

131 if created >= daily_cutoff: 

132 keep.add(record.id) 

133 continue 

134 iso = created.isocalendar() 

135 wkey = (iso[0], iso[1]) 

136 if wkey in weeks_seen: 

137 continue # this week is already represented by a newer backup 

138 if len(weeks_seen) < weeks: 

139 weeks_seen.add(wkey) 

140 keep.add(record.id) 

141 continue 

142 mkey = (created.year, created.month) 

143 if mkey in months_seen: 

144 continue 

145 if len(months_seen) < months: 

146 months_seen.add(mkey) 

147 keep.add(record.id) 

148 continue 

149 if created.year not in years_seen: 

150 years_seen.add(created.year) 

151 keep.add(record.id) 

152 return keep 

153 

154 

155def prune_old_backups(keep: "int | None" = None, today: "date | None" = None) -> int: 

156 """Delete successful backups that fall outside the retention scheme. 

157 

158 With an explicit `keep` (or OPENHANGAR_BACKUP_RETENTION=simple, the 

159 default) the newest `keep` backups survive. With the 'gfs' scheme the 

160 day/week/month/year tiers decide. Returns the number of records 

161 removed. A record whose file cannot be deleted is kept so the operator 

162 can still see (and clean up) the stranded archive. Failed records 

163 carry no archive and are left alone. 

164 """ 

165 from models import BackupRecord, db # pyright: ignore[reportMissingImports] 

166 

167 ok_records = ( 

168 BackupRecord.query.filter_by(status="ok") 

169 .order_by(BackupRecord.created_at.desc(), BackupRecord.id.desc()) 

170 .all() 

171 ) 

172 if keep is None and parse_backup_retention() == RETENTION_GFS: 

173 keep_ids = _gfs_keep_ids( 

174 ok_records, 

175 today or date.today(), 

176 parse_backup_keep_days(), 

177 parse_backup_keep_weeks(), 

178 parse_backup_keep_months(), 

179 ) 

180 to_delete = [r for r in ok_records if r.id not in keep_ids] 

181 else: 

182 if keep is None: 

183 keep = parse_backup_keep() 

184 to_delete = ok_records[keep:] 

185 

186 removed = 0 

187 for record in to_delete: 

188 try: 

189 if record.path and os.path.exists(record.path): 

190 os.remove(record.path) 

191 except OSError: 

192 log.warning( 

193 "Backup retention: could not delete %s — keeping its record", 

194 record.path, 

195 ) 

196 continue 

197 db.session.delete(record) 

198 removed += 1 

199 db.session.commit() 

200 return removed 

201 

202 

203def run_scheduled_backup(app: "object") -> None: 

204 """One scheduled tick: back up, then prune retention on success only.""" 

205 from models import db # pyright: ignore[reportMissingImports] 

206 

207 from services.advisory_lock import ( 

208 advisory_lock_scope, # pyright: ignore[reportMissingImports] 

209 ) 

210 

211 with app.app_context(): # type: ignore[attr-defined] 

212 try: 

213 with advisory_lock_scope(db, BACKUP_LOCK_ID) as acquired: 

214 if not acquired: 

215 log.info( 

216 "Scheduled backup: another worker holds the lock — skipping" 

217 ) 

218 return 

219 from config.routes import ( 

220 run_backup, # pyright: ignore[reportMissingImports] 

221 ) 

222 

223 try: 

224 record = run_backup() 

225 except RuntimeError: 

226 log.exception("Scheduled backup failed — retention pruning skipped") 

227 return 

228 log.info("Scheduled backup OK: %s", record.filename) 

229 from services.backup_verification import ( 

230 verify_and_alert, # pyright: ignore[reportMissingImports] 

231 ) 

232 

233 verify_and_alert(record) 

234 removed = prune_old_backups() 

235 if removed: 

236 log.info("Backup retention: pruned %d old backup(s)", removed) 

237 except Exception: 

238 log.exception("Error in scheduled backup run") 

239 

240 

241def _backup_daily_loop(app: "object", run_hour: int, run_minute: int) -> None: 

242 import time 

243 from datetime import datetime, timedelta 

244 

245 log.info("Backup scheduled daily at %02d:%02d UTC", run_hour, run_minute) 

246 while True: 

247 now = datetime.now(UTC) 

248 next_run = now.replace( 

249 hour=run_hour, minute=run_minute, second=0, microsecond=0 

250 ) 

251 if next_run <= now: 

252 next_run += timedelta(days=1) 

253 time.sleep((next_run - datetime.now(UTC)).total_seconds()) 

254 run_scheduled_backup(app) 

255 

256 

257def start_backup_scheduler(app: "object") -> None: 

258 """Start the daily backup thread when OPENHANGAR_BACKUP_TIME is set.""" 

259 import threading 

260 

261 schedule = parse_backup_time() 

262 if schedule is None: 

263 log.info("OPENHANGAR_BACKUP_TIME not set — built-in backup scheduling disabled") 

264 return 

265 threading.Thread( 

266 target=_backup_daily_loop, 

267 args=(app, schedule[0], schedule[1]), 

268 daemon=True, 

269 name="backup-scheduler", 

270 ).start()