Coverage for apio/managers/remote

1# This file is part of the Apio project

3# Project authors:

4# * Jesús Arroyo (Jesus89) (2016-2019)

5# * Juan Gonzalez (obijuan) (2019-2024)

6# * The Apio Team (2024-present)

7# License: GPLv2

9"""Manage the apio remote configuration"""

11# pylint: disable=duplicate-code

13import json

14from urllib.parse import urlparse, urlunparse, ParseResult

15from enum import Enum, unique

16from datetime import datetime

17from typing import Any

18from pathlib import Path, PurePosixPath

19import requests

20from apio.common.debug_util import is_debug

21from apio.common.apio_console import cout, fatal_error, cerror

22from apio.common.apio_styles import INFO, EMPH3

23from apio.utils import util, jsonc

24from apio.common.proto_util import (

25 proto_from_json_dict,

26 proto_to_json_dict,

27 check_is_required,

28)

29from apio.common.proto.stubs.apio_remote_config_pb2 import (

30 CachedRemoteConfigSpec,

31 RemoteConfigSpec,

32 CachedRemoteConfigMetadata,

33 RemoteConfigPackageSpec,

34)

37@unique

38class RemoteConfigPolicy(Enum):

39 """Represents possible requirements from the remote config."""

41 # -- Config is being used but can be a cached value, as long that it's

42 # -- not too old.

43 CACHED_OK = 1

44 # -- Config is being used and a fresh copy is that was fetch in this

45 # -- invocation of Apio is required.

46 GET_FRESH = 2

49def get_datetime_stamp(dt: datetime | None = None) -> str:

50 """Returns a string with time now as yyyy-mm-dd-hh-mm"""

51 if dt is None:

52 dt = datetime.now()

53 return dt.strftime("%Y-%m-%d-%H-%M")

56def days_between_datetime_stamps(ts1: str, ts2: str, default: Any) -> int:

57 """Given two values generated by get_datetime_stamp(), return the

58 number of days from ts1 to ts2. The value can be negative if ts2 is

59 earlier than ts1. Returns the given 'default' value if either timestamp

60 is invalid."""

62 # -- The parsing format.

63 fmt = "%Y-%m-%d-%H-%M"

65 # -- Convert to timedates

66 try:

67 datetime1 = datetime.strptime(ts1, fmt)

68 datetime2 = datetime.strptime(ts2, fmt)

69 except ValueError:

70 return default

72 # -- Round to beginning of day.

73 day1 = datetime(datetime1.year, datetime1.month, datetime1.day)

74 day2 = datetime(datetime2.year, datetime2.month, datetime2.day)

76 # -- Compute the diff in days.

77 delta_days: int = (day2 - day1).days

79 # -- All done.

80 assert isinstance(delta_days, int)

81 return delta_days

84def minutes_between_datetime_stamps(ts1: str, ts2: str, default: Any) -> int:

85 """Return the number of minutes from ts1 to ts2 or default if any of

86 the timestamps is invalid."""

88 # -- The parsing format.

89 fmt = "%Y-%m-%d-%H-%M"

91 # -- Convert to timedates

92 try:

93 datetime1 = datetime.strptime(ts1, fmt)

94 datetime2 = datetime.strptime(ts2, fmt)

95 except ValueError:

96 return default

98 # -- Calculate the diff in minutes.

99 delta_minutes = int((datetime2 - datetime1).total_seconds() / 60)

100 assert isinstance(delta_minutes, int), type(delta_minutes)

101 return delta_minutes

102

103

104class RemoteConfig:

105 """Class for managing the apio remote config and its local cache

106 ~/.apio/cached-remote-config.json

107 """

108

109 # -- Only these instance vars are allowed.

110 __slots__ = (

111 "remote_config_file_url",

112 "remote_config_ttl_days",

113 "remote_config_retry_minutes",

114 "_remote_config_policy",

115 "_cached_remote_config_path",

116 "_cached_remote_config",

117 )

118

119 remote_config_file_url: str

120 remote_config_ttl_days: int

121 remote_config_retry_minutes: int

122 _remote_config_policy: RemoteConfigPolicy

123 _cached_remote_config_path: Path

124 _cached_remote_config: CachedRemoteConfigSpec | None

125

126 def __init__(

127 self,

128 home_dir: Path,

129 remote_config_dir_url: str,

130 remote_config_ttl_days: int,

131 remote_config_retry_minutes: int,

132 remote_config_policy: RemoteConfigPolicy,

133 ):

134 """remote_config_url_template is a url string with the

135 placeholder {major} and {minor} for the apio's major and minor

136 version. '"""

137

138 # pylint: disable=too-many-arguments

139 # pylint: disable=too-many-positional-arguments

140

141 # -- Sanity check

142 assert isinstance(remote_config_ttl_days, int)

143 assert 0 < remote_config_ttl_days <= 30

144

145 # -- Sanity check

146 assert isinstance(remote_config_retry_minutes, int)

147 assert 0 < remote_config_retry_minutes <= (60 * 24)

148

149 # -- Get the current Apio version major.minor.patch.

150 ver_tuple = util.get_apio_version_tuple()

151

152 # -- Resolve and save the remote config file URL.

153 dir_url = urlparse(remote_config_dir_url)

154 dir_url_path = dir_url.path

155 file_url_path = str(

156 PurePosixPath(dir_url_path)

157 / f"apio-{ver_tuple[0]}.{ver_tuple[1]}.x.jsonc"

158 )

159 file_url = dir_url._replace(path=file_url_path)

160 self.remote_config_file_url: ParseResult = urlunparse(file_url)

161

162 if is_debug(1): 162 ↛ 163line 162 didn't jump to line 163 because the condition on line 162 was never true

163 cout(f"Remote config url: {self.remote_config_file_url}")

164

165 # -- Save remote url ttl setting.

166 self.remote_config_ttl_days: int = remote_config_ttl_days

167

168 # -- Save the remote config fetch retry minutes.

169 self.remote_config_retry_minutes: int = remote_config_retry_minutes

170

171 # -- Save remote config policy.

172 self._remote_config_policy: RemoteConfigPolicy = remote_config_policy

173

174 # -- Start with no remote config.

175 self._cached_remote_config: CachedRemoteConfigSpec | None = None

176

177 # -- Path to the local file with the cached remote config.

178 self._cached_remote_config_path: Path = (

179 home_dir / "cached-remote-config.json"

180 )

181

182 # -- Try to load the cached remote config. If found and read

183 # -- successfully it mutates self._cached_remote_config

184 self._maybe_load_cached_remote_config()

185

186 # -- Apply config policy

187 self._apply_remote_config_policy()

188

189 @staticmethod

190 def _announce_no_suitable_cached_config(reason: str):

191 """Show a message indicating that the cached remote config is being

192 skipped."""

193 cout(f"No suitable cached remote config file ({reason}).")

194

195 def _apply_remote_config_policy(self) -> None:

196 """Called after loading the profile file, to apply the remote config

197 policy for this invocation."""

198

199 # -- Case 1: A fresh config is required for the current command.

200 if self._remote_config_policy == RemoteConfigPolicy.GET_FRESH:

201 self._maybe_fetch_and_update_remote_config(error_is_fatal=True)

202 return

203

204 # -- Case 2: A fresh config is optional but there is no cached

205 # -- config so practically it's required.

206 assert self._remote_config_policy == RemoteConfigPolicy.CACHED_OK

207 if self._cached_remote_config is None:

208 if is_debug(1): 208 ↛ 209line 208 didn't jump to line 209 because the condition on line 208 was never true

209 cout("Cached remote config is not available.", style=INFO)

210 self._maybe_fetch_and_update_remote_config(error_is_fatal=True)

211 return

212

213 # -- Case 3: May need to fetch a new config but can continue with

214 # -- the cached config in case of a fetch failure.

215 #

216 # -- Get the cached config metadata.

217 check_is_required(

218 self._cached_remote_config,

219 "metadata",

220 "metadata.loaded_at",

221 "metadata.loaded_from",

222 )

223 metadata = self._cached_remote_config.metadata

224 last_fetch_timestamp = metadata.loaded_at

225 last_fetch_url = metadata.loaded_from

226

227 # -- Determine if we need a new config because the remote config URL

228 # -- was changed (e.g. with APIO_REMOTE_CONFIG_URL)

229 url_changed = last_fetch_url != self.remote_config_file_url

230

231 # -- Determine if there is time related reason to fetch a new config.

232 datetime_stamp_now = get_datetime_stamp()

233 days_since_last_fetch = days_between_datetime_stamps(

234 last_fetch_timestamp, datetime_stamp_now, default=99999

235 )

236 time_valid = 0 <= days_since_last_fetch < self.remote_config_ttl_days

237

238 minutes_since_refresh_failure = minutes_between_datetime_stamps(

239 metadata.refresh_failure_on, datetime_stamp_now, default=None

240 )

241 refresh_failed_recently = (

242 minutes_since_refresh_failure is not None

243 and (

244 0

245 <= minutes_since_refresh_failure

246 < self.remote_config_retry_minutes

247 )

248 )

249

250 # -- Dump info for debugging.

251 if is_debug(1): 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true

252 cout(

253 f"{days_since_last_fetch=}, {time_valid=}, {url_changed=}",

254 f"{minutes_since_refresh_failure=}, "

255 f"{refresh_failed_recently=}",

256 style=EMPH3,

257 )

258

259 # -- Fetch the new config if needed.

260 if url_changed or not time_valid:

261 reason = "source URL mismatch" if url_changed else "stale"

262 self._announce_no_suitable_cached_config(reason=reason)

263 if refresh_failed_recently: 263 ↛ 264line 263 didn't jump to line 264 because the condition on line 263 was never true

264 cout(

265 "Remote config fetch failed recently, "

266 + "will keep using cached config.",

267 style=INFO,

268 )

269 else:

270 self._maybe_fetch_and_update_remote_config(

271 error_is_fatal=False

272 )

273

274 @property

275 def data(self) -> RemoteConfigSpec:

276 """Returns the remote config that is applicable for this invocation.

277 Should not called if the context was initialized with NO_CONFIG."""

278 assert self._cached_remote_config is not None

279 return self._cached_remote_config.remote_config

280

281 @property

282 def metadata(self) -> CachedRemoteConfigMetadata:

283 """Returns the remote config metadata. Should not be called

284 if the context was initialized with NO_CONFIG."""

285 assert self._cached_remote_config is not None

286 return self._cached_remote_config.metadata

287

288 def get_package_config(

289 self,

290 package_name: str,

291 ) -> RemoteConfigPackageSpec:

292 """Given a package name, return the remote config information with the

293 version and fetch information.

294 """

295 return self.data.packages[package_name]

296

297 def _maybe_load_cached_remote_config(self):

298 """Try loading self._cached_remote_config from the local file

299 cached_remote_config.json. If the file not found, or from a different

300 version of apio do not do not change self._cached_remote_config.

301 This method doesn't check for file age or staleness.

302 """

303

304 # -- If the file doesn't exist then do nothing.

305 if not self._cached_remote_config_path.exists():

306 self._announce_no_suitable_cached_config("no cache file")

307 return

308

309 # -- Read the cached remote config file as a json dict and extract its

310 # -- fields. Handle invalid content gracefully, e.g. a corrupt or hand

311 # -- edited file, since this runs on every apio command.

312 try:

313 with open(

314 self._cached_remote_config_path, "r", encoding="utf8"

315 ) as f:

316 json_data = json.load(f)

317

318 # -- If the cached remote config is not from the same apio version

319 # -- as the current one, reject it.

320 config_apio_version = json_data.get("metadata", {}).get(

321 "loaded-by", ""

322 )

323 if config_apio_version != util.get_apio_version_str():

324 self._announce_no_suitable_cached_config(

325 "Apio version mismatch"

326 )

327 return

328

329 except (OSError, ValueError, AttributeError) as e:

330 self._announce_no_suitable_cached_config("couldn't proto parse")

331 cout(str(e), style=INFO)

332

333 # -- Convert to proto.

334 config_proto = proto_from_json_dict(json_data, CachedRemoteConfigSpec)

335 if config_proto is None:

336 self._announce_no_suitable_cached_config("Corrupt cached data")

337 return

338

339 # -- Update the remote config member.

340 self._cached_remote_config = config_proto

341

342 def _save(self):

343 """Save the cached remote config to a file"""

344

345 # -- Create the enclosing folder, if it does not exist yet

346 file_path = self._cached_remote_config_path

347 dir_path = file_path.parent

348 if not dir_path.exists(): 348 ↛ 349line 348 didn't jump to line 349 because the condition on line 348 was never true

349 dir_path.mkdir()

350

351 # -- Convert proto to json dict

352 json_dict = proto_to_json_dict(self._cached_remote_config)

353

354 # -- Write to file.

355 with open(self._cached_remote_config_path, "w", encoding="utf8") as f:

356 json.dump(json_dict, f, indent=2)

357

358 # -- Dump for debugging.

359 if is_debug(1): 359 ↛ 360line 359 didn't jump to line 360 because the condition on line 359 was never true

360 cout("Saved cached remote config:", style=EMPH3)

361 cout(json.dumps(json_dict, indent=2))

362

363 def _handle_soft_config_refresh_failure(

364 self, *, error_msg_lines: list[str]

365 ):

366 """Called to handle a soft failure of a remote config refresh.

367 That is, an error, from which we recover by using the cached

368 remote config."""

369

370 # -- Sanity check, the cached config exists.

371 assert self._cached_remote_config is not None

372

373 # -- Print the soft warning.

374 if error_msg_lines:

375 cerror(*error_msg_lines)

376

377 cout("Will try again at a latter time.", style=INFO)

378

379 # -- Memorize the time of the attempt so we don't retry too often.

380 self._cached_remote_config.metadata.refresh_failure_on = (

381 get_datetime_stamp()

382 )

383 self._save()

384

385 def _maybe_fetch_and_update_remote_config(

386 self, *, error_is_fatal: bool

387 ) -> None:

388 """Try to fetch remote config and if fetched successfully update in

389 self."""

390

391 # pylint: disable=broad-exception-caught

392

393 # -- Fetch the config text. Returns None if error_is_fatal=False and

394 # -- fetch failed.

395 remote_config_text: str | None = self._fetch_remote_config_text(

396 error_is_fatal=error_is_fatal

397 )

398

399 if remote_config_text is None: 399 ↛ 402line 399 didn't jump to line 402 because the condition on line 399 was never true

400 # -- Sanity check, If error_is_fatal, _fetch_remote_config_text()

401 # -- wouldn't return with None.

402 assert not error_is_fatal

403 return

404

405 # -- Print the file's content for debugging

406 if is_debug(1): 406 ↛ 407line 406 didn't jump to line 407 because the condition on line 406 was never true

407 cout(remote_config_text)

408

409 try:

410 json_text = jsonc.to_json(remote_config_text)

411 json_dict = json.loads(json_text)

412 except Exception as e:

413 error_msg = "Failed to parse the fetched Apio remote config json."

414 if error_is_fatal:

415 fatal_error(error_msg, cause=e)

416 self._handle_soft_config_refresh_failure(

417 error_msg_lines=[error_msg, f"{e}"]

418 )

419 return

420

421 # -- Convert the remote config json to proto.

422 remote_config = proto_from_json_dict(json_dict, RemoteConfigSpec)

423 if remote_config is None: 423 ↛ 424line 423 didn't jump to line 424 because the condition on line 423 was never true

424 error_msg = "Remote config data corrupt (proto parsing failed)"

425 if error_is_fatal:

426 fatal_error(error_msg)

427 self._handle_soft_config_refresh_failure(

428 error_msg_lines=[error_msg]

429 )

430 return

431

432 # -- Construct the metadata field.

433 metadata = CachedRemoteConfigMetadata(

434 loaded_by=util.get_apio_version_str(),

435 loaded_at=get_datetime_stamp(),

436 loaded_from=self.remote_config_file_url,

437 )

438

439 # -- Update the cached config member.

440 self._cached_remote_config = CachedRemoteConfigSpec(

441 remote_config=remote_config, metadata=metadata

442 )

443

444 # -- Flush to disk.

445 self._save()

446

447 def _fetch_remote_config_text(self, error_is_fatal: bool) -> str | None:

448 """Fetches and returns the apio remote config JSON text. In case

449 of an error, returns None."""

450

451 # pylint: disable=broad-exception-caught

452

453 # -- Announce the remote config url

454 cout(f"Fetching '{self.remote_config_file_url}'")

455

456 # -- If the URL has a file protocol, read from the file. This

457 # -- is used mostly for testing of a new package version.

458 if self.remote_config_file_url.startswith("file://"): 458 ↛ 479line 458 didn't jump to line 479 because the condition on line 458 was always true

459 file_path = self.remote_config_file_url[7:]

460 try:

461 with open(file_path, encoding="utf-8") as f:

462 file_text = f.read()

463 except Exception as e:

464 # -- Since local config file can't be fixed with a fresh fetch

465 # -- from a remote server, we treat this as a fatal error.

466 fatal_error(

467 "Failed to read a local config file.",

468 cause=e,

469 )

470

471 # -- Local file read OK.

472 return file_text

473

474 # -- Here is the normal case where the config url is not of a local

475 # -- file but at a remote URL.

476

477 # -- Fetch the remote config. With timeout = 10, this failed a

478 # -- few times on github workflow tests so increased to 25.

479 exception: Exception | None = None

480 try:

481 resp: requests.Response = requests.get(

482 self.remote_config_file_url, timeout=25

483 )

484 except Exception as e:

485 exception = e

486

487 error_msg = "Downloading of the latest Apio remote config file failed."

488

489 # -- Handle the case of an exception. This is the preferable option

490 # -- since it provides to fatal_error() a more detailed context of

491 # -- the error (which can be viewed with APIO_DEBUG=1)

492 if exception is not None:

493 if error_is_fatal:

494 fatal_error(error_msg, cause=exception)

495 self._handle_soft_config_refresh_failure(

496 error_msg_lines=[error_msg, str(exception)]

497 )

498 return None

499

500 # -- Handle the case of a status error with no exception.

501 if resp.status_code != 200:

502 error_lines = [

503 error_msg,

504 f"Expected HTTP status code 200, got {resp.status_code}.",

505 ]

506 if error_is_fatal:

507 fatal_error(*error_lines)

508 self._handle_soft_config_refresh_failure(

509 error_msg_lines=error_lines

510 )

511 return None

512

513 # -- Done ok.

514 assert resp.text is not None

515 return resp.text

Coverage for apio/managers/remote_config.py: 69%

198 statements