Coverage for apio/managers/remote_config.py: 71%
215 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-09-23 03:53 +0000
1# -*- coding: utf-8 -*-
2# -- This file is part of the Apio project
3# -- (C) 2016-2019 FPGAwars
4# -- Author Jesús Arroyo
5# -- License GPLv2
6"""Manage the apio remote configuration"""
8# pylint: disable=duplicate-code
10import json
11from enum import Enum, unique
12from dataclasses import dataclass
13from datetime import datetime
14from typing import Any
15from pathlib import Path
16import requests
17from jsonschema import validate
18from jsonschema.exceptions import ValidationError
19import json5
20from apio.common.debug_util import is_debug
21from apio.common.apio_console import cout, fatal_error
22from apio.common.apio_styles import INFO, EMPH3
23from apio.utils import util
25# -- JSON schema for validating the downloaded remote config files.
26REMOTE_CONFIG_SCHEMA = {
27 "$schema": "https://json-schema.org/draft/2020-12/schema",
28 "type": "object",
29 "required": ["packages"],
30 "properties": {
31 # -- Packages
32 "packages": {
33 "type": "object",
34 "patternProperties": {
35 "^.*$": {
36 "type": "object",
37 "required": ["repository", "release"],
38 "properties": {
39 # -- Repository
40 "repository": {
41 "type": "object",
42 "required": ["name", "organization"],
43 "properties": {
44 # -- Repo name, e.g. 'examples'
45 "name": {"type": "string"},
46 # -- Repo organization. e.g. "fpgawars"
47 "organization": {"type": "string"},
48 },
49 "additionalProperties": False,
50 },
51 # -- Release.
52 "release": {
53 "type": "object",
54 "required": [
55 "tag",
56 "package",
57 ],
58 "properties": {
59 # -- Tag
60 "tag": {
61 "type": "string",
62 "pattern": r"^\d{4}\-\d{2}\-\d{2}$",
63 },
64 # -- Package
65 "package": {"type": "string"},
66 },
67 "additionalProperties": False,
68 },
69 },
70 "additionalProperties": False,
71 }
72 },
73 "additionalProperties": False,
74 }
75 },
76 "additionalProperties": False,
77}
80@unique
81class RemoteConfigPolicy(Enum):
82 """Represents possible requirements from the remote config."""
84 # -- Config is being used but can be a cached value, as long that it's
85 # -- not too old.
86 CACHED_OK = 1
87 # -- Config is being used and a fresh copy is that was fetch in this
88 # -- invocation of Apio is required.
89 GET_FRESH = 2
92@dataclass(frozen=True)
93class PackageRemoteConfig:
94 """Contains a package info from the remote config."""
96 # -- E.g. "tools-oss-cad-suite"
97 repo_name: str
98 # -- E.g. "FPGAwars"
99 repo_organization: str
100 # -- E.g. "0.2.3"
101 release_version: str
102 # -- E.g. "${YYYY-MM-DD}""
103 release_tag: str
104 # -- E.g. "apio-oss-cad-suite-${PLATFORM}-${YYYYMMDD}.zip"
105 release_file: str
108def get_datetime_stamp(dt: datetime | None = None) -> str:
109 """Returns a string with time now as yyyy-mm-dd-hh-mm"""
110 if dt is None:
111 dt = datetime.now()
112 return dt.strftime("%Y-%m-%d-%H-%M")
115def days_between_datetime_stamps(ts1: str, ts2: str, default: Any) -> int:
116 """Given two values generated by get_datetime_stamp(), return the
117 number of days from ts1 to ts2. The value can be negative if ts2 is
118 earlier than ts1. Returns the given 'default' value if either timestamp
119 is invalid."""
121 # -- The parsing format.
122 fmt = "%Y-%m-%d-%H-%M"
124 # -- Convert to timedates
125 try:
126 datetime1 = datetime.strptime(ts1, fmt)
127 datetime2 = datetime.strptime(ts2, fmt)
128 except ValueError:
129 return default
131 # -- Round to beginning of day.
132 day1 = datetime(datetime1.year, datetime1.month, datetime1.day)
133 day2 = datetime(datetime2.year, datetime2.month, datetime2.day)
135 # -- Compute the diff in days.
136 delta_days: int = (day2 - day1).days
138 # -- All done.
139 assert isinstance(delta_days, int)
140 return delta_days
143def minutes_between_datetime_stamps(ts1: str, ts2: str, default: Any) -> int:
144 """Return the number of minutes from ts1 to ts2 or default if any of
145 the timestamps is invalid."""
147 # -- The parsing format.
148 fmt = "%Y-%m-%d-%H-%M"
150 # -- Convert to timedates
151 try:
152 datetime1 = datetime.strptime(ts1, fmt)
153 datetime2 = datetime.strptime(ts2, fmt)
154 except ValueError:
155 return default
157 # -- Calculate the diff in minutes.
158 delta_minutes = int((datetime2 - datetime1).total_seconds() / 60)
159 assert isinstance(delta_minutes, int), type(delta_minutes)
160 return delta_minutes
163class RemoteConfig:
164 """Class for managing the apio remote config and its local cache
165 ~/.apio/cached-remote-config.json
166 """
168 # -- Only these instance vars are allowed.
169 __slots__ = (
170 # "_profile_path",
171 # "_packages_index_path",
172 "remote_config_url",
173 "remote_config_ttl_days",
174 "remote_config_retry_minutes",
175 "_remote_config_policy",
176 "_cached_remote_config_path",
177 "_cached_remote_config",
178 # "preferences",
179 # "installed_packages",
180 )
182 def __init__(
183 self,
184 home_dir: Path,
185 # packages_dir: Path,
186 remote_config_url_template: str,
187 remote_config_ttl_days: int,
188 remote_config_retry_minutes: int,
189 remote_config_policy: RemoteConfigPolicy,
190 ):
191 """remote_config_url_template is a url string with the
192 placeholder {major} and {minor} for the apio's major and minor
193 version. '"""
195 # pylint: disable=too-many-arguments
196 # pylint: disable=too-many-positional-arguments
198 # -- Sanity check
199 assert isinstance(remote_config_ttl_days, int)
200 assert 0 < remote_config_ttl_days <= 30
202 # -- Sanity check
203 assert isinstance(remote_config_retry_minutes, int)
204 assert 0 < remote_config_retry_minutes <= (60 * 24)
206 # -- Resolve and cache the remote config url. Replaced the placeholders
207 # -- with the major and minor versions of apio. Path version is
208 # -- not used.
209 ver_tuple = util.get_apio_version_tuple()
210 url = remote_config_url_template
211 url = url.replace("{major}", str(ver_tuple[0]))
212 url = url.replace("{minor}", str(ver_tuple[1]))
213 self.remote_config_url = url
215 # -- Save remote url ttl setting.
216 self.remote_config_ttl_days = remote_config_ttl_days
218 # -- Save the remote config fetch retry minutes.
219 self.remote_config_retry_minutes = remote_config_retry_minutes
221 # -- Save remote config policy.
222 self._remote_config_policy = remote_config_policy
224 # -- Verify that we resolved all the remote config URL placeholders.
225 assert "{" not in self.remote_config_url, self.remote_config_url
227 if is_debug(1): 227 ↛ 228line 227 didn't jump to line 228 because the condition on line 227 was never true
228 cout(f"Remote config url: {self.remote_config_url}")
230 # -- Start with no remote config.
231 self._cached_remote_config: dict[str, Any] | None = None
233 # -- Path to the local file with the cached remote config.
234 self._cached_remote_config_path = (
235 home_dir / "cached-remote-config.json"
236 )
238 # -- Try to load the cached remote config. If found and read
239 # -- successfully it mutates self._cached_remote_config
240 self._maybe_load_cached_remote_config()
242 # -- Apply config policy
243 self._apply_remote_config_policy()
245 @staticmethod
246 def _skipping_cache_msg(reason: str):
247 """Show a message indicating that the cached remote config is being
248 skipped."""
249 cout(f"No suitable cached remote config file ({reason}).", style=INFO)
251 def _apply_remote_config_policy(self) -> None:
252 """Called after loading the profile file, to apply the remote config
253 policy for this invocation."""
255 # -- Case 1: A fresh config is required for the current command.
256 if self._remote_config_policy == RemoteConfigPolicy.GET_FRESH:
257 self._fetch_and_update_remote_config(error_is_fatal=True)
258 return
260 # -- Case 2: A fresh config is optional but there is no cached
261 # -- config so practically it's required.
262 assert self._remote_config_policy == RemoteConfigPolicy.CACHED_OK
263 if not self._cached_remote_config:
264 if is_debug(1): 264 ↛ 265line 264 didn't jump to line 265 because the condition on line 264 was never true
265 cout("Cached remote config is not available.", style=INFO)
266 self._fetch_and_update_remote_config(error_is_fatal=True)
267 return
269 # -- Case 3: May need to fetch a new config but can continue with
270 # -- the cached config in case of a fetch failure.
271 #
272 # -- Get the cached config metadata.
273 cashed_config_metadata = self._cached_remote_config.get("metadata", {})
274 last_fetch_timestamp = cashed_config_metadata.get("loaded-at", "")
275 last_fetch_url = cashed_config_metadata.get("loaded-from", "")
277 # -- Determine if we need a new config because the remote config URL
278 # -- was changed (e.g. with APIO_REMOTE_CONFIG_URL)
279 url_changed = last_fetch_url != self.remote_config_url
281 # -- Determine if there is time related reason to fetch a new config.
282 datetime_stamp_now = get_datetime_stamp()
283 days_since_last_fetch = days_between_datetime_stamps(
284 last_fetch_timestamp, datetime_stamp_now, default=99999
285 )
286 time_valid = 0 <= days_since_last_fetch < self.remote_config_ttl_days
288 # -- Determine if we already tried recently to refresh this config and
289 # -- failed.
290 refresh_failure_timestamp = cashed_config_metadata.get(
291 "refresh-failure-on", ""
292 )
293 minutes_since_refresh_failure = minutes_between_datetime_stamps(
294 refresh_failure_timestamp, datetime_stamp_now, default=99999
295 )
296 refresh_failed_recently = (
297 0
298 <= minutes_since_refresh_failure
299 < self.remote_config_retry_minutes
300 )
302 # -- Dump info for debugging.
303 if is_debug(1): 303 ↛ 304line 303 didn't jump to line 304 because the condition on line 303 was never true
304 cout(
305 f"{days_since_last_fetch=}, {time_valid=}, {url_changed=}",
306 f"{minutes_since_refresh_failure=}, "
307 f"{refresh_failed_recently=}",
308 style=EMPH3,
309 )
311 # -- Fetch the new config if needed.
312 if url_changed or not time_valid:
313 reason = "source URL mismatch" if url_changed else "stale"
314 self._skipping_cache_msg(reason=reason)
315 if refresh_failed_recently: 315 ↛ 316line 315 didn't jump to line 316 because the condition on line 315 was never true
316 cout("Remote config fetch failed recently, skipping.")
317 else:
318 self._fetch_and_update_remote_config(error_is_fatal=False)
320 @property
321 def data(self) -> dict:
322 """Returns the remote config that is applicable for this invocation.
323 Should not called if the context was initialized with NO_CONFIG."""
324 assert self._cached_remote_config is not None
325 return self._cached_remote_config.get("remote-config", {})
327 @property
328 def metadata(self) -> dict:
329 """Returns the remote config metadata. Should not be called
330 if the context was initialized with NO_CONFIG."""
331 assert self._cached_remote_config is not None
332 return self._cached_remote_config.get("metadata", {})
334 def get_package_config(
335 self,
336 package_name: str,
337 ) -> PackageRemoteConfig:
338 """Given a package name, return the remote config information with the
339 version and fetch information.
340 """
342 # -- Extract package's remote config.
343 package_config = self.data["packages"][package_name]
344 repo_name = package_config["repository"]["name"]
345 repo_organization = package_config["repository"]["organization"]
346 release_tag = package_config["release"]["tag"]
347 release_version = release_tag.replace("-", ".")
348 release_file = package_config["release"]["package"]
350 return PackageRemoteConfig(
351 repo_name=repo_name,
352 repo_organization=repo_organization,
353 release_version=release_version,
354 release_tag=release_tag,
355 release_file=release_file,
356 )
358 def _maybe_load_cached_remote_config(self):
359 """Try loading self._cached_remote_config from the local file
360 cached_remote_config.json. If the file not found, or from a different
361 version of apio do not do not change self._cached_remote_config.
362 This method doesn't check for file age or staleness.
363 """
365 # -- If the file doesn't exist then do nothing.
366 if not self._cached_remote_config_path.exists():
367 self._skipping_cache_msg("no cache file")
368 return
370 # -- Read the cached remote config file as a json dict and extract its
371 # -- fields. Handle invalid content gracefully, e.g. a corrupt or hand
372 # -- edited file, since this runs on every apio command.
373 try:
374 with open(
375 self._cached_remote_config_path, "r", encoding="utf8"
376 ) as f:
377 json_data = json.load(f)
379 # -- Determine if the cached remote config is usable.
380 config_apio_version = json_data.get("metadata", {}).get(
381 "loaded-by", ""
382 )
383 apio_version_matches = (
384 config_apio_version == util.get_apio_version_str()
385 )
387 # -- Not downloaded by this version of apio. Ignore.
388 if not apio_version_matches:
389 self._skipping_cache_msg("Apio version mismatch")
390 return
392 # -- Validate the remote config against the schema and keep
393 # -- it.
394 validate(
395 instance=json_data["remote-config"],
396 schema=REMOTE_CONFIG_SCHEMA,
397 )
398 self._cached_remote_config = json_data
400 except (OSError, ValueError, AttributeError) as e:
401 self._skipping_cache_msg("couldn't parse")
402 cout(str(e), style=INFO)
404 def _save(self):
405 """Save the cached remote config to a file"""
407 # -- Create the enclosing folder, if it does not exist yet
408 file_path = self._cached_remote_config_path
409 dir_path = file_path.parent
410 if not dir_path.exists(): 410 ↛ 411line 410 didn't jump to line 411 because the condition on line 410 was never true
411 dir_path.mkdir()
413 # -- Write to file.
414 with open(self._cached_remote_config_path, "w", encoding="utf8") as f:
415 json.dump(self._cached_remote_config, f, indent=2)
417 # -- Dump for debugging.
418 if is_debug(1): 418 ↛ 419line 418 didn't jump to line 419 because the condition on line 418 was never true
419 cout("Saved cached remote config:", style=EMPH3)
420 cout(json.dumps(self._cached_remote_config, indent=2))
422 def _handle_soft_config_refresh_failure(
423 self, *, error_msg_lines: list[str]
424 ):
425 """Called to handle a soft failure of a remote config refresh.
426 That is, an error, from which we recover by using the cached
427 remote config."""
429 # -- Sanity check, the cached config exists.
430 assert self._cached_remote_config, "No cached remote config"
432 # -- Print the soft warning.
433 cout(*error_msg_lines, style=INFO)
434 cout("Will try again at a latter time.", style=INFO)
436 # -- Memorize the time of the attempt so we don't retry too often.
437 metadata = self._cached_remote_config["metadata"]
438 metadata["refresh-failure-on"] = get_datetime_stamp()
439 self._save()
441 def _fetch_and_update_remote_config(self, *, error_is_fatal: bool) -> None:
442 """Returns the apio remote config JSON dict."""
444 # pylint: disable=broad-exception-caught
446 # -- Fetch the config text. Returns None if error_is_fatal=False and
447 # -- fetch failed.
448 config_text: str | None = self._fetch_remote_config_text(
449 error_is_fatal=error_is_fatal
450 )
452 if config_text is None: 452 ↛ 455line 452 didn't jump to line 455 because the condition on line 452 was never true
453 # -- Sanity check, If error_is_fatal, _fetch_remote_config_text()
454 # -- wouldn't return with None.
455 assert not error_is_fatal
456 return
458 # -- Print the file's content for debugging
459 if is_debug(1): 459 ↛ 460line 459 didn't jump to line 460 because the condition on line 459 was never true
460 cout(config_text)
462 try:
463 remote_config = json5.loads(config_text)
464 except Exception as e:
465 error_msg = "Failed to parse the latest Apio remote config file."
466 if error_is_fatal:
467 fatal_error(error_msg, cause=e)
468 self._handle_soft_config_refresh_failure(
469 error_msg_lines=[error_msg, f"{e}"]
470 )
471 return
473 # -- Do some checks and fail if invalid. This is not an exhaustive
474 # -- check.
475 ok = self._check_downloaded_remote_config(
476 remote_config, error_is_fatal=error_is_fatal
477 )
478 if not ok: 478 ↛ 479line 478 didn't jump to line 479 because the condition on line 478 was never true
479 return
481 # -- Create the cached remote config wrapper
482 cached_remote_config = {}
483 cached_remote_config["remote-config"] = remote_config
485 # -- Append remote config metadata. This also clear the
486 # -- "refresh-failure-on" field if exists.
487 metadata_dict: dict[str, Any] = {}
488 metadata_dict["loaded-by"] = util.get_apio_version_str()
489 metadata_dict["loaded-at"] = get_datetime_stamp()
490 metadata_dict["loaded-from"] = self.remote_config_url
491 cached_remote_config["metadata"] = metadata_dict
493 self._cached_remote_config = cached_remote_config
494 self._save()
496 def _check_downloaded_remote_config(
497 self, remote_config: dict, error_is_fatal: bool
498 ) -> bool:
499 """Check the downloaded remote config has a valid structure."""
500 try:
501 validate(instance=remote_config, schema=REMOTE_CONFIG_SCHEMA)
502 except ValidationError as e:
503 # -- Error.
504 error_msg = "Fetched remote config failed validation."
505 if error_is_fatal:
506 fatal_error(error_msg, cause=e)
507 self._handle_soft_config_refresh_failure(
508 error_msg_lines=[error_msg, str(e)]
509 )
510 return False
512 # -- Ok.
513 return True
515 def _fetch_remote_config_text(self, error_is_fatal: bool) -> str | None:
516 """Fetches and returns the apio remote config JSON text. In case
517 of an error, returns None."""
519 # pylint: disable=broad-exception-caught
521 # -- Announce the remote config url
522 cout(f"Fetching '{self.remote_config_url}'")
524 # -- If the URL has a file protocol, read from the file. This
525 # -- is used mostly for testing of a new package version.
526 if self.remote_config_url.startswith("file://"): 526 ↛ 547line 526 didn't jump to line 547 because the condition on line 526 was always true
527 file_path = self.remote_config_url[7:]
528 try:
529 with open(file_path, encoding="utf-8") as f:
530 file_text = f.read()
531 except Exception as e:
532 # -- Since local config file can't be fixed with a fresh fetch
533 # -- from a remote server, we treat this as a fatal error.
534 fatal_error(
535 "Failed to read a local config file.",
536 cause=e,
537 )
539 # -- Local file read OK.
540 return file_text
542 # -- Here is the normal case where the config url is not of a local
543 # -- file but at a remote URL.
545 # -- Fetch the remote config. With timeout = 10, this failed a
546 # -- few times on github workflow tests so increased to 25.
547 exception: Exception | None = None
548 try:
549 resp: requests.Response = requests.get(
550 self.remote_config_url, timeout=25
551 )
552 except Exception as e:
553 exception = e
555 context_msg = (
556 "Downloading of the latest Apio remote config file failed."
557 )
559 # -- Handle the case of an exception. This is the preferable option
560 # -- since it provides to fatal_error() a more detailed context of
561 # -- the error (which can be viewed with APIO_DEBUG=1)
562 if exception is not None:
563 if error_is_fatal:
564 fatal_error(context_msg, cause=exception)
565 self._handle_soft_config_refresh_failure(
566 error_msg_lines=[context_msg, str(exception)]
567 )
569 # -- Handle the case of a status error with no exception.
570 elif resp.status_code != 200:
571 error_lines = [
572 context_msg,
573 f"Expected HTTP status code 200, got {resp.status_code}.",
574 ]
575 if error_is_fatal:
576 fatal_error(*error_lines)
577 self._handle_soft_config_refresh_failure(
578 error_msg_lines=error_lines
579 )
581 return None
583 # -- Done ok.
584 assert resp.text is not None
585 return resp.text