Coverage for apio/managers/apio_definitions.py: 93%
139 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-07 05:08 +0000
1# This file is part of the Apio project
2# (C) 2016-present FPGAwars
3# Project authors:
4# * Jesús Arroyo (Jesus89) (2016-2019)
5# * Juan Gonzalez (obijuan) (2019-2024)
6# * The Apio Team (2024-present)
7# License: GPLv2
9"""The apio definitions manager class. This class manages the standard
10and custom boards, fpgas, and programmers definitions."""
13import re
14from dataclasses import dataclass
15from pathlib import Path
16import json
17from apio.utils import jsonc, env_options
18from apio.common import proto_util
19from apio.common.apio_console import cout, fatal_error
20from apio.common.proto.stubs.apio_definitions_pb2 import (
21 BoardDefinition,
22 FpgaDefinition,
23 ProgrammerDefinition,
24)
26# -- Boards definitions file name.
27BOARDS_JSONC = "boards.jsonc"
29# -- FPGAs definitions file name.
30FPGAS_JSONC = "fpgas.jsonc"
32# -- Programmers definitions file name.
33PROGRAMMERS_JSONC = "programmers.jsonc"
35# -- A regex for validating boards, fpgas, and programmers ids.
36DEFINITION_ID_FORMAT = re.compile(r"^[a-z][a-z0-9-]*$")
38# -- A regex for validating usb vid and pid values.
39USB_ID_FORMAT = re.compile(r"^[0-9a-f]{4}$")
42class ApioDefinitions:
43 """Contains the apio definitions in the form of json dictionaries."""
45 # pylint: disable=too-many-instance-attributes
47 def __init__(
48 self,
49 apio_packages_dir: Path,
50 project_definitions_dir: Path | None,
51 ):
52 # pylint: disable=too-many-locals
54 assert isinstance(apio_packages_dir, Path)
55 assert project_definitions_dir is None or isinstance(
56 project_definitions_dir, Path
57 )
59 self._apio_packages_dir = apio_packages_dir
60 self._project_definitions_dir = project_definitions_dir
62 # -- The dir with the standard boards, fpgas, and programmers
63 # -- definition files. Can be overridden with APIO_DEFINITIONS.
64 self._package_definitions_dir = self._resolve_package_definitions_dir(
65 apio_packages_dir
66 )
68 # -- Read boards definitions as a json_dict.
69 # -- Custom definitions overrides apio standard definitions.
70 boards_json, self.custom_boards_ids = self._load_definitions(
71 {},
72 BOARDS_JSONC,
73 self._package_definitions_dir,
74 self._project_definitions_dir,
75 )
77 # -- Convert the board definition to BoardDefinition protos and save.
78 self.boards: dict[str, BoardDefinition] = {}
79 for board_id, definition_dict in boards_json.items():
80 board_definition = proto_util.proto_from_json_dict(
81 definition_dict,
82 BoardDefinition,
83 )
84 if board_definition is None:
85 fatal_error(f"Failed to parse board definition '{board_id}")
86 self.boards[board_id] = board_definition
88 # -- Read the base fpga definitions exported by the underlying tools.
89 base_fpgas_dict = self._load_definitions_file(
90 self._apio_packages_dir / "oss-cad-suite" / "FPGAS.jsonc"
91 )
92 openxc7_fpgas_dict = self._load_definitions_file(
93 self._apio_packages_dir / "openxc7" / "FPGAS.jsonc"
94 )
95 for fpga_id, fpga_def in openxc7_fpgas_dict.items():
96 assert (
97 fpga_id not in base_fpgas_dict
98 ), f"Duplicate fpga id: {fpga_id}"
99 base_fpgas_dict[fpga_id] = fpga_def
101 # -- Read fpgas definitions as json dicts.
102 # -- Custom definitions overrides apio standard definitions.
103 fpgas_json, self.custom_fpgas_ids = self._load_definitions(
104 base_fpgas_dict,
105 FPGAS_JSONC,
106 self._package_definitions_dir,
107 self._project_definitions_dir,
108 )
110 # -- Convert the fpgas definition dicts to FpgasDefinition protos and
111 # -- save.
112 self.fpgas: dict[str, FpgaDefinition] = {}
113 for fpga_id, definition_dict in fpgas_json.items():
114 fpga_definition = proto_util.proto_from_json_dict(
115 definition_dict,
116 FpgaDefinition,
117 )
118 if fpga_definition is None:
119 fatal_error(f"Failed to parse fpga definition '{fpga_id}'")
120 self.fpgas[fpga_id] = fpga_definition
122 # -- Load programmers definitions as json dicts.
123 # -- Custom definitions overrides apio standard definitions.
124 programmers_json, self.custom_programmers_ids = self._load_definitions(
125 {},
126 PROGRAMMERS_JSONC,
127 self._package_definitions_dir,
128 self._project_definitions_dir,
129 )
131 # -- Convert the programmers definition dicts to FpgaDefinition protos
132 # -- and save.
133 self.programmers: dict[str, ProgrammerDefinition] = {}
134 for programmer_id, definition_dict in programmers_json.items():
135 programmer_definition = proto_util.proto_from_json_dict(
136 definition_dict,
137 ProgrammerDefinition,
138 )
139 if programmer_definition is None:
140 fatal_error(
141 f"Failed to parse programmer definition '{programmer_id}"
142 )
143 self.programmers[programmer_id] = programmer_definition
145 # -- Validate the definitions we just loaded.
146 self._validate_definitions()
148 @property
149 def package_definitions_dir(self) -> Path:
150 """Returns the dir from which the standard boards, fpgas, and
151 programmers definition files were loaded."""
152 return self._package_definitions_dir
154 @staticmethod
155 def _resolve_package_definitions_dir(apio_packages_dir: Path) -> Path:
156 """Returns the dir of the standard definition files. This is
157 <packages dir>/definitions, unless overridden by the APIO_DEFINITIONS
158 env var."""
159 override = env_options.get(env_options.APIO_DEFINITIONS)
160 if override is None:
161 return apio_packages_dir / "definitions"
163 override_dir = Path(override)
164 if not override_dir.is_dir():
165 fatal_error(
166 f"{env_options.APIO_DEFINITIONS} is not a directory: "
167 f"'{override}'"
168 )
169 return override_dir
171 def _validate_definitions(self):
172 """Validate the boards, fpgas, and programmers definitions of this
173 instance."""
175 # pylint: disable=too-many-branches
177 # -- Validate boards definitions
178 for board_id, board_definition in self.boards.items():
179 # -- Check that board id has a valid format.
180 if not DEFINITION_ID_FORMAT.match(board_id):
181 fatal_error(f"Board id `{board_id}` has an invalid format")
183 # -- Check that the definition proto is fully initialized.
184 proto_util.check_is_initialized(
185 board_definition,
186 f"Failed to initialized board definition '{board_id}'",
187 )
189 # -- Check that the fpga definition exits.
190 proto_util.check_is_required(board_definition, "fpga_id")
191 fpga_id = board_definition.fpga_id
192 if fpga_id not in self.fpgas: 192 ↛ 193line 192 didn't jump to line 193 because the condition on line 192 was never true
193 fatal_error(
194 f"Board `{board_id}` refers to non existing "
195 + f"fpga `{fpga_id}`"
196 )
198 # -- Check that the programmer definition exits.
199 proto_util.check_is_required(board_definition, "programmer.id")
200 programmer_id = board_definition.programmer.id
201 if programmer_id not in self.programmers: 201 ↛ 202line 201 didn't jump to line 202 because the condition on line 201 was never true
202 fatal_error(
203 f"Board `{board_id}` refers to non existing "
204 + f"programmer `{programmer_id}`"
205 )
207 # -- Validate the format of the optional usb.vid and usb.pid
208 # -- fields.
209 proto_util.check_not_required(board_definition, "usb")
210 if board_definition.HasField("usb"):
211 usb = board_definition.usb
212 # -- Check vid
213 proto_util.check_not_required(usb, "vid")
214 if usb.HasField("vid"): 214 ↛ 221line 214 didn't jump to line 221 because the condition on line 214 was always true
215 if not USB_ID_FORMAT.match(usb.vid): 215 ↛ 216line 215 didn't jump to line 216 because the condition on line 215 was never true
216 fatal_error(
217 "The usb.vid field of the board "
218 + f"'{board_id}' has invalid value `{usb.vid}`"
219 )
220 # -- check pid
221 proto_util.check_not_required(usb, "pid")
222 if usb.HasField("pid"): 222 ↛ 178line 222 didn't jump to line 178 because the condition on line 222 was always true
223 if not USB_ID_FORMAT.match(usb.pid): 223 ↛ 224line 223 didn't jump to line 224 because the condition on line 223 was never true
224 fatal_error(
225 "The usb.pid field of the board "
226 + f"'{board_id}' has invalid value `{usb.vip}`"
227 )
229 # -- Validate fpgas definitions.
230 for fpga_id, fpga_definition in self.fpgas.items():
231 # -- Check id format.
232 if not DEFINITION_ID_FORMAT.match(fpga_id):
233 fatal_error(f"FPGA id has an invalid format: {fpga_id}")
235 # -- Check that the definition proto is fully initialized.
236 proto_util.check_is_initialized(
237 fpga_definition,
238 f"Failed to initialize fpga definition '{fpga_id}'",
239 )
241 # -- Validate programmers definitions.
242 for programmer_id, programmer_definition in self.programmers.items():
243 # -- Check id format.
244 if not DEFINITION_ID_FORMAT.match(programmer_id):
245 fatal_error(
246 f"Programmer id has an invalid format: {programmer_id}"
247 )
249 # -- Check that the definition proto is fully initialized.
250 proto_util.check_is_initialized(
251 programmer_definition,
252 f"Failed to initialize programmer definition '{fpga_id}'",
253 )
255 def is_custom_board(self, board_id: str) -> bool:
256 """Returns true if the board's definition was loaded from a
257 project's boards.jsonc file."""
258 assert board_id in self.boards, board_id
259 return board_id in self.custom_boards_ids
261 def is_custom_fpga(self, fpga_id: str) -> bool:
262 """Returns true if the fpga's definition was loaded from a
263 project's fpgas.jsonc file."""
264 assert fpga_id in self.fpgas, fpga_id
265 return fpga_id in self.custom_fpgas_ids
267 def is_custom_programmer(self, programmer_id: str) -> bool:
268 """Returns true if the programmer's definition was loaded from a
269 project's programmers.jsonc file."""
270 assert programmer_id in self.programmers, programmer_id
271 return programmer_id in self.custom_programmers_ids
273 @classmethod
274 def _load_definitions(
275 cls,
276 base_definitions: dict[str, dict],
277 name: str,
278 package_definitions_dir: Path,
279 project_definitions_dir: Path | None,
280 ) -> tuple[dict[str, dict], set[str]]:
281 """Load a jsonc file. Try first from custom_dir, if given, and then
282 from standard dir. This method is called for resource files in
283 apio/resources and definitions files in the definitions packages.
284 Returns a tuple with the merged standard and custom resource
285 definitions (custom wins) and a set of boards ids in the custom
286 resource file.
287 """
288 # -- Level 1 - Start with the base definition.
289 combined_dict: dict[str, dict] = {}
290 combined_dict.update(base_definitions)
292 # -- Level 2 - Load the standard definition from the package.
293 package_file = package_definitions_dir / name
294 package_dict = cls._load_definitions_file(package_file)
295 combined_dict.update(package_dict)
297 # -- Level 3 - If there is a project specific override file, apply it
298 # -- on top of the standard apio definition dict. These are the custom
299 # -- definitions provided by the user.
300 custom_ids: set[str] = set()
301 if project_definitions_dir:
302 project_file = project_definitions_dir / name
303 if project_file.exists():
304 # -- Load the override json dict.
305 cout(f"Loading custom '{name}'.")
306 project_dict = cls._load_definitions_file(project_file)
307 # -- Apply the override. Entries in override replace same
308 # -- key entries in result or if unique are added.
309 combined_dict.update(project_dict)
310 custom_ids.update(project_dict.keys())
312 # -- Sort combined dict by key.
313 combined_dict = dict(sorted(combined_dict.items()))
315 # -- All done.
316 return (combined_dict, custom_ids)
318 @classmethod
319 def _load_definitions_file(cls, filepath: Path) -> dict:
320 """Load the resources from a given jsonc file path
321 * OUTPUT: A dictionary with the jsonc file data
322 In case of error it raises an exception and finish
323 """
325 # pylint: disable=broad-exception-caught
327 # -- Read the jsonc file
328 try:
329 jsonc_text = filepath.read_text(encoding="utf-8")
330 json_text = jsonc.to_json(jsonc_text)
331 json_dict = json.loads(json_text)
332 except Exception as e:
333 fatal_error(
334 f"Failed to read and parse definition file {filepath.name}",
335 cause=e,
336 )
338 # -- Return the object for the resource
339 return json_dict
342@dataclass(frozen=True)
343class ProjectDefinitions:
344 """Contains the definitions of the current project."""
346 board_id: str
347 board_definition: BoardDefinition
348 fpga_id: str
349 fpga_definition: FpgaDefinition
350 programmer_id: str
351 programmer_definition: ProgrammerDefinition
354def collect_project_definitions(
355 board_id: str,
356 definitions: ApioDefinitions,
357) -> ProjectDefinitions:
358 """Collect the board, fpga, and programmer definitions used by a project.
359 Since the definitions may be custom definition defined by the user, we
360 need to have a user friendly error handling and reporting."""
362 # -- Get the board definition.
363 board_definition = definitions.boards.get(board_id, None)
364 if board_definition is None: 364 ↛ 365line 364 didn't jump to line 365 because the condition on line 364 was never true
365 fatal_error(f"Unknown board id '{board_id}'.")
367 # -- We assume below that these fields are required.
368 proto_util.check_is_required(board_definition, "fpga_id", "programmer.id")
370 # -- Get fpga id and definition.
371 fpga_id = board_definition.fpga_id
372 fpga_definition = definitions.fpgas[fpga_id]
374 # -- Get programmer id and info.
375 programmer_id = board_definition.programmer.id
376 programmer_definition = definitions.programmers[programmer_id]
378 # -- Create the project definitions bundle.
379 project_definitions = ProjectDefinitions(
380 board_id,
381 board_definition,
382 fpga_id,
383 fpga_definition,
384 programmer_id,
385 programmer_definition,
386 )
388 # -- All done
389 return project_definitions