Coverage for apio/managers/project.py: 87%

1# -*- coding: utf-8 -*-

2# -- This file is part of the Apio project

4# -- Author Jesús Arroyo

5# -- License GPLv2

6# -- Derived from:

7# ---- Platformio project

9# ---- License Apache v2

10"""Utility functionality for apio click commands."""

12import sys

13import re

14import configparser

15from collections import OrderedDict

16from pathlib import Path

17from types import NoneType

18from typing import Dict, Optional, Union, Any, List

19from configobj import ConfigObj

20from apio.utils import util

21from apio.common.apio_console import cout, cerror, cwarning

22from apio.common.apio_styles import INFO, SUCCESS, EMPH2

25DEFAULT_TOP_MODULE = "main"

27ENV_NAME_REGEX = re.compile(r"^[a-z][a-z0-9-]*$")

29ENV_NAME_HINT = (

30 "Env names should start with a-z, "

31 "followed by any number of a-z, 0-9, and '-'."

32)

34TOP_COMMENT = """\

35APIO project configuration file.

36For details see https://fpgawars.github.io/apio/docs/project-file

37"""

39# -- Apio options. These are the options that appear in the [apio] section.

40# -- They are not subject to inheritance and resolution.

43APIO_OPTIONS = [

44 # -- Selecting the env to use if not overridden in command line. Otherwise

45 # -- the first env is the default.

46 "default-env",

47]

50# -- All env options.

51ENV_OPTIONS = {

52 "board",

53 "default-testbench",

54 "defines",

55 "format-verible-options",

56 "programmer-cmd",

57 "top-module",

58 "yosys-synth-extra-options",

59 "nextpnr-extra-options",

60 "constraint-file",

61}

63# -- The subset ENV_OPTIONS that is required.

64ENV_REQUIRED_OPTIONS = {

65 "board",

66}

68# -- Options that are parsed as a multi line list (vs a simple str)

69LIST_OPTIONS = {

70 "defines",

71 "format-verible-options",

72 "yosys-synth-extra-options",

73 "nextpnr-extra-options",

74}

77class Project:

78 """An instance of this class holds the information from the project's

79 apio.ini file.

80 """

82 def __init__(

83 self,

84 *,

85 apio_section: Optional[Dict[str, str]],

86 common_section: Optional[Dict[str, str]],

87 env_sections: Dict[str, Dict[str, str]],

88 env_arg: Optional[str],

89 boards: Dict[str, Dict],

90 ):

91 """Construct the project with information from apio.ini, command

92 line arg, and boards resources."""

94 # pylint: disable=too-many-arguments

96 if util.is_debug(1): 96 ↛ 97line 96 didn't jump to line 97 because the condition on line 96 was never true

97 cout()

98 cout("Parsed [apio] section:", style=EMPH2)

99 cout(f" {apio_section}\n")

100 cout("Parsed [common] section:", style=EMPH2)

101 cout(f" {common_section}\n")

102 for env_name, section_options in env_sections.items():

103 cout(f"Parsed [env:{env_name}] section:", style=EMPH2)

104 cout(f"{section_options}\n")

105

106 # -- Validate the format of the env_arg value.

107 if env_arg is not None:

108 if not ENV_NAME_REGEX.match(env_arg):

109 cerror(f"Invalid --env value '{env_arg}'.")

110 cout(ENV_NAME_HINT, style=INFO)

111 sys.exit(1)

112

113 # -- Patch legacy board ids in the common and env sections.

114 Project._patch_legacy_board_id(common_section, boards)

115 for section_options in env_sections.values():

116 Project._patch_legacy_board_id(section_options, boards)

117

118 # -- Validate the apio.ini sections. We prefer to perform as much

119 # -- validation as possible before we expand the env because the env

120 # -- expansion may hide some options.

121 Project._validate_all_sections(

122 apio_section=apio_section,

123 common_section=common_section,

124 env_sections=env_sections,

125 boards=boards,

126 )

127

128 # -- Keep the names of all envs

129 self.env_names = list(env_sections.keys())

130

131 # -- Determine the name of the active env.

132 self.env_name = Project._determine_default_env_name(

133 apio_section, env_sections, env_arg

134 )

135

136 # -- Expand and selected env options. This is also patches default

137 # -- values and validates the results.

138 self.env_options = Project._expand_env_options(

139 env_name=self.env_name,

140 common_section=common_section,

141 env_sections=env_sections,

142 )

143 if util.is_debug(1): 143 ↛ 144line 143 didn't jump to line 144 because the condition on line 143 was never true

144 cout("Selected env name:", style=EMPH2)

145 cout(f" {self.env_name}\n")

146 cout("Expanded env options:", style=EMPH2)

147 cout(f" {self.env_options}\n")

148

149 @staticmethod

150 def _patch_legacy_board_id(

151 section_options: Dict[str, Dict[str, str]], boards: Dict[str, Dict]

152 ) -> Optional[str]:

153 """Temporary patching of old board ids to new in an env or common

154 section. If there is a "board" option with a legacy board id,

155 then change it to the board's canonical name. Otherwise, leave the

156 options as are."""

157

158 # -- Get the value of the "board" option.

159 board_id = section_options.get("board", None)

160

161 # -- Nothing to do if no "board" option.

162 if board_id is None:

163 return

164

165 # -- Nothing to do if board_id is in the boards dict. It's

166 # -- a good (new style) board id.

167 if board_id in boards:

168 return

169

170 # -- Iterate the boards and if board_id matches the legacy name of

171 # -- a board, change the "board" option to the canonical name of that

172 # -- board.

173 for canonical_id, board_info in boards.items():

174 if board_id == board_info.get("legacy-name", None):

175 section_options["board"] = canonical_id

176 cwarning(

177 f"'Board {board_id}' was renamed to '{canonical_id}'. "

178 "Please update apio.ini."

179 )

180 return

181

182 @staticmethod

183 def _validate_all_sections(

184 apio_section: Optional[Dict[str, str]],

185 common_section: Optional[Dict[str, str]],

186 env_sections: Dict[str, Dict[str, str]],

187 boards: Dict[str, Dict],

188 ):

189 """Validate the parsed apio.ini sections."""

190

191 # -- Validate the common section.

192 Project._validate_env_section("[common]", common_section, boards)

193

194 # -- Validate the env sections.

195 if not env_sections:

196 cerror(

197 "Project file 'apio.ini' should have at least one "

198 "[env:name] section."

199 )

200 sys.exit(1)

201

202 for env_name, section_options in env_sections.items():

203 # -- Validate env name format.

204 if not ENV_NAME_REGEX.match(env_name):

205 cerror(f"Invalid env name '{env_name}' in apio.ini.")

206 cout(ENV_NAME_HINT, style=INFO)

207 sys.exit(1)

208 # -- Validate env section options.

209 Project._validate_env_section(

210 f"[env:{env_name}]", section_options, boards

211 )

212

213 # -- Validate the apio section. At this point the env_sections are

214 # -- already validated.

215 Project._validate_apio_section(apio_section, env_sections)

216

217 @staticmethod

218 def _validate_apio_section(

219 apio_section: Dict[str, str], env_sections: Dict[str, Dict[str, str]]

220 ):

221 """Validate the [apio] section. 'env_sections' are assumed to be

222 validated."""

223

224 # -- Look for unknown options.

225 for option in apio_section:

226 if option not in APIO_OPTIONS: 226 ↛ 227line 226 didn't jump to line 227 because the condition on line 226 was never true

227 cerror(

228 f"Unknown option '{option} in [apio] section of apio.ini'"

229 )

230 sys.exit(1)

231

232 # -- If 'default-env' option exists, verify the env name is valid and

233 # -- and the name exists.

234 default_env_name = apio_section.get("default-env", None)

235 if default_env_name:

236 # -- Validate env name format.

237 if not ENV_NAME_REGEX.match(default_env_name): 237 ↛ 238line 237 didn't jump to line 238 because the condition on line 237 was never true

238 cerror(

239 f"Invalid default env name '{default_env_name}' "

240 "in apio.ini."

241 )

242 cout(ENV_NAME_HINT, style=INFO)

243 sys.exit(1)

244 # -- Make sure the env exists.

245 if default_env_name not in env_sections:

246 cerror(f"Env '{default_env_name}' not found in apio.ini.")

247 cout(

248 f"Expecting an env section '{default_env_name}' "

249 "in apio.ini",

250 style=INFO,

251 )

252 sys.exit(1)

253

254 @staticmethod

255 def _validate_env_section(

256 section_title: str,

257 section_options: Dict[str, str],

258 boards: Dict[str, Dict],

259 ):

260 """Validate the options of a section that contains env options. This

261 includes the sections [env:*] and [common]."""

262

263 # -- Check that there are no unknown options.

264 for option in section_options:

265 if option not in ENV_OPTIONS:

266 cerror(

267 f"Unknown option '{option}' in {section_title} "

268 "section of apio.ini."

269 )

270 sys.exit(1)

271

272 # -- If 'board' option exists, verify that the board exists.

273 board_id = section_options.get("board", None)

274 if board_id is not None and board_id not in boards:

275 cerror(f"Unknown board id '{board_id}' in apio.ini.")

276 sys.exit(1)

277

278 @staticmethod

279 def _determine_default_env_name(

280 apio_section: Dict[str, str],

281 env_sections: Dict[str, Dict[str, str]],

282 env_arg: Optional[str],

283 ) -> str:

284 """Determines the active env name. Sections are assumed to be

285 validated. 'env_arg' is the value of the optional command line --env

286 which allows the user to select the env."""

287 # -- Priority #1 (highest): User specified env name in the command.

288 env_name = env_arg

289

290 # -- Priority #2: The optional default-env option in the

291 # -- [apio] section.

292 if env_name is None:

293 env_name = apio_section.get("default-env", None)

294

295 # -- Priority #3 (lowest): Picking the first env defined in apio.ini.

296 # -- Note that the envs order is preserved in env_sections.

297 if env_name is None:

298 # -- The env sections preserve the order in apio.ini.

299 env_name = list(env_sections.keys())[0]

300

301 # -- Error if the env doesn't exist.

302 if env_name not in env_sections:

303 cerror(f"Env '{env_name}' not found in apio.ini.")

304 cout(

305 f"Expecting an env section '[env:{env_name}] in apio.ini",

306 style=INFO,

307 )

308 sys.exit(1)

309

310 # -- All done.

311 return env_name

312

313 @staticmethod

314 def _expand_env_options(

315 env_name: str,

316 common_section: Dict[str, str],

317 env_sections: Dict[str, Dict[str, Union[str, List[str]]]],

318 ) -> Dict[str, str]:

319 """Expand the options of given env name. The given common and envs

320 sections are already validate. String options are returned as strings

321 and list options are returned as list of strings.

322 """

323

324 # -- Select the env section by name.

325 env_section = env_sections[env_name]

326

327 # -- Create an empty result dict.

328 # -- We will insert to it the relevant options by the oder they appear

329 # -- in apio.ini.

330 result: Dict[str, str] = {}

331

332 # -- Add common options that are not in env section

333 for name, val in common_section.items():

334 if name not in env_section: 334 ↛ 333line 334 didn't jump to line 333 because the condition on line 334 was always true

335 result[name] = val

336

337 # -- Add all the options from the env section.

338 for name, val in env_section.items():

339 result[name] = val

340

341 # -- check that all the required options exist.

342 for option in ENV_REQUIRED_OPTIONS:

343 if option not in result:

344 cerror(

345 f"Missing required option '{option}' "

346 f"for env '{env_name}'."

347 )

348 sys.exit(1)

349

350 # -- If top-module was not specified, fill in the default value.

351 if "top-module" not in result:

352 result["top-module"] = DEFAULT_TOP_MODULE

353 cout(

354 f"Option 'top-module' is missing for env {env_name}, "

355 f"assuming '{DEFAULT_TOP_MODULE}'.",

356 style=INFO,

357 )

358

359 # -- Convert the list options from strings to list.

360 for key, str_val in result.items():

361 if key in LIST_OPTIONS:

362 list_val = str_val.split("\n")

363 # -- Select the non empty items.

364 list_val = [x for x in list_val if x]

365 result[key] = list_val

366

367 return result

368

369 def get_str_option(

370 self, option: str, default: Any = None

371 ) -> Union[str, Any]:

372 """Lookup an env option value by name. Returns default if not found."""

373

374 # -- If this fails, this is a programming error.

375 assert option in ENV_OPTIONS, f"Invalid env option: [{option}]"

376 assert option not in LIST_OPTIONS, f"Not a str option: {option}"

377

378 # -- Lookup with default

379 value = self.env_options.get(option, None)

380

381 if value is None:

382 return default

383

384 assert isinstance(value, (str, NoneType))

385 return value

386

387 def get_list_option(

388 self, option: str, default: Any = None

389 ) -> Union[List[str], Any]:

390 """Lookup an env option value that has a line list format. Returns

391 the list of non empty lines or default if no value. Option

392 must be in OPTIONS."""

393

394 # -- If this fails, this is a programming error.

395 assert option in ENV_OPTIONS, f"Invalid env option: [{option}]"

396 assert option in LIST_OPTIONS, f"Not a list option: {option}"

397

398 # -- Get the option values, it's is expected to be a list of str.

399 values_list = self.env_options.get(option, None)

400

401 # -- If not found, return default

402 if values_list is None:

403 return default

404

405 # -- Return the list

406 assert isinstance(values_list, list), values_list

407 return values_list

408

409

410def load_project_from_file(

411 project_dir: Path, env_arg: Optional[str], boards: Dict[str, Dict]

412) -> Project:

413 """Read project file from given project dir. Returns None if file

414 does not exists. Exits on any error. Otherwise creates adn

415 return an Project with the values. To validate the project object

416 call its validate() method."""

417

418 # -- Construct the apio.ini path.

419 file_path = project_dir / "apio.ini"

420

421 # -- Currently, apio.ini is still optional so we just warn.

422 if not file_path.exists():

423 cerror(

424 "Missing project file apio.ini.",

425 f"Expected a file at '{file_path.absolute()}'",

426 )

427 sys.exit(1)

428

429 # -- Read and parse the file.

430 # -- By using OrderedDict we cause the parser to preserve the order of

431 # -- options in a section. The order of sections is already preserved by

432 # -- default.

433 parser = configparser.ConfigParser(dict_type=OrderedDict)

434 try:

435 parser.read(file_path)

436 except configparser.Error as e:

437 cerror(e)

438 sys.exit(1)

439

440 # -- Iterate and collect the sections in the order they appear in

441 # -- the apio.ini file. Section names are guaranteed to be unique with

442 # -- no duplicates.

443 sections_names = parser.sections()

444

445 apio_section = None

446 common_section = None

447 env_sections = {}

448

449 for section_name in sections_names:

450 # -- Handle the [apio[ section.]]

451 if section_name == "apio":

452 if common_section or env_sections: 452 ↛ 453line 452 didn't jump to line 453 because the condition on line 452 was never true

453 cerror("The [apio] section must be the first section.")

454 sys.exit(1)

455 apio_section = dict(parser.items(section_name))

456 continue

457

458 # -- Handle the [common] section.

459 if section_name == "common":

460 if env_sections: 460 ↛ 461line 460 didn't jump to line 461 because the condition on line 460 was never true

461 cerror("The [common] section must be before [env:] sections.")

462 sys.exit(1)

463 common_section = dict(parser.items(section_name))

464 continue

465

466 # TODO: Remove this option after this is released.

467 # -- Handle the legacy [env] section.

468 if section_name == "env" and len(sections_names) == 1:

469 # env_sections["default"] = parser.items(section_name)

470 cwarning(

471 "Apio.ini has a legacy [env] section. "

472 "Please rename it to [env:default]."

473 )

474 env_sections["default"] = dict(parser.items(section_name))

475 continue

476

477 # -- Handle the [env:env-name] sections.

478 tokes = section_name.split(":")

479 if len(tokes) == 2 and tokes[0] == "env": 479 ↛ 485line 479 didn't jump to line 485 because the condition on line 479 was always true

480 env_name = tokes[1]

481 env_sections[env_name] = dict(parser.items(section_name))

482 continue

483

484 # -- Handle unknown section name.

485 cerror(f"Invalid section name '{section_name}' in apio.ini.")

486 cout(

487 "The valid section names are [apio], [common], and [env:env-name]",

488 style=INFO,

489 )

490 sys.exit(1)

491

492 # -- Construct the Project object. Its constructor validates the options.

493 return Project(

494 apio_section=apio_section or {},

495 common_section=common_section or {},

496 env_sections=env_sections,

497 env_arg=env_arg,

498 boards=boards,

499 )

500

501

502def create_project_file(

503 project_dir: Path,

504 board_id: str,

505 top_module: str,

506):

507 """Creates a new basic apio project file. Exits on any error."""

508

509 # -- Construct the path

510 ini_path = project_dir / "apio.ini"

511

512 # -- Error if apio.ini already exists.

513 if ini_path.exists():

514 cerror("The file apio.ini already exists.")

515 sys.exit(1)

516

517 # -- Construct and write the apio.ini file..

518 cout(f"Creating {ini_path} file ...")

519

520 section_name = "env:default"

521

522 config = ConfigObj(str(ini_path))

523 config.initial_comment = TOP_COMMENT.split("\n")

524 config[section_name] = {}

525 config[section_name]["board"] = board_id

526 config[section_name]["top-module"] = top_module

527

528 config.write()

529 cout(f"The file '{ini_path}' was created successfully.", style=SUCCESS)