Coverage for tests/unit_tests/managers/test_apio_definitions.py: 100%

1# This file is part of the Apio project 

2# (C) 2016-present FPGAwars 

3# Project authors: 

4# * Jesús Arroyo (Jesus89) (2016-2019) 

5# * Juan Gonzalez (obijuan) (2019-2024) 

6# * The Apio Team (2024-present) 

7# License: GPLv2 

8 

9""" 

10Tests of apio_definitions.py 

11""" 

12 

13import os 

14import shutil 

15from pytest import raises 

16from google.protobuf.json_format import MessageToDict 

17from tests.conftest import ApioRunner 

18from apio.common import proto_util 

19from apio.common.proto.stubs.apio_definitions_pb2 import FpgaDefinition 

20from apio.apio_context import ( 

21 ApioContext, 

22 PackagesPolicy, 

23 ProjectPolicy, 

24 RemoteConfigPolicy, 

25) 

26 

27 

28def test_default_loading_no_project(apio_runner: ApioRunner): 

29 """Tests loading of apio standard definitions with no project.""" 

30 

31 with apio_runner.in_sandbox(): 

32 

33 apio_ctx = ApioContext( 

34 project_policy=ProjectPolicy.NO_PROJECT, 

35 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

36 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

37 ) 

38 

39 assert not apio_ctx.has_project 

40 assert apio_ctx.definitions is not None 

41 assert "alhambra-ii" in apio_ctx.definitions.boards 

42 assert "ice40hx4k-tq144-8k" in apio_ctx.definitions.fpgas 

43 assert "openfpgaloader" in apio_ctx.definitions.programmers 

44 

45 

46def test_default_loading_with_project(apio_runner: ApioRunner): 

47 """Tests loading of apio standard definitions with a project and 

48 without project's custom definition.""" 

49 

50 with apio_runner.in_sandbox() as sb: 

51 

52 # -- Fake apio project file. 

53 sb.write_default_apio_ini() 

54 

55 apio_ctx = ApioContext( 

56 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

57 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

58 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

59 ) 

60 

61 assert apio_ctx.has_project 

62 assert apio_ctx.definitions is not None 

63 assert "alhambra-ii" in apio_ctx.definitions.boards 

64 assert "ice40hx4k-tq144-8k" in apio_ctx.definitions.fpgas 

65 assert "openfpgaloader" in apio_ctx.definitions.programmers 

66 

67 

68def test_loading_with_custom_boards(apio_runner: ApioRunner): 

69 """Tests loading of apio standard definitions with a with project's 

70 custom boards definitions. 

71 """ 

72 

73 with apio_runner.in_sandbox() as sb: 

74 

75 # -- Fake apio project file. 

76 sb.write_default_apio_ini() 

77 

78 # -- Write fake custom boards 

79 board_definition1 = { 

80 "description": "My Custom Alhambra II", 

81 "fpga-id": "ice40hx4k-tq144-8k", 

82 "programmer": { 

83 "id": "openfpgaloader", 

84 }, 

85 "usb": { 

86 "vid": "0403", 

87 "pid": "6010", 

88 }, 

89 } 

90 

91 board_definition2 = { 

92 "description": "My new board", 

93 "fpga-id": "lfe5u-45f-6bg381c", 

94 "programmer": { 

95 "id": "openfpgaloader", 

96 }, 

97 } 

98 

99 sb.write_json_file( 

100 "boards.jsonc", 

101 { 

102 # -- Overrides a standard definition 

103 "alhambra-ii": board_definition1, 

104 # -- Adds a new definition 

105 "my-custom-board": board_definition2, 

106 }, 

107 ) 

108 

109 # -- Load 

110 apio_ctx = ApioContext( 

111 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

112 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

113 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

114 ) 

115 

116 # -- Verify 

117 definitions = apio_ctx.definitions 

118 assert definitions is not None 

119 

120 assert apio_ctx.has_project 

121 assert "ice40hx4k-tq144-8k" in definitions.fpgas 

122 assert "openfpgaloader" in definitions.programmers 

123 

124 assert ( 

125 MessageToDict(definitions.boards["alhambra-ii"]) 

126 == board_definition1 

127 ) 

128 assert ( 

129 MessageToDict(definitions.boards["my-custom-board"]) 

130 == board_definition2 

131 ) 

132 

133 assert definitions.is_custom_board("alhambra-ii") 

134 assert definitions.is_custom_board("my-custom-board") 

135 assert not definitions.is_custom_board("sipeed-tang-nano-9k") 

136 

137 

138def test_loading_with_custom_fpgas(apio_runner: ApioRunner): 

139 """Tests loading of apio standard definitions with a with project's 

140 custom fpgas definitions. 

141 """ 

142 

143 with apio_runner.in_sandbox() as sb: 

144 

145 # -- Fake apio project file. 

146 sb.write_default_apio_ini() 

147 

148 # -- Write fake custom boards 

149 fpga_json1 = { 

150 "arch": "ice40", 

151 "vendor-part": "ICE40HX4K-TQ144", 

152 "size": "99k", # Overriding 8k 

153 "ice40-params": { 

154 "nextpnr-type": "hx8k", 

155 "nextpnr-package": "tq144:4k", 

156 }, 

157 } 

158 

159 fpga_json2 = { 

160 "arch": "ice40", 

161 "vendor-part": "MY-CUSTOM-FPGA", 

162 "size": "4k", 

163 "ice40-params": { 

164 "nextpnr-type": "hx4k", 

165 "nextpnr-package": "bg121", 

166 }, 

167 } 

168 

169 sb.write_json_file( 

170 "fpgas.jsonc", 

171 { 

172 # -- Overrides a standard definition 

173 "ice40hx4k-tq144-8k": fpga_json1, 

174 # -- Adds a new definition 

175 "my-custom-fpga": fpga_json2, 

176 }, 

177 ) 

178 

179 # -- Load 

180 apio_ctx = ApioContext( 

181 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

182 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

183 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

184 ) 

185 

186 # -- Verify 

187 definitions = apio_ctx.definitions 

188 assert definitions is not None 

189 

190 assert apio_ctx.has_project 

191 assert "alhambra-ii" in definitions.boards 

192 assert "openfpgaloader" in definitions.programmers 

193 

194 # TODO: Use proto ascii literals instead of converting from json 

195 fpga_proto1 = proto_util.proto_from_json_dict( 

196 fpga_json1, FpgaDefinition 

197 ) 

198 assert fpga_proto1 is not None 

199 

200 fpga_proto2 = proto_util.proto_from_json_dict( 

201 fpga_json2, FpgaDefinition 

202 ) 

203 assert fpga_proto2 is not None 

204 

205 assert definitions.fpgas["ice40hx4k-tq144-8k"] == fpga_proto1 

206 assert definitions.fpgas["my-custom-fpga"] == fpga_proto2 

207 

208 assert definitions.is_custom_fpga("ice40hx4k-tq144-8k") 

209 assert definitions.is_custom_fpga("my-custom-fpga") 

210 assert not definitions.is_custom_fpga("ice40lp1k-cm36") 

211 

212 

213def test_loading_with_custom_programmer(apio_runner: ApioRunner): 

214 """Tests loading of apio standard definitions with a with project's 

215 custom programmer definitions. 

216 """ 

217 

218 with apio_runner.in_sandbox() as sb: 

219 

220 # -- Fake apio project file. 

221 sb.write_default_apio_ini() 

222 

223 # -- Write fake custom programmers 

224 programmer_info1 = { 

225 "command": "custom-iceprog", 

226 "args": "-d d:${BUS}/${DEV}", 

227 } 

228 programmer_info2 = {"command": "iceprog", "args": "my custom args"} 

229 

230 sb.write_json_file( 

231 "programmers.jsonc", 

232 { 

233 # -- Overrides a standard definition 

234 "iceprog": programmer_info1, 

235 # -- Adds a new definition 

236 "my-custom-programmer": programmer_info2, 

237 }, 

238 ) 

239 

240 # -- Load 

241 apio_ctx = ApioContext( 

242 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

243 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

244 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

245 ) 

246 

247 # -- Verify 

248 definitions = apio_ctx.definitions 

249 assert definitions is not None 

250 

251 assert apio_ctx.has_project 

252 assert "alhambra-ii" in definitions.boards 

253 assert "ice40hx4k-tq144-8k" in definitions.fpgas 

254 

255 programmer_definition1 = definitions.programmers["iceprog"] 

256 assert ( 

257 proto_util.proto_to_json_dict(programmer_definition1) 

258 == programmer_info1 

259 ) 

260 

261 programmer_definition2 = definitions.programmers[ 

262 "my-custom-programmer" 

263 ] 

264 assert ( 

265 proto_util.proto_to_json_dict(programmer_definition2) 

266 == programmer_info2 

267 ) 

268 

269 assert definitions.is_custom_programmer("iceprog") 

270 assert definitions.is_custom_programmer("my-custom-programmer") 

271 assert not definitions.is_custom_programmer("openfpgaloader") 

272 

273 

274def test_loading_invalid_custom_board(apio_runner: ApioRunner): 

275 """Tests loading attempt of an invalid board custom definition.""" 

276 

277 with apio_runner.in_sandbox() as sb: 

278 

279 # -- Fake apio project file. 

280 sb.write_default_apio_ini() 

281 

282 # -- Write a custom definition file with an invalid definition 

283 # -- Missing description field. 

284 sb.write_json_file( 

285 "boards.jsonc", 

286 { 

287 "alhambra-ii": { 

288 "fpga-id": "ice40hx4k-tq144-8k", 

289 "programmer": { 

290 "id": "openfpgaloader", 

291 }, 

292 "usb": { 

293 "vid": "0403", 

294 "pid": "6010", 

295 }, 

296 } 

297 }, 

298 ) 

299 

300 # -- Test 

301 with apio_runner.with_logger() as log: 

302 with raises(SystemExit) as e: 

303 _ = ApioContext( 

304 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

305 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

306 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

307 ) 

308 

309 # -- Verify 

310 assert e.value.code == 1 

311 assert "Missing required field 'description'" in log.out 

312 

313 

314def test_loading_invalid_custom_fpga(apio_runner: ApioRunner): 

315 """Tests loading attempt of an invalid board custom definition.""" 

316 

317 with apio_runner.in_sandbox() as sb: 

318 

319 # -- Fake apio project file. 

320 sb.write_default_apio_ini() 

321 

322 # -- Write a custom definition file with an invalid definition 

323 # -- Missing vendor-part field. 

324 sb.write_json_file( 

325 "fpgas.jsonc", 

326 { 

327 "ice40lp384-cm36": { 

328 "arch": "ice40", 

329 "size": "384", 

330 "ice40-params": { 

331 "nextpnr-type": "lp384", 

332 "nextpnr-package": "cm36", 

333 }, 

334 } 

335 }, 

336 ) 

337 

338 # -- Test 

339 with apio_runner.with_logger() as log: 

340 with raises(SystemExit) as e: 

341 _ = ApioContext( 

342 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

343 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

344 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

345 ) 

346 

347 # -- Verify 

348 assert e.value.code == 1 

349 assert "Missing required field 'vendor-part'" in log.out 

350 

351 

352def test_loading_invalid_custom_programmer(apio_runner: ApioRunner): 

353 """Tests loading attempt of an invalid programmer custom definition.""" 

354 

355 with apio_runner.in_sandbox() as sb: 

356 

357 # -- Fake apio project file. 

358 sb.write_default_apio_ini() 

359 

360 # -- Write a custom definition file with an invalid definition 

361 # -- Missing command field. 

362 sb.write_json_file( 

363 "programmers.jsonc", 

364 { 

365 "openfpgaloader": { 

366 "args": "--force-terminal-mode --verify", 

367 } 

368 }, 

369 ) 

370 

371 # -- Test 

372 with apio_runner.with_logger() as log: 

373 with raises(SystemExit) as e: 

374 _ = ApioContext( 

375 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

376 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

377 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

378 ) 

379 

380 # -- Verify 

381 assert e.value.code == 1 

382 assert "Missing required field 'command'" in log.out 

383 

384 

385def test_loading_invalid_custom_board_id(apio_runner: ApioRunner): 

386 """Tests loading attempt of an invalid board id.""" 

387 

388 with apio_runner.in_sandbox() as sb: 

389 

390 # -- Fake apio project file. 

391 sb.write_default_apio_ini() 

392 

393 # -- Write a custom definition file with an invalid definition 

394 # -- Missing description field. 

395 sb.write_json_file( 

396 "boards.jsonc", 

397 { 

398 "invalid-ID": { 

399 "description": "BlackIce MX", 

400 "fpga-id": "ice40hx4k-tq144-8k", 

401 "programmer": {"id": "blackiceprog"}, 

402 "usb": {"vid": "0483", "pid": "5740"}, 

403 } 

404 }, 

405 ) 

406 

407 # -- Test 

408 with apio_runner.with_logger() as log: 

409 with raises(SystemExit) as e: 

410 _ = ApioContext( 

411 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

412 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

413 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

414 ) 

415 

416 # -- Verify 

417 assert e.value.code == 1 

418 assert "Board id `invalid-ID` has an invalid format" in log.out 

419 

420 

421def test_loading_invalid_custom_fpga_id(apio_runner: ApioRunner): 

422 """Tests loading attempt of an invalid fpga id.""" 

423 

424 with apio_runner.in_sandbox() as sb: 

425 

426 # -- Fake apio project file. 

427 sb.write_default_apio_ini() 

428 

429 # -- Write a custom definition file with an invalid definition 

430 # -- Missing description field. 

431 sb.write_json_file( 

432 "fpgas.jsonc", 

433 { 

434 "invalid-ID": { 

435 "arch": "ice40", 

436 "vendor-part": "ICE40LP1K-SWG16TR", 

437 "size": "1k", 

438 "ice40-params": { 

439 "nextpnr-type": "lp1k", 

440 "nextpnr-package": "swg16tr", 

441 }, 

442 } 

443 }, 

444 ) 

445 

446 # -- Test 

447 with apio_runner.with_logger() as log: 

448 

449 with raises(SystemExit) as e: 

450 _ = ApioContext( 

451 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

452 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

453 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

454 ) 

455 

456 # -- Verify 

457 assert e.value.code == 1 

458 assert "FPGA id has an invalid format: invalid-ID" in log.out 

459 

460 

461def test_loading_invalid_custom_programmer_id(apio_runner: ApioRunner): 

462 """Tests loading attempt of an invalid programmer id.""" 

463 

464 with apio_runner.in_sandbox() as sb: 

465 

466 # -- Fake apio project file. 

467 sb.write_default_apio_ini() 

468 

469 # -- Write a custom definition file with an invalid definition 

470 # -- Missing description field. 

471 sb.write_json_file( 

472 "programmers.jsonc", 

473 { 

474 "invalid-ID": { 

475 "command": "openFPGALoader", 

476 "args": "--force-terminal-mode --verify", 

477 } 

478 }, 

479 ) 

480 

481 # -- Test 

482 with apio_runner.with_logger() as log: 

483 with raises(SystemExit) as e: 

484 _ = ApioContext( 

485 project_policy=ProjectPolicy.PROJECT_REQUIRED, 

486 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

487 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

488 ) 

489 

490 # -- Verify 

491 assert e.value.code == 1 

492 assert "Programmer id has an invalid format: invalid-ID" in log.out 

493 

494 

495def test_definitions_env_var(apio_runner: ApioRunner): 

496 """Tests that APIO_DEFINITIONS overrides the definitions dir.""" 

497 

498 with apio_runner.in_sandbox() as sb: 

499 

500 # -- Make sure the packages are installed. 

501 _ = ApioContext( 

502 project_policy=ProjectPolicy.NO_PROJECT, 

503 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

504 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

505 ) 

506 

507 # -- Copy the definitions to another dir and point to it. 

508 my_dir = sb.sandbox_dir / "my-definitions" 

509 shutil.copytree(sb.packages_dir / "definitions", my_dir) 

510 sb.write_json_file(my_dir / "boards.jsonc", {}, exists_ok=True) 

511 os.environ["APIO_DEFINITIONS"] = str(my_dir) 

512 

513 apio_ctx = ApioContext( 

514 project_policy=ProjectPolicy.NO_PROJECT, 

515 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

516 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

517 ) 

518 

519 assert apio_ctx.definitions is not None 

520 assert apio_ctx.definitions.package_definitions_dir == my_dir 

521 # -- The boards came from the empty boards.jsonc in my_dir. 

522 assert not apio_ctx.definitions.boards 

523 

524 

525def test_definitions_env_var_bad_dir(apio_runner: ApioRunner): 

526 """Tests that APIO_DEFINITIONS with a non existing dir is an error.""" 

527 

528 with apio_runner.in_sandbox() as sb: 

529 

530 os.environ["APIO_DEFINITIONS"] = str(sb.sandbox_dir / "no-such-dir") 

531 

532 with apio_runner.with_logger() as log: 

533 with raises(SystemExit) as e: 

534 _ = ApioContext( 

535 project_policy=ProjectPolicy.NO_PROJECT, 

536 remote_config_policy=RemoteConfigPolicy.CACHED_OK, 

537 packages_policy=PackagesPolicy.ENSURE_PACKAGES, 

538 ) 

539 

540 assert e.value.code == 1 

541 assert "APIO_DEFINITIONS is not a directory" in log.out