Coverage for apio / managers / scons_manager.py: 80%

1"""A manager class for to dispatch the Apio SCONS targets.""" 

2 

3# -*- coding: utf-8 -*- 

4# -- This file is part of the Apio project 

5# -- (C) 2016-2019 FPGAwars 

6# -- Author Jesús Arroyo 

7# -- License GPLv2 

8 

9import traceback 

10import os 

11import sys 

12import time 

13import shutil 

14from functools import wraps 

15from datetime import datetime 

16from google.protobuf import text_format 

17from apio.common import apio_console 

18from apio.common.apio_console import cout, cerror, cstyle, cunstyle 

19from apio.common.apio_styles import SUCCESS, ERROR, EMPH3, INFO 

20from apio.utils import util 

21from apio.apio_context import ApioContext 

22from apio.managers.scons_filter import SconsFilter 

23from apio.common.proto.apio_pb2 import ( 

24 FORCE_PIPE, 

25 FORCE_TERMINAL, 

26 Verbosity, 

27 Environment, 

28 SconsParams, 

29 TargetParams, 

30 FpgaInfo, 

31 ApioEnvParams, 

32 Ice40FpgaInfo, 

33 Ecp5FpgaInfo, 

34 GowinFpgaInfo, 

35 ApioArch, 

36 GraphParams, 

37 LintParams, 

38 SimParams, 

39 ApioTestParams, 

40 UploadParams, 

41) 

42 

43# from apio.common import rich_lib_windows 

44 

45 

46# W0703: Catching too general exception Exception (broad-except) 

47# pylint: disable=W0703 

48# 

49# -- Based on 

50# -- https://stackoverflow.com/questions/5929107/decorators-with-parameters 

51def on_exception(*, exit_code: int): 

52 """Decorator for functions that return int exit code. If the function 

53 throws an exception, the error message is printed, and the caller see the 

54 returned value exit_code instead of the exception. 

55 """ 

56 

57 def decorator(function): 

58 @wraps(function) 

59 def wrapper(*args, **kwargs): 

60 try: 

61 return function(*args, **kwargs) 

62 except Exception as exc: 

63 if util.is_debug(1): 

64 traceback.print_tb(exc.__traceback__) 

65 

66 if str(exc): 

67 cerror(str(exc)) 

68 

69 if not util.is_debug(1): 

70 cout( 

71 "Running with APIO_DEBUG=1 may provide " 

72 "additional diagnostic information.", 

73 style=INFO, 

74 ) 

75 return exit_code 

76 

77 return wrapper 

78 

79 return decorator 

80 

81 

82class SConsManager: 

83 """Class for managing the scons tools""" 

84 

85 def __init__(self, apio_ctx: ApioContext): 

86 """Initialization.""" 

87 # -- Cache the apio context. 

88 self.apio_ctx = apio_ctx 

89 

90 # -- Change to the project's folder. 

91 os.chdir(apio_ctx.project_dir) 

92 

93 @on_exception(exit_code=1) 

94 def graph(self, graph_params: GraphParams, verbosity: Verbosity) -> int: 

95 """Runs a scons subprocess with the 'graph' target. Returns process 

96 exit code, 0 if ok.""" 

97 

98 # -- Construct scons params with graph command info. 

99 scons_params = self.construct_scons_params( 

100 target_params=TargetParams(graph=graph_params), 

101 verbosity=verbosity, 

102 ) 

103 

104 # -- Run the scons process. 

105 return self._run_scons_subprocess("graph", scons_params=scons_params) 

106 

107 @on_exception(exit_code=1) 

108 def lint(self, lint_params: LintParams) -> int: 

109 """Runs a scons subprocess with the 'lint' target. Returns process 

110 exit code, 0 if ok.""" 

111 

112 # -- Construct scons params with graph command info. 

113 scons_params = self.construct_scons_params( 

114 target_params=TargetParams(lint=lint_params) 

115 ) 

116 

117 # -- Run the scons process. 

118 return self._run_scons_subprocess("lint", scons_params=scons_params) 

119 

120 @on_exception(exit_code=1) 

121 def sim(self, sim_params: SimParams) -> int: 

122 """Runs a scons subprocess with the 'sim' target. Returns process 

123 exit code, 0 if ok.""" 

124 

125 # -- Construct scons params with graph command info. 

126 scons_params = self.construct_scons_params( 

127 target_params=TargetParams(sim=sim_params) 

128 ) 

129 

130 # -- Run the scons process. 

131 return self._run_scons_subprocess("sim", scons_params=scons_params) 

132 

133 @on_exception(exit_code=1) 

134 def test(self, test_params: ApioTestParams) -> int: 

135 """Runs a scons subprocess with the 'test' target. Returns process 

136 exit code, 0 if ok.""" 

137 

138 # -- Construct scons params with graph command info. 

139 scons_params = self.construct_scons_params( 

140 target_params=TargetParams(test=test_params) 

141 ) 

142 

143 # -- Run the scons process. 

144 return self._run_scons_subprocess("test", scons_params=scons_params) 

145 

146 @on_exception(exit_code=1) 

147 def build(self, verbosity: Verbosity) -> int: 

148 """Runs a scons subprocess with the 'build' target. Returns process 

149 exit code, 0 if ok.""" 

150 

151 # -- Construct the scons params object. 

152 scons_params = self.construct_scons_params(verbosity=verbosity) 

153 

154 # -- Run the scons process. 

155 return self._run_scons_subprocess("build", scons_params=scons_params) 

156 

157 @on_exception(exit_code=1) 

158 def report(self, verbosity: Verbosity) -> int: 

159 """Runs a scons subprocess with the 'report' target. Returns process 

160 exit code, 0 if ok.""" 

161 

162 # -- Construct the scons params object. 

163 scons_params = self.construct_scons_params(verbosity=verbosity) 

164 

165 # -- Run the scons process. 

166 return self._run_scons_subprocess("report", scons_params=scons_params) 

167 

168 @on_exception(exit_code=1) 

169 def upload(self, upload_params: UploadParams) -> int: 

170 """Runs a scons subprocess with the 'time' target. Returns process 

171 exit code, 0 if ok. 

172 """ 

173 

174 # -- Construct the scons params. 

175 scons_params = self.construct_scons_params( 

176 target_params=TargetParams(upload=upload_params) 

177 ) 

178 

179 # -- Execute Scons for uploading! 

180 exit_code = self._run_scons_subprocess( 

181 "upload", scons_params=scons_params 

182 ) 

183 

184 return exit_code 

185 

186 def construct_scons_params( 

187 self, 

188 *, 

189 target_params: TargetParams = None, 

190 verbosity: Verbosity = None, 

191 ) -> SconsParams: 

192 """Populate and return the SconsParam proto to pass to the scons 

193 process.""" 

194 

195 # -- Create a shortcut. 

196 apio_ctx = self.apio_ctx 

197 

198 # -- Create an empty proto object that will be populated. 

199 result = SconsParams() 

200 

201 # -- Populate the timestamp. We use to to make sure scons reads the 

202 # -- correct version of the scons.params file. 

203 ts = datetime.now() 

204 result.timestamp = ts.strftime("%d%H%M%S%f")[:-3] 

205 

206 # -- Get the project data. All commands that invoke scons are expected 

207 # -- to be in a project context. 

208 assert apio_ctx.has_project, "Scons encountered a missing project." 

209 project = apio_ctx.project 

210 

211 # -- Get the project resource.s 

212 pr = apio_ctx.project_resources 

213 

214 # -- Populate the common values of FpgaInfo. 

215 result.fpga_info.MergeFrom( 

216 FpgaInfo( 

217 fpga_id=pr.fpga_id, 

218 part_num=pr.fpga_info["part-num"], 

219 size=pr.fpga_info["size"], 

220 ) 

221 ) 

222 

223 # - Populate the architecture specific values of result.fpga_info. 

224 fpga_arch = pr.fpga_info["arch"] 

225 match fpga_arch: 

226 case "ice40": 

227 result.arch = ApioArch.ICE40 

228 result.fpga_info.ice40.MergeFrom( 

229 Ice40FpgaInfo( 

230 type=pr.fpga_info["type"], pack=pr.fpga_info["pack"] 

231 ) 

232 ) 

233 case "ecp5": 

234 result.arch = ApioArch.ECP5 

235 result.fpga_info.ecp5.MergeFrom( 

236 Ecp5FpgaInfo( 

237 type=pr.fpga_info["type"], 

238 pack=pr.fpga_info["pack"], 

239 speed=pr.fpga_info["speed"], 

240 ) 

241 ) 

242 case "gowin": 242 ↛ 247line 242 didn't jump to line 247 because the pattern on line 242 always matched

243 result.arch = ApioArch.GOWIN 

244 result.fpga_info.gowin.MergeFrom( 

245 GowinFpgaInfo(family=pr.fpga_info["type"]) 

246 ) 

247 case _: 

248 cerror(f"Unexpected fpga_arch value {fpga_arch}") 

249 sys.exit(1) 

250 

251 # -- We are done populating The FpgaInfo params.. 

252 assert result.fpga_info.IsInitialized(), result 

253 

254 # -- Populate the optional Verbosity params. 

255 if verbosity: 

256 result.verbosity.MergeFrom(verbosity) 

257 assert result.verbosity.IsInitialized(), result 

258 

259 # -- Populate the Environment params. 

260 assert apio_ctx.platform_id, "Missing platform_id in apio context" 

261 oss_vars = apio_ctx.all_packages["oss-cad-suite"]["env"]["vars"] 

262 

263 result.environment.MergeFrom( 

264 Environment( 

265 platform_id=apio_ctx.platform_id, 

266 is_windows=apio_ctx.is_windows, 

267 terminal_mode=( 

268 FORCE_TERMINAL 

269 if apio_console.is_terminal() 

270 else FORCE_PIPE 

271 ), 

272 theme_name=apio_console.current_theme_name(), 

273 debug_level=util.debug_level(), 

274 yosys_path=oss_vars["YOSYS_LIB"], 

275 trellis_path=oss_vars["TRELLIS"], 

276 scons_shell_id=apio_ctx.scons_shell_id, 

277 ) 

278 ) 

279 assert result.environment.IsInitialized(), result 

280 

281 # -- Populate the Project params. 

282 result.apio_env_params.MergeFrom( 

283 ApioEnvParams( 

284 env_name=apio_ctx.project.env_name, 

285 board_id=pr.board_id, 

286 top_module=project.get_str_option("top-module"), 

287 defines=apio_ctx.project.get_list_option( 

288 "defines", default=[] 

289 ), 

290 yosys_synth_extra_options=apio_ctx.project.get_list_option( 

291 "yosys-synth-extra-options", None 

292 ), 

293 nextpnr_extra_options=apio_ctx.project.get_list_option( 

294 "nextpnr-extra-options", None 

295 ), 

296 gtkwave_extra_options=apio_ctx.project.get_list_option( 

297 "gtkwave-extra-options", None 

298 ), 

299 constraint_file=apio_ctx.project.get_str_option( 

300 "constraint-file", None 

301 ), 

302 ) 

303 ) 

304 assert result.apio_env_params.IsInitialized(), result 

305 

306 # -- Populate the optional command specific params. 

307 if target_params: 

308 result.target.MergeFrom(target_params) 

309 assert result.target.IsInitialized(), result 

310 

311 # -- All done. 

312 assert result.IsInitialized(), result 

313 return result 

314 

315 def _run_scons_subprocess( 

316 self, scons_command: str, *, scons_params: SconsParams = None 

317 ): 

318 """Invoke an scons subprocess.""" 

319 

320 # pylint: disable=too-many-locals 

321 

322 # -- Create a shortcut. 

323 apio_ctx = self.apio_ctx 

324 

325 # -- Pass to the scons process the name of the sconstruct file it 

326 # -- should use. 

327 scons_dir = util.get_path_in_apio_package("scons") 

328 scons_file_path = scons_dir / "SConstruct" 

329 variables = ["-f", f"{scons_file_path}"] 

330 

331 # -- Pass the path to the proto params file. The path is relative 

332 # -- to the project root. 

333 params_file_path = apio_ctx.env_build_path / "scons.params" 

334 variables += [f"params={str(params_file_path)}"] 

335 

336 # -- Pass to the scons process the timestamp of the scons params we 

337 # -- pass via a file. This is for verification purposes only. 

338 variables += [f"timestamp={scons_params.timestamp}"] 

339 

340 # -- We set the env variables also for a command such as 'clean' 

341 # -- which doesn't use the packages, to satisfy the required env 

342 # -- variables of the scons arg parser. 

343 apio_ctx.set_env_for_packages() 

344 

345 if util.is_debug(1): 345 ↛ 346line 345 didn't jump to line 346 because the condition on line 345 was never true

346 cout("\nSCONS CALL:", style=EMPH3) 

347 cout(f"* command: {scons_command}") 

348 cout(f"* variables: {variables}") 

349 cout(f"* scons params: \n{scons_params}") 

350 cout() 

351 

352 # -- Get the terminal width (typically 80) 

353 terminal_width, _ = shutil.get_terminal_size() 

354 

355 # -- Read the time (for measuring how long does it take 

356 # -- to execute the apio command) 

357 start_time = time.time() 

358 

359 # -- Subtracting 1 to avoid line overflow on windows, Observed with 

360 # -- Windows 10 and cmd.exe shell. 

361 if apio_ctx.is_windows: 361 ↛ 362line 361 didn't jump to line 362 because the condition on line 361 was never true

362 terminal_width -= 1 

363 

364 # -- Print a horizontal line 

365 cout("-" * terminal_width) 

366 

367 # -- Create the scons debug options. See details at 

368 # -- https://scons.org/doc/2.4.1/HTML/scons-man.html 

369 debug_options = ( 

370 ["--debug=explain,prepare,stacktrace", "--tree=all"] 

371 if util.is_debug(1) 

372 else [] 

373 ) 

374 

375 # -- Construct the scons command line. 

376 # -- 

377 # -- sys.executable is resolved to the full path of the python 

378 # -- interpreter or to apio if running from a pyinstall setup. 

379 # -- See https://github.com/orgs/pyinstaller/discussions/9023 for more 

380 # -- information. 

381 # -- 

382 # -- We use exec -m SCons instead of scones also for non pyinstaller 

383 # -- deployment in case the scons binary is not on the PATH. 

384 cmd = ( 

385 [sys.executable, "-m", "apio", "--scons"] 

386 + ["-Q", scons_command] 

387 + debug_options 

388 + variables 

389 ) 

390 

391 # -- An output filter that manipulates the scons stdout/err lines as 

392 # -- needed and write them to stdout. 

393 scons_filter = SconsFilter( 

394 colors_enabled=apio_console.is_colors_enabled() 

395 ) 

396 

397 # -- Write the scons parameters to a temp file in the build 

398 # -- directory. It will be cleaned up as part of 'apio cleanup'. 

399 # -- At this point, the project is the current directory, even if 

400 # -- the command used the --project-dir option. 

401 os.makedirs(apio_ctx.env_build_path, exist_ok=True) 

402 with open(params_file_path, "w", encoding="utf8") as f: 

403 f.write(text_format.MessageToString(scons_params)) 

404 

405 if util.is_debug(1): 405 ↛ 406line 405 didn't jump to line 406 because the condition on line 405 was never true

406 cout(f"\nFull scons command: {cmd}\n\n") 

407 

408 # -- Execute the scons builder! 

409 result = util.exec_command( 

410 cmd, 

411 stdout=util.AsyncPipe(scons_filter.on_stdout_line), 

412 stderr=util.AsyncPipe(scons_filter.on_stderr_line), 

413 ) 

414 

415 # -- Is there an error? True/False 

416 is_error = result.exit_code != 0 

417 

418 # -- Calculate the time it took to execute the command 

419 duration = time.time() - start_time 

420 

421 # -- Determine status message 

422 if is_error: 422 ↛ 423line 422 didn't jump to line 423 because the condition on line 422 was never true

423 styled_status = cstyle("ERROR", style=ERROR) 

424 else: 

425 styled_status = cstyle("SUCCESS", style=SUCCESS) 

426 

427 # -- Determine the summary text 

428 summary = f"Took {duration:.2f} seconds" 

429 

430 # -- Construct the entire message. 

431 styled_msg = f" [{styled_status}] {summary} " 

432 msg_len = len(cunstyle(styled_msg)) 

433 

434 # -- Determine the lengths of the paddings before and after 

435 # -- the message. Should be correct for odd and even terminal 

436 # -- widths. 

437 pad1_len = (terminal_width - msg_len) // 2 

438 pad2_len = terminal_width - pad1_len - msg_len 

439 

440 # -- Print the entire line. 

441 cout(f"{'=' * pad1_len}{styled_msg}{'=' * pad2_len}") 

442 

443 # -- Return the exit code 

444 return result.exit_code