Coverage for apio / managers / scons_manager.py: 80%

1"""A manager class for to dispatch the Apio SCONS targets.""" 

2 

3# -*- coding: utf-8 -*- 

4# -- This file is part of the Apio project 

5# -- (C) 2016-2019 FPGAwars 

6# -- Author Jesús Arroyo 

7# -- License GPLv2 

8 

9import traceback 

10import os 

11import sys 

12import time 

13import shutil 

14from functools import wraps 

15from datetime import datetime 

16from google.protobuf import text_format 

17from apio.common import apio_console 

18from apio.common.apio_console import cout, cerror, cstyle, cunstyle 

19from apio.common.apio_styles import SUCCESS, ERROR, EMPH3, INFO 

20from apio.utils import util 

21from apio.apio_context import ApioContext 

22from apio.managers.scons_filter import SconsFilter 

23from apio.common.proto.apio_pb2 import ( 

24 FORCE_PIPE, 

25 FORCE_TERMINAL, 

26 Verbosity, 

27 Environment, 

28 SconsParams, 

29 TargetParams, 

30 FpgaInfo, 

31 ApioEnvParams, 

32 Ice40FpgaInfo, 

33 Ecp5FpgaInfo, 

34 GowinFpgaInfo, 

35 ApioArch, 

36 GraphParams, 

37 LintParams, 

38 SimParams, 

39 ApioTestParams, 

40 UploadParams, 

41) 

42 

43# from apio.common import rich_lib_windows 

44 

45 

46# W0703: Catching too general exception Exception (broad-except) 

47# pylint: disable=W0703 

48# 

49# -- Based on 

50# -- https://stackoverflow.com/questions/5929107/decorators-with-parameters 

51def on_exception(*, exit_code: int): 

52 """Decorator for functions that return int exit code. If the function 

53 throws an exception, the error message is printed, and the caller see the 

54 returned value exit_code instead of the exception. 

55 """ 

56 

57 def decorator(function): 

58 @wraps(function) 

59 def wrapper(*args, **kwargs): 

60 try: 

61 return function(*args, **kwargs) 

62 except Exception as exc: 

63 if util.is_debug(1): 

64 traceback.print_tb(exc.__traceback__) 

65 

66 if str(exc): 

67 cerror(str(exc)) 

68 

69 if not util.is_debug(1): 

70 cout( 

71 "Running with APIO_DEBUG=1 may provide " 

72 "additional diagnostic information.", 

73 style=INFO, 

74 ) 

75 return exit_code 

76 

77 return wrapper 

78 

79 return decorator 

80 

81 

82class SConsManager: 

83 """Class for managing the scons tools""" 

84 

85 def __init__(self, apio_ctx: ApioContext): 

86 """Initialization.""" 

87 # -- Cache the apio context. 

88 self.apio_ctx = apio_ctx 

89 

90 # -- Change to the project's folder. 

91 os.chdir(apio_ctx.project_dir) 

92 

93 @on_exception(exit_code=1) 

94 def graph(self, graph_params: GraphParams, verbosity: Verbosity) -> int: 

95 """Runs a scons subprocess with the 'graph' target. Returns process 

96 exit code, 0 if ok.""" 

97 

98 # -- Construct scons params with graph command info. 

99 scons_params = self.construct_scons_params( 

100 target_params=TargetParams(graph=graph_params), 

101 verbosity=verbosity, 

102 ) 

103 

104 # -- Run the scons process. 

105 return self._run_scons_subprocess("graph", scons_params=scons_params) 

106 

107 @on_exception(exit_code=1) 

108 def lint(self, lint_params: LintParams) -> int: 

109 """Runs a scons subprocess with the 'lint' target. Returns process 

110 exit code, 0 if ok.""" 

111 

112 # -- Construct scons params with graph command info. 

113 scons_params = self.construct_scons_params( 

114 target_params=TargetParams(lint=lint_params) 

115 ) 

116 

117 # -- Run the scons process. 

118 return self._run_scons_subprocess("lint", scons_params=scons_params) 

119 

120 @on_exception(exit_code=1) 

121 def sim(self, sim_params: SimParams) -> int: 

122 """Runs a scons subprocess with the 'sim' target. Returns process 

123 exit code, 0 if ok.""" 

124 

125 # -- Construct scons params with graph command info. 

126 scons_params = self.construct_scons_params( 

127 target_params=TargetParams(sim=sim_params) 

128 ) 

129 

130 # -- Run the scons process. 

131 return self._run_scons_subprocess("sim", scons_params=scons_params) 

132 

133 @on_exception(exit_code=1) 

134 def test(self, test_params: ApioTestParams) -> int: 

135 """Runs a scons subprocess with the 'test' target. Returns process 

136 exit code, 0 if ok.""" 

137 

138 # -- Construct scons params with graph command info. 

139 scons_params = self.construct_scons_params( 

140 target_params=TargetParams(test=test_params) 

141 ) 

142 

143 # -- Run the scons process. 

144 return self._run_scons_subprocess("test", scons_params=scons_params) 

145 

146 @on_exception(exit_code=1) 

147 def build(self, verbosity: Verbosity) -> int: 

148 """Runs a scons subprocess with the 'build' target. Returns process 

149 exit code, 0 if ok.""" 

150 

151 # -- Construct the scons params object. 

152 scons_params = self.construct_scons_params(verbosity=verbosity) 

153 

154 # -- Run the scons process. 

155 return self._run_scons_subprocess("build", scons_params=scons_params) 

156 

157 @on_exception(exit_code=1) 

158 def report(self, verbosity: Verbosity) -> int: 

159 """Runs a scons subprocess with the 'report' target. Returns process 

160 exit code, 0 if ok.""" 

161 

162 # -- Construct the scons params object. 

163 scons_params = self.construct_scons_params(verbosity=verbosity) 

164 

165 # -- Run the scons process. 

166 return self._run_scons_subprocess("report", scons_params=scons_params) 

167 

168 @on_exception(exit_code=1) 

169 def upload(self, upload_params: UploadParams) -> int: 

170 """Runs a scons subprocess with the 'time' target. Returns process 

171 exit code, 0 if ok. 

172 """ 

173 

174 # -- Construct the scons params. 

175 scons_params = self.construct_scons_params( 

176 target_params=TargetParams(upload=upload_params) 

177 ) 

178 

179 # -- Execute Scons for uploading! 

180 exit_code = self._run_scons_subprocess( 

181 "upload", scons_params=scons_params 

182 ) 

183 

184 return exit_code 

185 

186 def construct_scons_params( 

187 self, 

188 *, 

189 target_params: TargetParams = None, 

190 verbosity: Verbosity = None, 

191 ) -> SconsParams: 

192 """Populate and return the SconsParam proto to pass to the scons 

193 process.""" 

194 

195 # -- Create a shortcut. 

196 apio_ctx = self.apio_ctx 

197 

198 # -- Create an empty proto object that will be populated. 

199 result = SconsParams() 

200 

201 # -- Populate the timestamp. We use to to make sure scons reads the 

202 # -- correct version of the scons.params file. 

203 ts = datetime.now() 

204 result.timestamp = ts.strftime("%d%H%M%S%f")[:-3] 

205 

206 # -- Get the project data. All commands that invoke scons are expected 

207 # -- to be in a project context. 

208 assert apio_ctx.has_project, "Scons encountered a missing project." 

209 project = apio_ctx.project 

210 

211 # -- Get the project resource.s 

212 pr = apio_ctx.project_resources 

213 

214 # -- Populate the common values of FpgaInfo. 

215 result.fpga_info.MergeFrom( 

216 FpgaInfo( 

217 fpga_id=pr.fpga_id, 

218 part_num=pr.fpga_info["part-num"], 

219 size=pr.fpga_info["size"], 

220 ) 

221 ) 

222 

223 # - Populate the architecture specific values of result.fpga_info. 

224 fpga_arch = pr.fpga_info["arch"] 

225 match fpga_arch: 

226 case "ice40": 

227 result.arch = ApioArch.ICE40 

228 result.fpga_info.ice40.MergeFrom( 

229 Ice40FpgaInfo( 

230 type=pr.fpga_info["type"], pack=pr.fpga_info["pack"] 

231 ) 

232 ) 

233 case "ecp5": 

234 result.arch = ApioArch.ECP5 

235 result.fpga_info.ecp5.MergeFrom( 

236 Ecp5FpgaInfo( 

237 type=pr.fpga_info["type"], 

238 pack=pr.fpga_info["pack"], 

239 speed=pr.fpga_info["speed"], 

240 ) 

241 ) 

242 case "gowin": 242 ↛ 247line 242 didn't jump to line 247 because the pattern on line 242 always matched

243 result.arch = ApioArch.GOWIN 

244 result.fpga_info.gowin.MergeFrom( 

245 GowinFpgaInfo(family=pr.fpga_info["type"]) 

246 ) 

247 case _: 

248 cerror(f"Unexpected fpga_arch value {fpga_arch}") 

249 sys.exit(1) 

250 

251 # -- We are done populating The FpgaInfo params.. 

252 assert result.fpga_info.IsInitialized(), result 

253 

254 # -- Populate the optional Verbosity params. 

255 if verbosity: 

256 result.verbosity.MergeFrom(verbosity) 

257 assert result.verbosity.IsInitialized(), result 

258 

259 # -- Populate the Environment params. 

260 assert apio_ctx.platform_id, "Missing platform_id in apio context" 

261 oss_vars = apio_ctx.all_packages["oss-cad-suite"]["env"]["vars"] 

262 

263 result.environment.MergeFrom( 

264 Environment( 

265 platform_id=apio_ctx.platform_id, 

266 is_windows=apio_ctx.is_windows, 

267 terminal_mode=( 

268 FORCE_TERMINAL 

269 if apio_console.is_terminal() 

270 else FORCE_PIPE 

271 ), 

272 theme_name=apio_console.current_theme_name(), 

273 debug_level=util.debug_level(), 

274 yosys_path=oss_vars["YOSYS_LIB"], 

275 trellis_path=oss_vars["TRELLIS"], 

276 scons_shell_id=apio_ctx.scons_shell_id, 

277 ) 

278 ) 

279 assert result.environment.IsInitialized(), result 

280 

281 # -- Populate the Project params. 

282 result.apio_env_params.MergeFrom( 

283 ApioEnvParams( 

284 env_name=apio_ctx.project.env_name, 

285 board_id=pr.board_id, 

286 top_module=project.get_str_option("top-module"), 

287 defines=apio_ctx.project.get_list_option( 

288 "defines", default=[] 

289 ), 

290 yosys_synth_extra_options=apio_ctx.project.get_list_option( 

291 "yosys-synth-extra-options", None 

292 ), 

293 nextpnr_extra_options=apio_ctx.project.get_list_option( 

294 "nextpnr-extra-options", None 

295 ), 

296 constraint_file=apio_ctx.project.get_str_option( 

297 "constraint-file", None 

298 ), 

299 ) 

300 ) 

301 assert result.apio_env_params.IsInitialized(), result 

302 

303 # -- Populate the optional command specific params. 

304 if target_params: 

305 result.target.MergeFrom(target_params) 

306 assert result.target.IsInitialized(), result 

307 

308 # -- All done. 

309 assert result.IsInitialized(), result 

310 return result 

311 

312 def _run_scons_subprocess( 

313 self, scons_command: str, *, scons_params: SconsParams = None 

314 ): 

315 """Invoke an scons subprocess.""" 

316 

317 # pylint: disable=too-many-locals 

318 

319 # -- Create a shortcut. 

320 apio_ctx = self.apio_ctx 

321 

322 # -- Pass to the scons process the name of the sconstruct file it 

323 # -- should use. 

324 scons_dir = util.get_path_in_apio_package("scons") 

325 scons_file_path = scons_dir / "SConstruct" 

326 variables = ["-f", f"{scons_file_path}"] 

327 

328 # -- Pass the path to the proto params file. The path is relative 

329 # -- to the project root. 

330 params_file_path = apio_ctx.env_build_path / "scons.params" 

331 variables += [f"params={str(params_file_path)}"] 

332 

333 # -- Pass to the scons process the timestamp of the scons params we 

334 # -- pass via a file. This is for verification purposes only. 

335 variables += [f"timestamp={scons_params.timestamp}"] 

336 

337 # -- We set the env variables also for a command such as 'clean' 

338 # -- which doesn't use the packages, to satisfy the required env 

339 # -- variables of the scons arg parser. 

340 apio_ctx.set_env_for_packages() 

341 

342 if util.is_debug(1): 342 ↛ 343line 342 didn't jump to line 343 because the condition on line 342 was never true

343 cout("\nSCONS CALL:", style=EMPH3) 

344 cout(f"* command: {scons_command}") 

345 cout(f"* variables: {variables}") 

346 cout(f"* scons params: \n{scons_params}") 

347 cout() 

348 

349 # -- Get the terminal width (typically 80) 

350 terminal_width, _ = shutil.get_terminal_size() 

351 

352 # -- Read the time (for measuring how long does it take 

353 # -- to execute the apio command) 

354 start_time = time.time() 

355 

356 # -- Subtracting 1 to avoid line overflow on windows, Observed with 

357 # -- Windows 10 and cmd.exe shell. 

358 if apio_ctx.is_windows: 358 ↛ 359line 358 didn't jump to line 359 because the condition on line 358 was never true

359 terminal_width -= 1 

360 

361 # -- Print a horizontal line 

362 cout("-" * terminal_width) 

363 

364 # -- Create the scons debug options. See details at 

365 # -- https://scons.org/doc/2.4.1/HTML/scons-man.html 

366 debug_options = ( 

367 ["--debug=explain,prepare,stacktrace", "--tree=all"] 

368 if util.is_debug(1) 

369 else [] 

370 ) 

371 

372 # -- Construct the scons command line. 

373 # -- 

374 # -- sys.executable is resolved to the full path of the python 

375 # -- interpreter or to apio if running from a pyinstall setup. 

376 # -- See https://github.com/orgs/pyinstaller/discussions/9023 for more 

377 # -- information. 

378 # -- 

379 # -- We use exec -m SCons instead of scones also for non pyinstaller 

380 # -- deployment in case the scons binary is not on the PATH. 

381 cmd = ( 

382 [sys.executable, "-m", "apio", "--scons"] 

383 + ["-Q", scons_command] 

384 + debug_options 

385 + variables 

386 ) 

387 

388 # -- An output filter that manipulates the scons stdout/err lines as 

389 # -- needed and write them to stdout. 

390 scons_filter = SconsFilter( 

391 colors_enabled=apio_console.is_colors_enabled() 

392 ) 

393 

394 # -- Write the scons parameters to a temp file in the build 

395 # -- directory. It will be cleaned up as part of 'apio cleanup'. 

396 # -- At this point, the project is the current directory, even if 

397 # -- the command used the --project-dir option. 

398 os.makedirs(apio_ctx.env_build_path, exist_ok=True) 

399 with open(params_file_path, "w", encoding="utf8") as f: 

400 f.write(text_format.MessageToString(scons_params)) 

401 

402 if util.is_debug(1): 402 ↛ 403line 402 didn't jump to line 403 because the condition on line 402 was never true

403 cout(f"\nFull scons command: {cmd}\n\n") 

404 

405 # -- Execute the scons builder! 

406 result = util.exec_command( 

407 cmd, 

408 stdout=util.AsyncPipe(scons_filter.on_stdout_line), 

409 stderr=util.AsyncPipe(scons_filter.on_stderr_line), 

410 ) 

411 

412 # -- Is there an error? True/False 

413 is_error = result.exit_code != 0 

414 

415 # -- Calculate the time it took to execute the command 

416 duration = time.time() - start_time 

417 

418 # -- Determine status message 

419 if is_error: 419 ↛ 420line 419 didn't jump to line 420 because the condition on line 419 was never true

420 styled_status = cstyle("ERROR", style=ERROR) 

421 else: 

422 styled_status = cstyle("SUCCESS", style=SUCCESS) 

423 

424 # -- Determine the summary text 

425 summary = f"Took {duration:.2f} seconds" 

426 

427 # -- Construct the entire message. 

428 styled_msg = f" [{styled_status}] {summary} " 

429 msg_len = len(cunstyle(styled_msg)) 

430 

431 # -- Determine the lengths of the paddings before and after 

432 # -- the message. Should be correct for odd and even terminal 

433 # -- widths. 

434 pad1_len = (terminal_width - msg_len) // 2 

435 pad2_len = terminal_width - pad1_len - msg_len 

436 

437 # -- Print the entire line. 

438 cout(f"{'=' * pad1_len}{styled_msg}{'=' * pad2_len}") 

439 

440 # -- Return the exit code 

441 return result.exit_code