Coverage for apio/managers/scons_manager.py: 80%

1"""A manager class for to dispatch the Apio SCONS targets.""" 

2 

3# -*- coding: utf-8 -*- 

4# -- This file is part of the Apio project 

5# -- (C) 2016-2019 FPGAwars 

6# -- Author Jesús Arroyo 

7# -- License GPLv2 

8 

9import traceback 

10import os 

11import sys 

12import time 

13import shutil 

14from functools import wraps 

15from datetime import datetime 

16from google.protobuf import text_format 

17from apio.common import apio_console 

18from apio.common.apio_console import cout, cerror, cstyle, cunstyle 

19from apio.common.apio_styles import SUCCESS, ERROR, EMPH3, INFO 

20from apio.utils import util 

21from apio.apio_context import ApioContext 

22from apio.managers.scons_filter import SconsFilter 

23from apio.common.proto.apio_pb2 import ( 

24 FORCE_PIPE, 

25 FORCE_TERMINAL, 

26 Verbosity, 

27 Environment, 

28 SconsParams, 

29 TargetParams, 

30 FpgaInfo, 

31 ApioEnvParams, 

32 Ice40FpgaInfo, 

33 Ecp5FpgaInfo, 

34 GowinFpgaInfo, 

35 ApioArch, 

36 GraphParams, 

37 LintParams, 

38 SimParams, 

39 ApioTestParams, 

40 UploadParams, 

41) 

42 

43# from apio.common import rich_lib_windows 

44 

45 

46# W0703: Catching too general exception Exception (broad-except) 

47# pylint: disable=W0703 

48# 

49# -- Based on 

50# -- https://stackoverflow.com/questions/5929107/decorators-with-parameters 

51def on_exception(*, exit_code: int): 

52 """Decorator for functions that return int exit code. If the function 

53 throws an exception, the error message is printed, and the caller see the 

54 returned value exit_code instead of the exception. 

55 """ 

56 

57 def decorator(function): 

58 @wraps(function) 

59 def wrapper(*args, **kwargs): 

60 try: 

61 return function(*args, **kwargs) 

62 except Exception as exc: 

63 if util.is_debug(1): 

64 traceback.print_tb(exc.__traceback__) 

65 

66 if str(exc): 

67 cerror(str(exc)) 

68 

69 if not util.is_debug(1): 

70 cout( 

71 "Running with APIO_DEBUG=1 may provide " 

72 "additional diagnostic information.", 

73 style=INFO, 

74 ) 

75 return exit_code 

76 

77 return wrapper 

78 

79 return decorator 

80 

81 

82class SConsManager: 

83 """Class for managing the scons tools""" 

84 

85 def __init__(self, apio_ctx: ApioContext): 

86 """Initialization.""" 

87 # -- Cache the apio context. 

88 self.apio_ctx = apio_ctx 

89 

90 # -- Change to the project's folder. 

91 os.chdir(apio_ctx.project_dir) 

92 

93 @on_exception(exit_code=1) 

94 def graph(self, graph_params: GraphParams, verbosity: Verbosity) -> int: 

95 """Runs a scons subprocess with the 'graph' target. Returns process 

96 exit code, 0 if ok.""" 

97 

98 # -- Construct scons params with graph command info. 

99 scons_params = self.construct_scons_params( 

100 target_params=TargetParams(graph=graph_params), 

101 verbosity=verbosity, 

102 ) 

103 

104 # -- Run the scons process. 

105 return self._run_scons_subprocess("graph", scons_params=scons_params) 

106 

107 @on_exception(exit_code=1) 

108 def lint(self, lint_params: LintParams) -> int: 

109 """Runs a scons subprocess with the 'lint' target. Returns process 

110 exit code, 0 if ok.""" 

111 

112 # -- Construct scons params with graph command info. 

113 scons_params = self.construct_scons_params( 

114 target_params=TargetParams(lint=lint_params) 

115 ) 

116 

117 # -- Run the scons process. 

118 return self._run_scons_subprocess("lint", scons_params=scons_params) 

119 

120 @on_exception(exit_code=1) 

121 def sim(self, sim_params: SimParams) -> int: 

122 """Runs a scons subprocess with the 'sim' target. Returns process 

123 exit code, 0 if ok.""" 

124 

125 # -- Construct scons params with graph command info. 

126 scons_params = self.construct_scons_params( 

127 target_params=TargetParams(sim=sim_params) 

128 ) 

129 

130 # -- Run the scons process. 

131 return self._run_scons_subprocess("sim", scons_params=scons_params) 

132 

133 @on_exception(exit_code=1) 

134 def test(self, test_params: ApioTestParams) -> int: 

135 """Runs a scons subprocess with the 'test' target. Returns process 

136 exit code, 0 if ok.""" 

137 

138 # -- Construct scons params with graph command info. 

139 scons_params = self.construct_scons_params( 

140 target_params=TargetParams(test=test_params) 

141 ) 

142 

143 # -- Run the scons process. 

144 return self._run_scons_subprocess("test", scons_params=scons_params) 

145 

146 @on_exception(exit_code=1) 

147 def build(self, verbosity: Verbosity) -> int: 

148 """Runs a scons subprocess with the 'build' target. Returns process 

149 exit code, 0 if ok.""" 

150 

151 # -- Construct the scons params object. 

152 scons_params = self.construct_scons_params(verbosity=verbosity) 

153 

154 # -- Run the scons process. 

155 return self._run_scons_subprocess("build", scons_params=scons_params) 

156 

157 @on_exception(exit_code=1) 

158 def report(self, verbosity: Verbosity) -> int: 

159 """Runs a scons subprocess with the 'report' target. Returns process 

160 exit code, 0 if ok.""" 

161 

162 # -- Construct the scons params object. 

163 scons_params = self.construct_scons_params(verbosity=verbosity) 

164 

165 # -- Run the scons process. 

166 return self._run_scons_subprocess("report", scons_params=scons_params) 

167 

168 @on_exception(exit_code=1) 

169 def upload(self, upload_params: UploadParams) -> int: 

170 """Runs a scons subprocess with the 'time' target. Returns process 

171 exit code, 0 if ok. 

172 """ 

173 

174 # -- Construct the scons params. 

175 scons_params = self.construct_scons_params( 

176 target_params=TargetParams(upload=upload_params) 

177 ) 

178 

179 # -- Execute Scons for uploading! 

180 exit_code = self._run_scons_subprocess( 

181 "upload", scons_params=scons_params 

182 ) 

183 

184 return exit_code 

185 

186 def construct_scons_params( 

187 self, 

188 *, 

189 target_params: TargetParams = None, 

190 verbosity: Verbosity = None, 

191 ) -> SconsParams: 

192 """Populate and return the SconsParam proto to pass to the scons 

193 process.""" 

194 

195 # -- Create a shortcut. 

196 apio_ctx = self.apio_ctx 

197 

198 # -- Create an empty proto object that will be populated. 

199 result = SconsParams() 

200 

201 # -- Populate the timestamp. We use to to make sure scons reads the 

202 # -- correct version of the scons.params file. 

203 ts = datetime.now() 

204 result.timestamp = ts.strftime("%d%H%M%S%f")[:-3] 

205 

206 # -- Get the project data. All commands that invoke scons are expected 

207 # -- to be in a project context. 

208 assert apio_ctx.has_project, "Scons encountered a missing project." 

209 project = apio_ctx.project 

210 

211 # -- Get the project resource.s 

212 pr = apio_ctx.project_resources 

213 

214 # -- Populate the common values of FpgaInfo. 

215 result.fpga_info.MergeFrom( 

216 FpgaInfo( 

217 fpga_id=pr.fpga_id, 

218 part_num=pr.fpga_info["part-num"], 

219 size=pr.fpga_info["size"], 

220 ) 

221 ) 

222 

223 # - Populate the architecture specific values of result.fpga_info. 

224 fpga_arch = pr.fpga_info["arch"] 

225 match fpga_arch: 

226 case "ice40": 

227 result.arch = ApioArch.ICE40 

228 result.fpga_info.ice40.MergeFrom( 

229 Ice40FpgaInfo( 

230 type=pr.fpga_info["type"], pack=pr.fpga_info["pack"] 

231 ) 

232 ) 

233 case "ecp5": 

234 result.arch = ApioArch.ECP5 

235 result.fpga_info.ecp5.MergeFrom( 

236 Ecp5FpgaInfo( 

237 type=pr.fpga_info["type"], 

238 pack=pr.fpga_info["pack"], 

239 speed=pr.fpga_info["speed"], 

240 ) 

241 ) 

242 case "gowin": 242 ↛ 247line 242 didn't jump to line 247 because the pattern on line 242 always matched

243 result.arch = ApioArch.GOWIN 

244 result.fpga_info.gowin.MergeFrom( 

245 GowinFpgaInfo(family=pr.fpga_info["type"]) 

246 ) 

247 case _: 

248 cerror(f"Unexpected fpga_arch value {fpga_arch}") 

249 sys.exit(1) 

250 

251 # -- We are done populating The FpgaInfo params.. 

252 assert result.fpga_info.IsInitialized(), result 

253 

254 # -- Populate the optional Verbosity params. 

255 if verbosity: 

256 result.verbosity.MergeFrom(verbosity) 

257 assert result.verbosity.IsInitialized(), result 

258 

259 # -- Populate the Environment params. 

260 assert apio_ctx.platform_id, "Missing platform_id in apio context" 

261 oss_vars = apio_ctx.all_packages["oss-cad-suite"]["env"]["vars"] 

262 

263 result.environment.MergeFrom( 

264 Environment( 

265 platform_id=apio_ctx.platform_id, 

266 is_windows=apio_ctx.is_windows, 

267 terminal_mode=( 

268 FORCE_TERMINAL 

269 if apio_console.is_terminal() 

270 else FORCE_PIPE 

271 ), 

272 theme_name=apio_console.current_theme_name(), 

273 debug_level=util.debug_level(), 

274 yosys_path=oss_vars["YOSYS_LIB"], 

275 trellis_path=oss_vars["TRELLIS"], 

276 ) 

277 ) 

278 assert result.environment.IsInitialized(), result 

279 

280 # -- Populate the Project params. 

281 result.apio_env_params.MergeFrom( 

282 ApioEnvParams( 

283 env_name=apio_ctx.project.env_name, 

284 board_id=pr.board_id, 

285 top_module=project.get_str_option("top-module"), 

286 defines=apio_ctx.project.get_list_option( 

287 "defines", default=[] 

288 ), 

289 yosys_synth_extra_options=apio_ctx.project.get_list_option( 

290 "yosys-synth-extra-options", None 

291 ), 

292 nextpnr_extra_options=apio_ctx.project.get_list_option( 

293 "nextpnr-extra-options", None 

294 ), 

295 constraint_file=apio_ctx.project.get_str_option( 

296 "constraint-file", None 

297 ), 

298 ) 

299 ) 

300 assert result.apio_env_params.IsInitialized(), result 

301 

302 # -- Populate the optional command specific params. 

303 if target_params: 

304 result.target.MergeFrom(target_params) 

305 assert result.target.IsInitialized(), result 

306 

307 # -- All done. 

308 assert result.IsInitialized(), result 

309 return result 

310 

311 def _run_scons_subprocess( 

312 self, scons_command: str, *, scons_params: SconsParams = None 

313 ): 

314 """Invoke an scons subprocess.""" 

315 

316 # pylint: disable=too-many-locals 

317 

318 # -- Create a shortcut. 

319 apio_ctx = self.apio_ctx 

320 

321 # -- Pass to the scons process the name of the sconstruct file it 

322 # -- should use. 

323 scons_dir = util.get_path_in_apio_package("scons") 

324 scons_file_path = scons_dir / "SConstruct" 

325 variables = ["-f", f"{scons_file_path}"] 

326 

327 # -- Pass the path to the proto params file. The path is relative 

328 # -- to the project root. 

329 params_file_path = apio_ctx.env_build_path / "scons.params" 

330 variables += [f"params={str(params_file_path)}"] 

331 

332 # -- Pass to the scons process the timestamp of the scons params we 

333 # -- pass via a file. This is for verification purposes only. 

334 variables += [f"timestamp={scons_params.timestamp}"] 

335 

336 # -- We set the env variables also for a command such as 'clean' 

337 # -- which doesn't use the packages, to satisfy the required env 

338 # -- variables of the scons arg parser. 

339 apio_ctx.set_env_for_packages() 

340 

341 if util.is_debug(1): 341 ↛ 342line 341 didn't jump to line 342 because the condition on line 341 was never true

342 cout("\nSCONS CALL:", style=EMPH3) 

343 cout(f"* command: {scons_command}") 

344 cout(f"* variables: {variables}") 

345 cout(f"* scons params: \n{scons_params}") 

346 cout() 

347 

348 # -- Get the terminal width (typically 80) 

349 terminal_width, _ = shutil.get_terminal_size() 

350 

351 # -- Read the time (for measuring how long does it take 

352 # -- to execute the apio command) 

353 start_time = time.time() 

354 

355 # -- Subtracting 1 to avoid line overflow on windows, Observed with 

356 # -- Windows 10 and cmd.exe shell. 

357 if apio_ctx.is_windows: 357 ↛ 358line 357 didn't jump to line 358 because the condition on line 357 was never true

358 terminal_width -= 1 

359 

360 # -- Print a horizontal line 

361 cout("-" * terminal_width) 

362 

363 # -- Create the scons debug options. See details at 

364 # -- https://scons.org/doc/2.4.1/HTML/scons-man.html 

365 debug_options = ( 

366 ["--debug=explain,prepare,stacktrace", "--tree=all"] 

367 if util.is_debug(1) 

368 else [] 

369 ) 

370 

371 # -- Construct the scons command line. 

372 # -- 

373 # -- sys.executable is resolved to the full path of the python 

374 # -- interpreter or to apio if running from a pyinstall setup. 

375 # -- See https://github.com/orgs/pyinstaller/discussions/9023 for more 

376 # -- information. 

377 # -- 

378 cmd = ( 

379 [sys.executable, "-m", "SCons"] 

380 + ["-Q", scons_command] 

381 + debug_options 

382 + variables 

383 ) 

384 

385 # -- An output filter that manipulates the scons stdout/err lines as 

386 # -- needed and write them to stdout. 

387 scons_filter = SconsFilter( 

388 colors_enabled=apio_console.is_colors_enabled() 

389 ) 

390 

391 # -- Write the scons parameters to a temp file in the build 

392 # -- directory. It will be cleaned up as part of 'apio cleanup'. 

393 # -- At this point, the project is the current directory, even if 

394 # -- the command used the --project-dir option. 

395 os.makedirs(apio_ctx.env_build_path, exist_ok=True) 

396 with open(params_file_path, "w", encoding="utf8") as f: 

397 f.write(text_format.MessageToString(scons_params)) 

398 

399 if util.is_debug(1): 399 ↛ 400line 399 didn't jump to line 400 because the condition on line 399 was never true

400 cout(f"\nFull scons command: {cmd}\n\n") 

401 

402 # -- Execute the scons builder! 

403 result = util.exec_command( 

404 cmd, 

405 stdout=util.AsyncPipe(scons_filter.on_stdout_line), 

406 stderr=util.AsyncPipe(scons_filter.on_stderr_line), 

407 ) 

408 

409 # -- Is there an error? True/False 

410 is_error = result.exit_code != 0 

411 

412 # -- Calculate the time it took to execute the command 

413 duration = time.time() - start_time 

414 

415 # -- Determine status message 

416 if is_error: 416 ↛ 417line 416 didn't jump to line 417 because the condition on line 416 was never true

417 styled_status = cstyle("ERROR", style=ERROR) 

418 else: 

419 styled_status = cstyle("SUCCESS", style=SUCCESS) 

420 

421 # -- Determine the summary text 

422 summary = f"Took {duration:.2f} seconds" 

423 

424 # -- Construct the entire message. 

425 styled_msg = f" [{styled_status}] {summary} " 

426 msg_len = len(cunstyle(styled_msg)) 

427 

428 # -- Determine the lengths of the paddings before and after 

429 # -- the message. Should be correct for odd and even terminal 

430 # -- widths. 

431 pad1_len = (terminal_width - msg_len) // 2 

432 pad2_len = terminal_width - pad1_len - msg_len 

433 

434 # -- Print the entire line. 

435 cout(f"{'=' * pad1_len}{styled_msg}{'=' * pad2_len}") 

436 

437 # -- Return the exit code 

438 return result.exit_code