Coverage for apio/managers/downloader.py: 93%

1# -*- coding: utf-8 -*- 

2# -- This file is part of the Apio project 

3# -- (C) 2016-2019 FPGAwars 

4# -- Author Jesús Arroyo 

5# -- License GPLv2 

6# -- Derived from: 

7# ---- Platformio project 

8# ---- (C) 2014-2016 Ivan Kravets <me@ikravets.com> 

9# ---- License Apache v2 

10"""Implement a remote file downloader. Used to fetch packages from github 

11packages release repositories. 

12""" 

13 

14# TODO: capture all the exceptions and return them as method return status. 

15# Motivation is simplifying the usage. 

16 

17from math import ceil 

18from pathlib import Path 

19import requests 

20from rich.progress import track 

21from apio.utils import util 

22from apio.common.apio_console import cout, console 

23from apio.common.apio_styles import ERROR 

24 

25 

26# -- Timeout for getting a response from the server when downloading 

27# -- a file (in seconds). We had github tests failing with timeout=10 

28TIMEOUT_SECS = 30 

29 

30 

31class FileDownloader: 

32 """Class for downloading files""" 

33 

34 CHUNK_SIZE = 1024 

35 

36 def __init__(self, url: str, dest_dir=None): 

37 """Initialize a FileDownloader object 

38 * INPUTs: 

39 * url: File to download (full url) 

40 (Ex. 'https://github.com/FPGAwars/apio-examples/ 

41 releases/download/0.0.35/apio-examples-0.0.35.zip') 

42 * dest_dir: Destination folder (where to download the file) 

43 """ 

44 

45 # -- Initialize the request field first, so that __del__ works even 

46 # -- if the construction fails below, e.g. on a connection timeout. 

47 self._request = None 

48 

49 # -- Store the url 

50 self._url = url 

51 

52 # -- Get the file from the url 

53 # -- Ex: 'apio-examples-0.0.35.zip' 

54 self.fname = url.split("/")[-1] 

55 

56 # -- Build the destination path 

57 self.destination: Path = Path(self.fname) 

58 if dest_dir: 

59 

60 # -- Add the path 

61 self.destination = dest_dir / self.fname 

62 

63 # -- Request the file 

64 self._request = requests.get(url, stream=True, timeout=TIMEOUT_SECS) 

65 

66 # -- Raise an exception in case of download error... 

67 if self._request.status_code != 200: 67 ↛ 68line 67 didn't jump to line 68 because the condition on line 67 was never true

68 cout( 

69 "Got an unexpected HTTP status code: " 

70 f"{self._request.status_code}", 

71 f"When downloading {url}", 

72 style=ERROR, 

73 ) 

74 raise util.ApioException() 

75 

76 def get_size(self) -> int: 

77 """Return the size (in bytes) of the latest bytes block received""" 

78 

79 return int(self._request.headers["content-length"]) 

80 

81 def start(self): 

82 """Start the downloading of the file""" 

83 

84 # -- Download iterator 

85 itercontent = self._request.iter_content(chunk_size=self.CHUNK_SIZE) 

86 

87 # -- Open destination file, for writing bytes 

88 with open(self.destination, "wb") as file: 

89 

90 # -- Get the file length in Kbytes 

91 num_chunks = int(ceil(self.get_size() / float(self.CHUNK_SIZE))) 

92 

93 # -- Download and write the chunks, while displaying the progress. 

94 for _ in track( 

95 range(num_chunks), 

96 description="Downloading", 

97 console=console(), 

98 ): 

99 

100 file.write(next(itercontent)) 

101 

102 # -- Check that the iterator reached its end. When the end is 

103 # -- reached, next() returns the default value None. 

104 assert next(itercontent, None) is None 

105 

106 # -- Download done! 

107 self._request.close() 

108 

109 def __del__(self): 

110 """Close any pending request""" 

111 

112 # -- Using getattr() since __del__ can run on a partially constructed 

113 # -- object, and an exception here would surface as an 'unraisable 

114 # -- exception' at an arbitrary point of the program. Comparing to 

115 # -- None rather than testing truthiness since bool(Response) is 

116 # -- Response.ok, which is False for non-2xx error responses. 

117 request = getattr(self, "_request", None) 

118 if request is not None: 

119 request.close()