diff --git a/slpkg/downloader.py b/slpkg/downloader.py index 70eb3658..d10c6efd 100644 --- a/slpkg/downloader.py +++ b/slpkg/downloader.py @@ -14,11 +14,11 @@ from slpkg.progress_bar import ProgressBar class Downloader(Configs, Utilities): - def __init__(self, path: Union[str, Path], url: str, flags: list): + def __init__(self, path: Union[str, Path], urls: list, flags: list): super(Configs, self).__init__() super(Utilities, self).__init__() self.path = path - self.url: str = url + self.urls: list = urls self.flags: list = flags self.color = self.colour() @@ -28,7 +28,6 @@ class Downloader(Configs, Utilities): self.stderr = None self.stdout = None - self.filename: str = url.split('/')[-1] self.bold: str = self.color['bold'] self.green: str = self.color['green'] self.yellow = self.color['yellow'] @@ -39,17 +38,37 @@ class Downloader(Configs, Utilities): self.bred: str = f'{self.bold}{self.red}' self.flag_no_silent: list = ['-n', '--no-silent'] - def transfer_tools(self) -> None: + def download(self): + process: list = [] + for url in self.urls: + + filename: str = url.split('/')[-1] + print(f'[{self.green}Downloading{self.endc}] {filename} ', flush=True) + + dwn = Process(target=self.tools, args=(url,)) + process.append(dwn) + dwn.start() + + for proc in process: + proc.join() + + def tools(self, url) -> None: """ Downloader tools wget, curl and lftp. """ + filename: str = url.split('/')[-1] + + if self.silent_mode and not self.is_option(self.flag_no_silent, self.flags): + self.stderr = subprocess.DEVNULL + self.stdout = subprocess.DEVNULL + if self.downloader == 'wget': self.output = subprocess.call(f'{self.downloader} {self.wget_options} --directory-prefix={self.path} ' - f'"{self.url}"', shell=True, stderr=self.stderr, stdout=self.stdout) + f'"{url}"', shell=True, stderr=self.stderr, stdout=self.stdout) elif self.downloader == 'curl': - self.output = subprocess.call(f'{self.downloader} {self.curl_options} "{self.url}" --output ' - f'{self.path}/{self.filename}', shell=True, stderr=self.stderr, + self.output = subprocess.call(f'{self.downloader} {self.curl_options} "{url}" --output ' + f'{self.path}/{filename}', shell=True, stderr=self.stderr, stdout=self.stdout) elif self.downloader == 'lftp': - self.output = subprocess.call(f'lftp {self.lftp_get_options} {self.url} -o {self.path}', + self.output = subprocess.call(f'lftp {self.lftp_get_options} {url} -o {self.path}', shell=True, stderr=self.stderr, stdout=self.stdout) else: raise SystemExit(f"{self.red}Error:{self.endc} Downloader '{self.downloader}' not supported.\n") @@ -57,52 +76,13 @@ class Downloader(Configs, Utilities): if self.output != 0: raise SystemExit(self.output) - def check_if_downloaded(self) -> None: + self.check_if_downloaded(url) + + def check_if_downloaded(self, url) -> None: """ Checks if the file downloaded. """ - url: str = unquote(self.url) + url: str = unquote(url) file: str = url.split('/')[-1] path_file = Path(self.path, file) - if not path_file.exists(): - raise SystemExit(f"\n{self.red}FAILED {self.output}:{self.endc} '{self.blue}{self.url}{self.endc}' " + raise SystemExit(f"\n{self.red}FAILED {self.output}:{self.endc} '{self.blue}{url}{self.endc}' " f"to download.\n") - - def download(self) -> None: - """ Starting multiprocessing download process. """ - if self.silent_mode and not self.is_option(self.flag_no_silent, self.flags): - - done: str = f' {self.byellow} Done{self.endc}' - self.stderr = subprocess.DEVNULL - self.stdout = subprocess.DEVNULL - - message: str = f'[{self.green}Downloading{self.endc}]' - - # Starting multiprocessing - p1 = Process(target=self.transfer_tools) - p2 = Process(target=self.progress.bar, args=(message, self.filename)) - - p1.start() - p2.start() - - # Wait until process 1 finish - p1.join() - - # Terminate process 2 if process 1 finished - if not p1.is_alive(): - - if p1.exitcode != 0: - done: str = f'{self.bred:>10} Failed{self.endc}' - self.output: int = p1.exitcode - - print(f'{self.endc}{done}', end='') - p2.terminate() - - # Wait until process 2 finish - p2.join() - - # Restore the terminal cursor - print('\x1b[?25h', self.endc) - else: - self.transfer_tools() - - self.check_if_downloaded() diff --git a/slpkg/slackbuild.py b/slpkg/slackbuild.py index 91f1b645..1ad13721 100644 --- a/slpkg/slackbuild.py +++ b/slpkg/slackbuild.py @@ -132,6 +132,8 @@ class Slackbuilds(Configs): def download_slackbuilds(self) -> None: """ Downloads files and sources. """ + sbos_urls: list = [] + for sbo in self.install_order: if self.is_for_skipped(sbo): @@ -159,15 +161,33 @@ class Slackbuilds(Configs): continue else: sbo_url: str = f'{self.sbo_repo_url}{location}/{file}' - down_sbo = Downloader(self.tmp_slpkg, sbo_url, self.flags) - down_sbo.download() + sbos_urls.append(sbo_url) - self.utils.untar_archive(self.tmp_slpkg, file, self.build_path) + down_sbo = Downloader(self.build_path, sbos_urls, self.flags) + down_sbo.download() - self.patch_sbo_tag(sbo) + for sbo in self.install_order: + file: str = f'{sbo}{self.sbo_tar_suffix}' + self.utils.untar_archive(self.build_path, file) + self.patch_sbo_tag(sbo) - sources = SBoQueries(sbo).sources() - self.download_sources(sbo, sources) + self.download_sources() + + def download_sources(self) -> None: + """ Download the sources. """ + for sbo in self.install_order: + sources = SBoQueries(sbo).sources() + checksums = SBoQueries(sbo).checksum() + + path = Path(self.build_path, sbo) + + down_source = Downloader(path, sources, self.flags) + down_source.download() + + for source, checksum in zip(sources, checksums): + + md5sum = Md5sum(self.flags) + md5sum.check(path, source, checksum, sbo) def build_and_install(self) -> None: """ Build the slackbuilds and install. """ @@ -278,18 +298,6 @@ class Slackbuilds(Configs): """ Set number of processors. """ os.environ['MAKEFLAGS'] = f'-j {cpu_count()}' - def download_sources(self, name: str, sources: list) -> None: - """ Download the sources. """ - path = Path(self.build_path, name) - checksums: list = SBoQueries(name).checksum() - - for source, checksum in zip(sources, checksums): - down_source = Downloader(path, source, self.flags) - down_source.download() - - md5sum = Md5sum(self.flags) - md5sum.check(path, source, checksum, name) - def multi_process(self, command: str, filename: str, message: str) -> None: """ Starting multiprocessing install/upgrade process. """ if self.silent_mode and not self.utils.is_option(self.flag_no_silent, self.flags): diff --git a/slpkg/utilities.py b/slpkg/utilities.py index 9f08404d..e2bec6cf 100644 --- a/slpkg/utilities.py +++ b/slpkg/utilities.py @@ -59,11 +59,11 @@ class Utilities: yield self.split_installed_pkg(file.name)[0] @staticmethod - def untar_archive(path: str, archive: str, ext_path: str) -> None: + def untar_archive(path: str, archive: str) -> None: """ Untar the file to the build folder. """ tar_file = Path(path, archive) untar = tarfile.open(tar_file) - untar.extractall(ext_path) + untar.extractall(path) untar.close() @staticmethod