Updated for multiprocess download

slpkg/downloader.py

@@ -14,11 +14,11 @@ from slpkg.progress_bar import ProgressBar
 
 class Downloader(Configs, Utilities):
 
-    def __init__(self, path: Union[str, Path], url: str, flags: list):
+    def __init__(self, path: Union[str, Path], urls: list, flags: list):
         super(Configs, self).__init__()
         super(Utilities, self).__init__()
         self.path = path
-        self.url: str = url
+        self.urls: list = urls
         self.flags: list = flags
 
         self.color = self.colour()
@@ -28,7 +28,6 @@ class Downloader(Configs, Utilities):
         self.stderr = None
         self.stdout = None
 
-        self.filename: str = url.split('/')[-1]
         self.bold: str = self.color['bold']
         self.green: str = self.color['green']
         self.yellow = self.color['yellow']
@@ -39,17 +38,37 @@ class Downloader(Configs, Utilities):
         self.bred: str = f'{self.bold}{self.red}'
         self.flag_no_silent: list = ['-n', '--no-silent']
 
-    def transfer_tools(self) -> None:
+    def download(self):
+        process: list = []
+        for url in self.urls:
+
+            filename: str = url.split('/')[-1]
+            print(f'[{self.green}Downloading{self.endc}] {filename} ', flush=True)
+
+            dwn = Process(target=self.tools, args=(url,))
+            process.append(dwn)
+            dwn.start()
+
+        for proc in process:
+            proc.join()
+
+    def tools(self, url) -> None:
         """ Downloader tools wget, curl and lftp. """
+        filename: str = url.split('/')[-1]
+
+        if self.silent_mode and not self.is_option(self.flag_no_silent, self.flags):
+            self.stderr = subprocess.DEVNULL
+            self.stdout = subprocess.DEVNULL
+
         if self.downloader == 'wget':
             self.output = subprocess.call(f'{self.downloader} {self.wget_options} --directory-prefix={self.path} '
-                                          f'"{self.url}"', shell=True, stderr=self.stderr, stdout=self.stdout)
+                                          f'"{url}"', shell=True, stderr=self.stderr, stdout=self.stdout)
         elif self.downloader == 'curl':
-            self.output = subprocess.call(f'{self.downloader} {self.curl_options} "{self.url}" --output '
+            self.output = subprocess.call(f'{self.downloader} {self.curl_options} "{url}" --output '
-                                          f'{self.path}/{self.filename}', shell=True, stderr=self.stderr,
+                                          f'{self.path}/{filename}', shell=True, stderr=self.stderr,
                                           stdout=self.stdout)
         elif self.downloader == 'lftp':
-            self.output = subprocess.call(f'lftp {self.lftp_get_options} {self.url} -o {self.path}',
+            self.output = subprocess.call(f'lftp {self.lftp_get_options} {url} -o {self.path}',
                                           shell=True, stderr=self.stderr, stdout=self.stdout)
         else:
             raise SystemExit(f"{self.red}Error:{self.endc} Downloader '{self.downloader}' not supported.\n")
@@ -57,52 +76,13 @@ class Downloader(Configs, Utilities):
         if self.output != 0:
             raise SystemExit(self.output)
 
-    def check_if_downloaded(self) -> None:
+        self.check_if_downloaded(url)
+
+    def check_if_downloaded(self, url) -> None:
         """ Checks if the file downloaded. """
-        url: str = unquote(self.url)
+        url: str = unquote(url)
         file: str = url.split('/')[-1]
         path_file = Path(self.path, file)
-
         if not path_file.exists():
-            raise SystemExit(f"\n{self.red}FAILED {self.output}:{self.endc} '{self.blue}{self.url}{self.endc}' "
+            raise SystemExit(f"\n{self.red}FAILED {self.output}:{self.endc} '{self.blue}{url}{self.endc}' "
                              f"to download.\n")
-
-    def download(self) -> None:
-        """ Starting multiprocessing download process. """
-        if self.silent_mode and not self.is_option(self.flag_no_silent, self.flags):
-
-            done: str = f' {self.byellow} Done{self.endc}'
-            self.stderr = subprocess.DEVNULL
-            self.stdout = subprocess.DEVNULL
-
-            message: str = f'[{self.green}Downloading{self.endc}]'
-
-            # Starting multiprocessing
-            p1 = Process(target=self.transfer_tools)
-            p2 = Process(target=self.progress.bar, args=(message, self.filename))
-
-            p1.start()
-            p2.start()
-
-            # Wait until process 1 finish
-            p1.join()
-
-            # Terminate process 2 if process 1 finished
-            if not p1.is_alive():
-
-                if p1.exitcode != 0:
-                    done: str = f'{self.bred:>10} Failed{self.endc}'
-                    self.output: int = p1.exitcode
-
-                print(f'{self.endc}{done}', end='')
-                p2.terminate()
-
-            # Wait until process 2 finish
-            p2.join()
-
-            # Restore the terminal cursor
-            print('\x1b[?25h', self.endc)
-        else:
-            self.transfer_tools()
-
-        self.check_if_downloaded()

slpkg/slackbuild.py

@@ -132,6 +132,8 @@ class Slackbuilds(Configs):
 
     def download_slackbuilds(self) -> None:
         """ Downloads files and sources. """
+        sbos_urls: list = []
+
         for sbo in self.install_order:
 
             if self.is_for_skipped(sbo):
@@ -159,15 +161,33 @@ class Slackbuilds(Configs):
                         continue
                 else:
                     sbo_url: str = f'{self.sbo_repo_url}{location}/{file}'
-                    down_sbo = Downloader(self.tmp_slpkg, sbo_url, self.flags)
+                    sbos_urls.append(sbo_url)
-                    down_sbo.download()
 
-                    self.utils.untar_archive(self.tmp_slpkg, file, self.build_path)
+        down_sbo = Downloader(self.build_path, sbos_urls, self.flags)
+        down_sbo.download()
 
-                self.patch_sbo_tag(sbo)
+        for sbo in self.install_order:
+            file: str = f'{sbo}{self.sbo_tar_suffix}'
+            self.utils.untar_archive(self.build_path, file)
+            self.patch_sbo_tag(sbo)
 
-                sources = SBoQueries(sbo).sources()
+        self.download_sources()
-                self.download_sources(sbo, sources)
+
+    def download_sources(self) -> None:
+        """ Download the sources. """
+        for sbo in self.install_order:
+            sources = SBoQueries(sbo).sources()
+            checksums = SBoQueries(sbo).checksum()
+
+            path = Path(self.build_path, sbo)
+
+            down_source = Downloader(path, sources, self.flags)
+            down_source.download()
+
+            for source, checksum in zip(sources, checksums):
+
+                md5sum = Md5sum(self.flags)
+                md5sum.check(path, source, checksum, sbo)
 
     def build_and_install(self) -> None:
         """ Build the slackbuilds and install. """
@@ -278,18 +298,6 @@ class Slackbuilds(Configs):
         """ Set number of processors. """
         os.environ['MAKEFLAGS'] = f'-j {cpu_count()}'
 
-    def download_sources(self, name: str, sources: list) -> None:
-        """ Download the sources. """
-        path = Path(self.build_path, name)
-        checksums: list = SBoQueries(name).checksum()
-
-        for source, checksum in zip(sources, checksums):
-            down_source = Downloader(path, source, self.flags)
-            down_source.download()
-
-            md5sum = Md5sum(self.flags)
-            md5sum.check(path, source, checksum, name)
-
     def multi_process(self, command: str, filename: str, message: str) -> None:
         """ Starting multiprocessing install/upgrade process. """
         if self.silent_mode and not self.utils.is_option(self.flag_no_silent, self.flags):

slpkg/utilities.py

@@ -59,11 +59,11 @@ class Utilities:
                 yield self.split_installed_pkg(file.name)[0]
 
     @staticmethod
-    def untar_archive(path: str, archive: str, ext_path: str) -> None:
+    def untar_archive(path: str, archive: str) -> None:
         """ Untar the file to the build folder. """
         tar_file = Path(path, archive)
         untar = tarfile.open(tar_file)
-        untar.extractall(ext_path)
+        untar.extractall(path)
         untar.close()
 
     @staticmethod