mirror of
https://gitlab.com/dslackw/slpkg.git
synced 2024-12-31 10:26:39 +01:00
Updated for multiprocess download
This commit is contained in:
Dimitris Zlatanidis
parent
9ee67614fd
commit
616e25c92f
3 changed files with 60 additions and 72 deletions
|
|
@ -14,11 +14,11 @@ from slpkg.progress_bar import ProgressBar
|
|||
|
||||
class Downloader(Configs, Utilities):
|
||||
|
||||
def __init__(self, path: Union[str, Path], url: str, flags: list):
|
||||
def __init__(self, path: Union[str, Path], urls: list, flags: list):
|
||||
super(Configs, self).__init__()
|
||||
super(Utilities, self).__init__()
|
||||
self.path = path
|
||||
self.url: str = url
|
||||
self.urls: list = urls
|
||||
self.flags: list = flags
|
||||
|
||||
self.color = self.colour()
|
||||
|
|
@ -28,7 +28,6 @@ class Downloader(Configs, Utilities):
|
|||
self.stderr = None
|
||||
self.stdout = None
|
||||
|
||||
self.filename: str = url.split('/')[-1]
|
||||
self.bold: str = self.color['bold']
|
||||
self.green: str = self.color['green']
|
||||
self.yellow = self.color['yellow']
|
||||
|
|
@ -39,17 +38,37 @@ class Downloader(Configs, Utilities):
|
|||
self.bred: str = f'{self.bold}{self.red}'
|
||||
self.flag_no_silent: list = ['-n', '--no-silent']
|
||||
|
||||
def transfer_tools(self) -> None:
|
||||
def download(self):
|
||||
process: list = []
|
||||
for url in self.urls:
|
||||
|
||||
filename: str = url.split('/')[-1]
|
||||
print(f'[{self.green}Downloading{self.endc}] {filename} ', flush=True)
|
||||
|
||||
dwn = Process(target=self.tools, args=(url,))
|
||||
process.append(dwn)
|
||||
dwn.start()
|
||||
|
||||
for proc in process:
|
||||
proc.join()
|
||||
|
||||
def tools(self, url) -> None:
|
||||
""" Downloader tools wget, curl and lftp. """
|
||||
filename: str = url.split('/')[-1]
|
||||
|
||||
if self.silent_mode and not self.is_option(self.flag_no_silent, self.flags):
|
||||
self.stderr = subprocess.DEVNULL
|
||||
self.stdout = subprocess.DEVNULL
|
||||
|
||||
if self.downloader == 'wget':
|
||||
self.output = subprocess.call(f'{self.downloader} {self.wget_options} --directory-prefix={self.path} '
|
||||
f'"{self.url}"', shell=True, stderr=self.stderr, stdout=self.stdout)
|
||||
f'"{url}"', shell=True, stderr=self.stderr, stdout=self.stdout)
|
||||
elif self.downloader == 'curl':
|
||||
self.output = subprocess.call(f'{self.downloader} {self.curl_options} "{self.url}" --output '
|
||||
f'{self.path}/{self.filename}', shell=True, stderr=self.stderr,
|
||||
self.output = subprocess.call(f'{self.downloader} {self.curl_options} "{url}" --output '
|
||||
f'{self.path}/{filename}', shell=True, stderr=self.stderr,
|
||||
stdout=self.stdout)
|
||||
elif self.downloader == 'lftp':
|
||||
self.output = subprocess.call(f'lftp {self.lftp_get_options} {self.url} -o {self.path}',
|
||||
self.output = subprocess.call(f'lftp {self.lftp_get_options} {url} -o {self.path}',
|
||||
shell=True, stderr=self.stderr, stdout=self.stdout)
|
||||
else:
|
||||
raise SystemExit(f"{self.red}Error:{self.endc} Downloader '{self.downloader}' not supported.\n")
|
||||
|
|
@ -57,52 +76,13 @@ class Downloader(Configs, Utilities):
|
|||
if self.output != 0:
|
||||
raise SystemExit(self.output)
|
||||
|
||||
def check_if_downloaded(self) -> None:
|
||||
self.check_if_downloaded(url)
|
||||
|
||||
def check_if_downloaded(self, url) -> None:
|
||||
""" Checks if the file downloaded. """
|
||||
url: str = unquote(self.url)
|
||||
url: str = unquote(url)
|
||||
file: str = url.split('/')[-1]
|
||||
path_file = Path(self.path, file)
|
||||
|
||||
if not path_file.exists():
|
||||
raise SystemExit(f"\n{self.red}FAILED {self.output}:{self.endc} '{self.blue}{self.url}{self.endc}' "
|
||||
raise SystemExit(f"\n{self.red}FAILED {self.output}:{self.endc} '{self.blue}{url}{self.endc}' "
|
||||
f"to download.\n")
|
||||
|
||||
def download(self) -> None:
|
||||
""" Starting multiprocessing download process. """
|
||||
if self.silent_mode and not self.is_option(self.flag_no_silent, self.flags):
|
||||
|
||||
done: str = f' {self.byellow} Done{self.endc}'
|
||||
self.stderr = subprocess.DEVNULL
|
||||
self.stdout = subprocess.DEVNULL
|
||||
|
||||
message: str = f'[{self.green}Downloading{self.endc}]'
|
||||
|
||||
# Starting multiprocessing
|
||||
p1 = Process(target=self.transfer_tools)
|
||||
p2 = Process(target=self.progress.bar, args=(message, self.filename))
|
||||
|
||||
p1.start()
|
||||
p2.start()
|
||||
|
||||
# Wait until process 1 finish
|
||||
p1.join()
|
||||
|
||||
# Terminate process 2 if process 1 finished
|
||||
if not p1.is_alive():
|
||||
|
||||
if p1.exitcode != 0:
|
||||
done: str = f'{self.bred:>10} Failed{self.endc}'
|
||||
self.output: int = p1.exitcode
|
||||
|
||||
print(f'{self.endc}{done}', end='')
|
||||
p2.terminate()
|
||||
|
||||
# Wait until process 2 finish
|
||||
p2.join()
|
||||
|
||||
# Restore the terminal cursor
|
||||
print('\x1b[?25h', self.endc)
|
||||
else:
|
||||
self.transfer_tools()
|
||||
|
||||
self.check_if_downloaded()
|
||||
|
|
|
|||
|
|
@ -132,6 +132,8 @@ class Slackbuilds(Configs):
|
|||
|
||||
def download_slackbuilds(self) -> None:
|
||||
""" Downloads files and sources. """
|
||||
sbos_urls: list = []
|
||||
|
||||
for sbo in self.install_order:
|
||||
|
||||
if self.is_for_skipped(sbo):
|
||||
|
|
@ -159,15 +161,33 @@ class Slackbuilds(Configs):
|
|||
continue
|
||||
else:
|
||||
sbo_url: str = f'{self.sbo_repo_url}{location}/{file}'
|
||||
down_sbo = Downloader(self.tmp_slpkg, sbo_url, self.flags)
|
||||
down_sbo.download()
|
||||
sbos_urls.append(sbo_url)
|
||||
|
||||
self.utils.untar_archive(self.tmp_slpkg, file, self.build_path)
|
||||
down_sbo = Downloader(self.build_path, sbos_urls, self.flags)
|
||||
down_sbo.download()
|
||||
|
||||
self.patch_sbo_tag(sbo)
|
||||
for sbo in self.install_order:
|
||||
file: str = f'{sbo}{self.sbo_tar_suffix}'
|
||||
self.utils.untar_archive(self.build_path, file)
|
||||
self.patch_sbo_tag(sbo)
|
||||
|
||||
sources = SBoQueries(sbo).sources()
|
||||
self.download_sources(sbo, sources)
|
||||
self.download_sources()
|
||||
|
||||
def download_sources(self) -> None:
|
||||
""" Download the sources. """
|
||||
for sbo in self.install_order:
|
||||
sources = SBoQueries(sbo).sources()
|
||||
checksums = SBoQueries(sbo).checksum()
|
||||
|
||||
path = Path(self.build_path, sbo)
|
||||
|
||||
down_source = Downloader(path, sources, self.flags)
|
||||
down_source.download()
|
||||
|
||||
for source, checksum in zip(sources, checksums):
|
||||
|
||||
md5sum = Md5sum(self.flags)
|
||||
md5sum.check(path, source, checksum, sbo)
|
||||
|
||||
def build_and_install(self) -> None:
|
||||
""" Build the slackbuilds and install. """
|
||||
|
|
@ -278,18 +298,6 @@ class Slackbuilds(Configs):
|
|||
""" Set number of processors. """
|
||||
os.environ['MAKEFLAGS'] = f'-j {cpu_count()}'
|
||||
|
||||
def download_sources(self, name: str, sources: list) -> None:
|
||||
""" Download the sources. """
|
||||
path = Path(self.build_path, name)
|
||||
checksums: list = SBoQueries(name).checksum()
|
||||
|
||||
for source, checksum in zip(sources, checksums):
|
||||
down_source = Downloader(path, source, self.flags)
|
||||
down_source.download()
|
||||
|
||||
md5sum = Md5sum(self.flags)
|
||||
md5sum.check(path, source, checksum, name)
|
||||
|
||||
def multi_process(self, command: str, filename: str, message: str) -> None:
|
||||
""" Starting multiprocessing install/upgrade process. """
|
||||
if self.silent_mode and not self.utils.is_option(self.flag_no_silent, self.flags):
|
||||
|
|
|
|||
|
|
@ -59,11 +59,11 @@ class Utilities:
|
|||
yield self.split_installed_pkg(file.name)[0]
|
||||
|
||||
@staticmethod
|
||||
def untar_archive(path: str, archive: str, ext_path: str) -> None:
|
||||
def untar_archive(path: str, archive: str) -> None:
|
||||
""" Untar the file to the build folder. """
|
||||
tar_file = Path(path, archive)
|
||||
untar = tarfile.open(tar_file)
|
||||
untar.extractall(ext_path)
|
||||
untar.extractall(path)
|
||||
untar.close()
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
Loading…
Reference in a new issue