Split download methods

This commit is contained in:
Dimitris Zlatanidis 2023-05-29 21:45:16 +03:00
parent bfc2788ebb
commit 49d149b8ac

View file

@ -33,21 +33,28 @@ class Downloader(Configs):
def download(self, sources: dict) -> None:
""" Starting the processing for downloading. """
processes: list = []
if self.parallel_downloads or self.option_for_parallel:
for urls, path in sources.values():
for url in urls:
proc = Process(target=self.tools, args=(url, path))
processes.append(proc)
proc.start()
for process in processes:
process.join()
self.parallel_download(sources)
else:
for urls, path in sources.values():
for url in urls:
self.tools(url, path)
self.normal_download(sources)
def parallel_download(self, sources):
""" Download sources with parallel mode. """
processes: list = []
for urls, path in sources.values():
for url in urls:
proc = Process(target=self.tools, args=(url, path))
processes.append(proc)
proc.start()
for process in processes:
process.join()
def normal_download(self, sources):
""" Download sources with normal mode. """
for urls, path in sources.values():
for url in urls:
self.tools(url, path)
def tools(self, url: str, path: Path) -> None:
url_parse: str = urlparse(url).path