Miroirs/slpkg
1
0
Fork 0
mirror of https://gitlab.com/dslackw/slpkg.git synced 2024-12-26 09:58:31 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixed for max parallel download

Browse source 
Signed-off-by: Dimitris Zlatanidis <d.zlatanidis@gmail.com>
This commit is contained in:
Dimitris Zlatanidis 2024-07-22 23:08:50 +03:00
parent 0a9c900bb4
commit 6fc01d225e
2 changed files with 10 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
ChangeLog.txt
View file

@ -8,6 +8,7 @@
- Fixed: - Fixed:
* Fixed to create custom repositories (Thanks to mac-a-r0ni) * Fixed to create custom repositories (Thanks to mac-a-r0ni)
* Fixed OSError: [Errno 24] too many open files for parallel download (Thanks to rizitis)
### 5.1.0 - 06/06/2024 ### 5.1.0 - 06/06/2024

17
slpkg/downloader.py
View file

@ -3,8 +3,9 @@
import os import os
import threading
from pathlib import Path from pathlib import Path
from multiprocessing import Process, Semaphore from multiprocessing import Process
from urllib.parse import unquote, urlparse from urllib.parse import unquote, urlparse
from slpkg.configs import Configs from slpkg.configs import Configs
@ -35,7 +36,8 @@ class Downloader(Configs): # pylint: disable=[R0902]
'lftp': self.set_lftp_downloader 'lftp': self.set_lftp_downloader
} }
self.semaphore = Semaphore(self.maximum_parallel) # Semaphore to control the number of concurrent threads
self.semaphore = threading.BoundedSemaphore(int(self.maximum_parallel))
self.option_for_parallel: bool = self.utils.is_option( self.option_for_parallel: bool = self.utils.is_option(
('-P', '--parallel'), flags) ('-P', '--parallel'), flags)
@ -51,10 +53,11 @@ class Downloader(Configs): # pylint: disable=[R0902]
"""Download sources with parallel mode.""" """Download sources with parallel mode."""
processes: list = [] processes: list = []
for urls, path in sources.values(): for urls, path in sources.values():
for url in urls: with self.semaphore:
proc = Process(target=self.tools, args=(url, path)) for url in urls:
processes.append(proc) proc = Process(target=self.tools, args=(url, path))
proc.start() processes.append(proc)
proc.start()
for process in processes: for process in processes:
process.join() process.join()
@ -72,7 +75,6 @@ class Downloader(Configs): # pylint: disable=[R0902]
url (str): The URL link. url (str): The URL link.
path (Path): Path to save. path (Path): Path to save.
""" """
self.semaphore.acquire()
url_parse: str = urlparse(url).path url_parse: str = urlparse(url).path
self.filename: str = unquote(Path(url_parse).name) self.filename: str = unquote(Path(url_parse).name)
@ -83,7 +85,6 @@ class Downloader(Configs): # pylint: disable=[R0902]
self.multi_process.process(self.downloader_command) self.multi_process.process(self.downloader_command)
self.check_if_downloaded(url, path) self.check_if_downloaded(url, path)
self.semaphore.release()
def set_wget_downloader(self, url: str, path: Path) -> None: def set_wget_downloader(self, url: str, path: Path) -> None:
"""Set for wget tool. """Set for wget tool.