From ee8f02307c4486df5e9d85a5de02a1a656c92a27 Mon Sep 17 00:00:00 2001 From: Dimitris Zlatanidis Date: Sat, 18 Mar 2023 21:48:10 +0200 Subject: [PATCH] Updated for binaries --- configs/repositories.toml | 7 +++ slpkg/install_data.py | 52 +++++++++++++++++++++- slpkg/main.py | 14 ++++-- slpkg/models/models.py | 21 +++++++++ slpkg/repositories.py | 39 ++++++++++++++++ slpkg/slackbuild.py | 4 +- slpkg/update_repository.py | 91 +++++++++++++++++++++++++++----------- slpkg/upgrade.py | 2 +- slpkg/utilities.py | 14 +++--- slpkg/views/views.py | 4 +- tests/test_utilities.py | 10 ++--- 11 files changed, 210 insertions(+), 48 deletions(-) create mode 100644 configs/repositories.toml create mode 100644 slpkg/repositories.py diff --git a/configs/repositories.toml b/configs/repositories.toml new file mode 100644 index 00000000..8beccc16 --- /dev/null +++ b/configs/repositories.toml @@ -0,0 +1,7 @@ +# Repositories configuration file. + +[[BINARY]] +REPO_NAME = "gnome" +REPO_URL = "https://reddoglinux.ddns.net/linux/gnome/43.x/x86_64/" +REPO_TXT = "PACKAGES.TXT" +REPO_CHGLOG_TXT = "ChangeLog.txt" diff --git a/slpkg/install_data.py b/slpkg/install_data.py index adf564bb..9d43a785 100644 --- a/slpkg/install_data.py +++ b/slpkg/install_data.py @@ -7,7 +7,7 @@ from slpkg.configs import Configs from slpkg.queries import SBoQueries from slpkg.utilities import Utilities from slpkg.models.models import session as Session -from slpkg.models.models import SBoTable, PonceTable +from slpkg.models.models import SBoTable, PonceTable, BinariesTable class CreateData(Configs): @@ -67,3 +67,53 @@ class CreateData(Configs): print('Done') self.session.commit() + + def install_binaries_table(self, repo: str, repo_txt: str) -> None: + """ Install the data for SBo repository. """ + pkg_tags = [ + 'PACKAGE NAME:', + 'PACKAGE MIRROR:', + 'PACKAGE LOCATION:', + 'PACKAGE SIZE (compressed):', + 'PACKAGE SIZE (uncompressed):', + 'PACKAGE REQUIRED:', + 'PACKAGE CONFLICTS:', + 'PACKAGE SUGGESTS:', + 'PACKAGE DESCRIPTION:' + ] + path = Path(self.lib_path, 'repositories', repo, repo_txt) + + pkg_txt: list = self.utils.read_file(path) + + cache: list = [] # init cache + + print('Creating the database... ', end='', flush=True) + + for i, line in enumerate(pkg_txt, 1): + + for tag in pkg_tags: + if line.startswith(tag): + + if line.startswith(pkg_tags[0]): + line = line.replace(tag, '').strip() + cache.append(self.utils.split_binary_pkg(line)[0]) + cache.append(self.utils.split_binary_pkg(line)[1]) + + line = line.replace(tag, '').strip() + cache.append(line) + + for line in cache: + print(line) + # if (i % 11) == 0: + # data: str = BinariesTable(name=cache[0], location=cache[1].split('/')[1], + # files=cache[2], version=cache[3], + # download=cache[4], download64=cache[5], + # md5sum=cache[6], md5sum64=cache[7], + # requires=cache[8], short_description=cache[9]) + # self.session.add(data) + # + # cache: list = [] # reset cache after 11 lines + # + # print('Done') + # + # self.session.commit() diff --git a/slpkg/main.py b/slpkg/main.py index 50c665ee..e7078ee1 100644 --- a/slpkg/main.py +++ b/slpkg/main.py @@ -79,6 +79,8 @@ class Argparse(Configs): self.flag_short_generate: str = '-G' self.flag_parallel: str = '--parallel' self.flag_short_parallel: str = '-P' + self.flag_binary: str = '--binary' + self.flag_short_binary: str = '-B' self.flag_directory: str = '--directory=' self.flag_short_directory: str = '-z=' self.flag_file_pattern: str = '--file-pattern=' @@ -109,6 +111,8 @@ class Argparse(Configs): self.flag_short_generate, self.flag_parallel, self.flag_short_parallel, + self.flag_binary, + self.flag_short_binary, self.flag_directory, self.flag_short_directory, self.flag_file_pattern, @@ -124,6 +128,8 @@ class Argparse(Configs): self.flag_short_yes, self.flag_generate, self.flag_short_generate, + self.flag_binary, + self.flag_short_binary, self.flag_parallel, self.flag_short_parallel, ], @@ -385,8 +391,8 @@ class Argparse(Configs): if method in ['remove', 'find']: for package in installed: - name: str = self.utils.split_installed_pkg(package)[0] - version: str = self.utils.split_installed_pkg(package)[1] + name: str = self.utils.split_binary_pkg(package)[0] + version: str = self.utils.split_binary_pkg(package)[1] for pkg in packages: if pkg in name: @@ -399,10 +405,10 @@ class Argparse(Configs): if pkg == package: repo_ver: str = SBoQueries(package).version() inst_pkg: str = self.utils.is_package_installed(package, self.file_pattern) - inst_ver: str = self.utils.split_installed_pkg(inst_pkg)[1] + inst_ver: str = self.utils.split_binary_pkg(inst_pkg)[1] repo_build_tag = self.utils.read_build_tag(package) - inst_build_tag = self.utils.split_installed_pkg(inst_pkg)[3] + inst_build_tag = self.utils.split_binary_pkg(inst_pkg)[3] choices += [(package, f'{inst_ver} -> {repo_ver}', True, f'Installed: {package}-{inst_ver} Build: {inst_build_tag} -> ' diff --git a/slpkg/models/models.py b/slpkg/models/models.py index 9f53fce7..bf3cae94 100644 --- a/slpkg/models/models.py +++ b/slpkg/models/models.py @@ -57,6 +57,27 @@ class PonceTable(Base): short_description: str = Column(Text) # type: ignore +@dataclass +class BinariesTable(Base): + """ The main table for the SBo repository. """ + + __tablename__ = 'binariestable' + + id: int = Column(Integer, primary_key=True) # type: ignore + repo: str = Column(Text) # type: ignore + name: str = Column(Text) # type: ignore + version: str = Column(Text) # type: ignore + package: str = Column(Text) # type: ignore + mirror: str = Column(Text) # type: ignore + location: str = Column(Text) # type: ignore + size_comp: str = Column(Text) # type: ignore + size_uncomp: str = Column(Text) # type: ignore + requires: str = Column(Text) # type: ignore + conflicts: str = Column(Text) # type: ignore + suggests: str = Column(Text) # type: ignore + description: str = Column(Text) # type: ignore + + @dataclass class LogsDependencies(Base): """ The table that stores the dependencies after installing a package. """ diff --git a/slpkg/repositories.py b/slpkg/repositories.py new file mode 100644 index 00000000..8eabdc55 --- /dev/null +++ b/slpkg/repositories.py @@ -0,0 +1,39 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + + +import tomli +from pathlib import Path + +from slpkg.configs import Configs +from slpkg.models.models import session as Session + + +class Repositories(Configs): + + def __init__(self): + super(Configs, self).__init__() + self.session = Session + + self.color = self.colour() + self.bold: str = self.color['bold'] + self.red: str = self.color['red'] + self.cyan: str = self.color['cyan'] + self.endc: str = self.color['endc'] + self.bred: str = f'{self.bold}{self.red}' + self.repositories_file_toml = Path(self.etc_path, 'repositories.toml') + + def configs(self) -> list: + """ Reads the repositories file. """ + + if self.repositories_file_toml.is_file(): + try: + with open(self.repositories_file_toml, 'rb') as repo: + return tomli.load(repo)['BINARY'] + except (tomli.TOMLDecodeError, KeyError) as error: + raise SystemExit(f"\n{self.prog_name} {self.bred}Error{self.endc}: {error}: in the configuration file " + f"'{self.repositories_file_toml}'.\n" + f"\nIf you have upgraded the '{self.prog_name}' probably you need to run:\n" + f"'mv {self.repositories_file_toml}.new {self.repositories_file_toml}'.\n" + f"or '{self.cyan}slpkg_new-configs{self.endc}' command.\n") + return [] diff --git a/slpkg/slackbuild.py b/slpkg/slackbuild.py index 9a54b121..c3553e23 100644 --- a/slpkg/slackbuild.py +++ b/slpkg/slackbuild.py @@ -208,7 +208,7 @@ class Slackbuilds(Configs): self.logging_installed_dependencies(sbo) else: package: str = self.utils.is_package_installed(sbo, self.file_pattern) - version: str = self.utils.split_installed_pkg(package)[1] + version: str = self.utils.split_binary_pkg(package)[1] self.view_message.view_skipping_packages(sbo, version) def patch_sbo_tag(self, sbo: str) -> None: @@ -246,7 +246,7 @@ class Slackbuilds(Configs): def install_package(self, package: str) -> None: """ Install the packages that before created in the tmp directory. """ - pkg: str = self.utils.split_installed_pkg(package)[0] + pkg: str = self.utils.split_binary_pkg(package)[0] execute: str = self.installpkg if (self.utils.is_option(self.flag_reinstall, self.flags) and diff --git a/slpkg/update_repository.py b/slpkg/update_repository.py index 63f5f024..189627a1 100644 --- a/slpkg/update_repository.py +++ b/slpkg/update_repository.py @@ -13,7 +13,10 @@ from slpkg.progress_bar import ProgressBar from slpkg.check_updates import CheckUpdates from slpkg.models.models import session as Session from slpkg.models.models import (Base, engine, SBoTable, - PonceTable) + PonceTable, BinariesTable) + +from slpkg.repositories import Repositories +from slpkg.downloader import Downloader class UpdateRepository(Configs): @@ -25,9 +28,12 @@ class UpdateRepository(Configs): self.session = Session self.view = ViewMessage(self.flags) + self.repos = Repositories() + self.repositories: list = self.repos.configs() self.progress = ProgressBar() self.utils = Utilities() self.color = self.colour() + self.data = CreateData() self.update: int = 0 self.bold: str = self.color['bold'] @@ -38,6 +44,7 @@ class UpdateRepository(Configs): self.bred: str = f'{self.bold}{self.red}' self.endc: str = self.color['endc'] self.flag_generate: list = ['-G', '--generate-only'] + self.flag_binary: list = ['-B', '--binary'] def update_the_repository(self) -> None: """ Updated the sbo repository. """ @@ -46,40 +53,67 @@ class UpdateRepository(Configs): else: print() - if self.ponce_repo: + if not self.utils.is_option(self.flag_binary, self.flags): + if self.ponce_repo: - if not self.utils.is_option(self.flag_generate, self.flags): + if not self.utils.is_option(self.flag_generate, self.flags): + print('Updating the packages list.\n') + print(f"Downloading the '{self.green}ponce{self.endc}' repository, please wait...\n") + self.delete_file(self.ponce_repo_path, self.ponce_chglog_txt) + lftp_command: str = f'lftp {self.lftp_mirror_options} {self.ponce_repo_url} {self.ponce_repo_path}' + self.utils.process(lftp_command) + + # Remove the SLACKBUILDS.TXT file before generating the new one. + sbo_file_txt = Path(self.ponce_repo_path, self.ponce_txt) + if sbo_file_txt.is_file(): + sbo_file_txt.unlink() + + # Generating the ponce SLACKBUILDS.TXT file. + print(f'Generating the {self.ponce_txt} file... ', end='', flush=True) + os.chdir(self.ponce_repo_path) + gen_command: str = f'./gen_sbo_txt.sh > {self.ponce_txt}' + self.utils.process(gen_command) + print('\n') + + else: print('Updating the packages list.\n') - print(f"Downloading the '{self.green}ponce{self.endc}' repository, please wait...\n") - self.delete_file(self.ponce_repo_path, self.ponce_chglog_txt) - lftp_command: str = f'lftp {self.lftp_mirror_options} {self.ponce_repo_url} {self.ponce_repo_path}' + + self.delete_file(self.sbo_repo_path, self.sbo_txt) + self.delete_file(self.sbo_repo_path, self.sbo_chglog_txt) + + print(f"Downloading the '{self.green}sbo{self.endc}' repository, please wait...\n") + lftp_command: str = f'lftp {self.lftp_mirror_options} {self.sbo_repo_url} {self.sbo_repo_path}' self.utils.process(lftp_command) - # Remove the SLACKBUILDS.TXT file before generating the new one. - sbo_file_txt = Path(self.ponce_repo_path, self.ponce_txt) - if sbo_file_txt.is_file(): - sbo_file_txt.unlink() - - # Generating the ponce SLACKBUILDS.TXT file. - print(f'Generating the {self.ponce_txt} file... ', end='', flush=True) - os.chdir(self.ponce_repo_path) - gen_command: str = f'./gen_sbo_txt.sh > {self.ponce_txt}' - self.utils.process(gen_command) - print('\n') + self.delete_sbo_data() + self.data.install_sbo_table() else: - print('Updating the packages list.\n') + self.make_dirs() + urls: list = [] + for repo in self.repositories: - self.delete_file(self.sbo_repo_path, self.sbo_txt) - self.delete_file(self.sbo_repo_path, self.sbo_chglog_txt) + urls.append(f"{repo['REPO_URL']}{repo['REPO_TXT']}") + urls.append(f"{repo['REPO_URL']}{repo['REPO_CHGLOG_TXT']}") + repo_path = Path(self.lib_path, 'repositories', repo['REPO_NAME']) - print(f"Downloading the '{self.green}sbo{self.endc}' repository, please wait...\n") - lftp_command: str = f'lftp {self.lftp_mirror_options} {self.sbo_repo_url} {self.sbo_repo_path}' - self.utils.process(lftp_command) + # self.delete_file(f"{self.lib_path}/repositories/{repo['REPO_NAME']}", repo['REPO_TXT']) + # self.delete_file(f"{self.lib_path}/repositories/{repo['REPO_NAME']}", repo['REPO_CHGLOG_TXT']) - self.delete_sbo_data() - data = CreateData() - data.install_sbo_table() + down = Downloader(repo_path, urls, self.flags) + down.download() + + urls: list = [] # Reset for the next repo + + # self.delete_binaries_data() + self.data.install_binaries_table(repo['REPO_NAME'], repo['REPO_TXT']) + + def make_dirs(self): + """ Creating the repositories folders. """ + for repo in self.repositories: + path = Path(self.lib_path, 'repositories', repo['REPO_NAME']) + if not os.path.isdir(path): + os.makedirs(path) def check(self, queue) -> None: check_updates = CheckUpdates() @@ -136,6 +170,11 @@ class UpdateRepository(Configs): self.session.query(SBoTable).delete() self.session.commit() + def delete_binaries_data(self) -> None: + """ Delete all the data from a table of the database. """ + self.session.query(BinariesTable).delete() + self.session.commit() + def drop_the_tables(self): """ Drop all the tables from the database. """ print(f'\n{self.prog_name}: {self.bred}WARNING{self.endc}: All the data from the database will be deleted!') diff --git a/slpkg/upgrade.py b/slpkg/upgrade.py index a031f695..aab7fc0b 100644 --- a/slpkg/upgrade.py +++ b/slpkg/upgrade.py @@ -26,7 +26,7 @@ class Upgrade(Configs, Utilities): installed: list = list(self.all_installed(self.file_pattern)) for pkg in installed: - inst_pkg_name: str = self.split_installed_pkg(pkg)[0] + inst_pkg_name: str = self.split_binary_pkg(pkg)[0] if inst_pkg_name in repo_packages: diff --git a/slpkg/utilities.py b/slpkg/utilities.py index ddf94371..7fe4b58f 100644 --- a/slpkg/utilities.py +++ b/slpkg/utilities.py @@ -35,7 +35,7 @@ class Utilities: installed: list = list(self.all_installed(pattern)) for package in installed: - pkg: str = self.split_installed_pkg(package)[0] + pkg: str = self.split_binary_pkg(package)[0] if pkg == name: return package @@ -48,7 +48,7 @@ class Utilities: try: for file in var_log_packages.glob(pattern): - package_name = self.split_installed_pkg(file.name)[0] + package_name = self.split_binary_pkg(file.name)[0] if package_name not in self.black.packages(): yield file.name @@ -61,10 +61,10 @@ class Utilities: try: for file in var_log_packages.glob(pattern): - package_name = self.split_installed_pkg(file.name)[0] + package_name = self.split_binary_pkg(file.name)[0] if package_name not in self.black.packages(): - yield self.split_installed_pkg(file.name)[0] + yield self.split_binary_pkg(file.name)[0] except ValueError: pass @@ -89,7 +89,7 @@ class Utilities: if not directory.exists(): directory.mkdir(parents=True, exist_ok=True) - def split_installed_pkg(self, package: str) -> list: + def split_binary_pkg(self, package: str) -> list: """ Split the package by the name, version, arch, build and tag. """ name: str = '-'.join(package.split('-')[:-3]) version: str = ''.join(package[len(name):].split('-')[:-2]) @@ -115,7 +115,7 @@ class Utilities: if not repo_build_tag: repo_build_tag: str = '' - inst_build_tag: str = self.split_installed_pkg(installed)[3] + inst_build_tag: str = self.split_binary_pkg(installed)[3] if not inst_build_tag: inst_build_tag: str = '' @@ -123,7 +123,7 @@ class Utilities: repository_version: str = '0' if installed: - installed_version: str = self.split_installed_pkg(installed)[1] + installed_version: str = self.split_binary_pkg(installed)[1] return (str(LooseVersion(repository_version + repo_build_tag)) > str(LooseVersion(installed_version + inst_build_tag))) diff --git a/slpkg/views/views.py b/slpkg/views/views.py index f13998fe..525ca588 100644 --- a/slpkg/views/views.py +++ b/slpkg/views/views.py @@ -159,10 +159,10 @@ class ViewMessage(Configs): if self.utils.is_package_installed(name, self.file_pattern): for package in installed: - pkg: str = self.utils.split_installed_pkg(package)[0] + pkg: str = self.utils.split_binary_pkg(package)[0] if pkg == name: self.installed_packages.append(package) - version = self.utils.split_installed_pkg(package)[1] + version = self.utils.split_binary_pkg(package)[1] self.view_packages(pkg, version, mode='remove') def choose_dependencies_for_remove(self, dependencies: list) -> list: diff --git a/tests/test_utilities.py b/tests/test_utilities.py index f592c8fa..c4b40384 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -16,19 +16,19 @@ class TestUtilities(unittest.TestCase): self.assertEqual(self.package, self.utils.is_package_installed('fish', self.file_pattern)) def test_split_name(self): - self.assertEqual('fish', self.utils.split_installed_pkg(self.package)[0]) + self.assertEqual('fish', self.utils.split_binary_pkg(self.package)[0]) def test_split_version(self): - self.assertEqual('3.6.0', self.utils.split_installed_pkg(self.package)[1]) + self.assertEqual('3.6.0', self.utils.split_binary_pkg(self.package)[1]) def test_split_arch(self): - self.assertEqual('x86_64', self.utils.split_installed_pkg(self.package)[2]) + self.assertEqual('x86_64', self.utils.split_binary_pkg(self.package)[2]) def test_split_build(self): - self.assertEqual('1', self.utils.split_installed_pkg(self.package)[3]) + self.assertEqual('1', self.utils.split_binary_pkg(self.package)[3]) def test_split_tag(self): - self.assertEqual('SBo', self.utils.split_installed_pkg(self.package)[4]) + self.assertEqual('SBo', self.utils.split_binary_pkg(self.package)[4]) if __name__ == '__main__':