From b78e71a67be200dd07bfaf4e369cf0fb8f14261b Mon Sep 17 00:00:00 2001 From: Dimitris Zlatanidis Date: Thu, 6 Jun 2024 23:05:23 +0300 Subject: [PATCH] Replace tomli with tomlkit Signed-off-by: Dimitris Zlatanidis --- ChangeLog.txt | 1 + pyproject.toml | 3 +- requirements.txt | 2 +- salixos_requires.txt | 2 +- slpkg/blacklist.py | 15 ++--- slpkg/configs.py | 94 ++++++++++++++++--------------- slpkg/dialog_configs.py | 119 +++++++++++----------------------------- slpkg/repositories.py | 95 +++++++++++++++----------------- slpkg/utilities.py | 16 ------ 9 files changed, 133 insertions(+), 214 deletions(-) diff --git a/ChangeLog.txt b/ChangeLog.txt index fc0288a5..f25199e4 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -6,6 +6,7 @@ * Added PACKAGE_METHOD config to choose the upgrade method * Added DOWNGRADE_PACKAGES config to allow to downgrade packages (Thanks to marav) * Added PACKAGE_TYPE config if you prefer to add your own binary type + * Added python3-tomlkit to replace tomli and tomllib for -current - Updated: * Updated message for invalid package version diff --git a/pyproject.toml b/pyproject.toml index 5bedc718..ae40f1bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,8 @@ classifiers = [ ] dependencies = [ - "pythondialog>=3.5.3" + "pythondialog>=3.5.3", + "tomlkit>=0.12.5" ] [tool.flit_core] diff --git a/requirements.txt b/requirements.txt index 6d20cf9b..c821aa5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ pythondialog >= 3.5.3 -build >= 0.10.0 +tomlkit >= 0.12.5 [socks] PySocks >= 1.7.1 \ No newline at end of file diff --git a/salixos_requires.txt b/salixos_requires.txt index fe1d64ee..999f976f 100644 --- a/salixos_requires.txt +++ b/salixos_requires.txt @@ -3,7 +3,7 @@ Extra requirements for Salix GNU/Linux distribution. There are some additional requirements for salixos users: python-urllib3 -python-tomli +python-tomlkit python-packaging pyparsing lftp \ No newline at end of file diff --git a/slpkg/blacklist.py b/slpkg/blacklist.py index 02bc9cb1..c58cf81a 100644 --- a/slpkg/blacklist.py +++ b/slpkg/blacklist.py @@ -2,11 +2,8 @@ # -*- coding: utf-8 -*- -try: - import tomli -except ModuleNotFoundError: - import tomllib as tomli - +import tomlkit +from tomlkit.exceptions import TOMLKitError from pathlib import Path from slpkg.configs import Configs @@ -27,10 +24,10 @@ class Blacklist(Configs): # pylint: disable=[R0903] packages: list = [] if self.blacklist_file_toml.is_file(): try: - with open(self.blacklist_file_toml, 'rb') as black_file: - black: dict = {k.lower(): v for k, v in tomli.load(black_file).items()} - packages: list = black['packages'] - except (tomli.TOMLDecodeError, KeyError) as error: + with open(self.blacklist_file_toml, 'r', encoding='utf-8') as file: + black: dict = tomlkit.parse(file.read()) + packages: list = black['PACKAGES'] + except (KeyError, TOMLKitError) as error: print() self.toml_errors.raise_toml_error_message(error, self.blacklist_file_toml) diff --git a/slpkg/configs.py b/slpkg/configs.py index 54c5eb3d..c48a625a 100644 --- a/slpkg/configs.py +++ b/slpkg/configs.py @@ -2,10 +2,8 @@ # -*- coding: utf-8 -*- -try: - import tomli -except ModuleNotFoundError: - import tomllib as tomli +import tomlkit +from tomlkit.exceptions import TOMLKitError import platform from typing import Any @@ -76,57 +74,57 @@ class Configs: # pylint: disable=[R0902] try: # Load user configuration. - configs = {} + conf = {} config_path_file = Path(etc_path, f'{prog_name}.toml') if config_path_file.exists(): - with open(config_path_file, 'rb') as conf: - configs = {k.lower(): v for k, v in tomli.load(conf).items()} + with open(config_path_file, 'r', encoding='utf-8') as file: + conf = tomlkit.parse(file.read()) - if configs: - config = {k.lower(): v for k, v in configs['configs'].items()} + if conf: + config = conf['CONFIGS'] - os_arch: str = config['os_arch'] - file_list_suffix: str = config['file_list_suffix'] - package_type = config['package_type'] - installpkg: str = config['installpkg'] - reinstall: str = config['reinstall'] - removepkg: str = config['removepkg'] - kernel_version: str = config['kernel_version'] - colors: bool = config['colors'] - makeflags: str = config['makeflags'] - gpg_verification: bool = config['gpg_verification'] - checksum_md5: bool = config['checksum_md5'] - dialog: bool = config['dialog'] - view_missing_deps: bool = config['view_missing_deps'] - package_method: bool = config['package_method'] - downgrade_packages: bool = config['downgrade_packages'] - delete_sources: bool = config['delete_sources'] - downloader: str = config['downloader'] - wget_options: str = config['wget_options'] - curl_options: str = config['curl_options'] - lftp_get_options: str = config['lftp_get_options'] - lftp_mirror_options: str = config['lftp_mirror_options'] - download_only_path: Path = Path(config['download_only_path']) - ascii_characters: bool = config['ascii_characters'] - ask_question: bool = config['ask_question'] - parallel_downloads: bool = config['parallel_downloads'] - maximum_parallel: int = config['maximum_parallel'] - progress_bar_conf: bool = config['progress_bar'] - progress_spinner: str = config['progress_spinner'] - spinner_color: str = config['spinner_color'] - border_color: str = config['border_color'] - process_log: bool = config['process_log'] + os_arch: str = config['OS_ARCH'] + file_list_suffix: str = config['FILE_LIST_SUFFIX'] + package_type = config['PACKAGE_TYPE'] + installpkg: str = config['INSTALLPKG'] + reinstall: str = config['REINSTALL'] + removepkg: str = config['REMOVEPKG'] + kernel_version: str = config['KERNEL_VERSION'] + colors: bool = config['COLORS'] + makeflags: str = config['MAKEFLAGS'] + gpg_verification: bool = config['GPG_VERIFICATION'] + checksum_md5: bool = config['CHECKSUM_MD5'] + dialog: bool = config['DIALOG'] + view_missing_deps: bool = config['VIEW_MISSING_DEPS'] + package_method: bool = config['PACKAGE_METHOD'] + downgrade_packages: bool = config['DOWNGRADE_PACKAGES'] + delete_sources: bool = config['DELETE_SOURCES'] + downloader: str = config['DOWNLOADER'] + wget_options: str = config['WGET_OPTIONS'] + curl_options: str = config['CURL_OPTIONS'] + lftp_get_options: str = config['LFTP_GET_OPTIONS'] + lftp_mirror_options: str = config['LFTP_MIRROR_OPTIONS'] + download_only_path: Path = Path(config['DOWNLOAD_ONLY_PATH']) + ascii_characters: bool = config['ASCII_CHARACTERS'] + ask_question: bool = config['ASK_QUESTION'] + parallel_downloads: bool = config['PARALLEL_DOWNLOADS'] + maximum_parallel: int = config['MAXIMUM_PARALLEL'] + progress_bar_conf: bool = config['PROGRESS_BAR'] + progress_spinner: str = config['PROGRESS_SPINNER'] + spinner_color: str = config['SPINNER_COLOR'] + border_color: str = config['BORDER_COLOR'] + process_log: bool = config['PROCESS_LOG'] - urllib_retries: Any = config['urllib_retries'] - urllib_redirect: Any = config['urllib_redirect'] - urllib_timeout: float = config['urllib_timeout'] + urllib_retries: Any = config['URLLIB_RETRIES'] + urllib_redirect: Any = config['URLLIB_REDIRECT'] + urllib_timeout: float = config['URLLIB_TIMEOUT'] - proxy_address: str = config['proxy_address'] - proxy_username: str = config['proxy_username'] - proxy_password: str = config['proxy_password'] + proxy_address: str = config['PROXY_ADDRESS'] + proxy_username: str = config['PROXY_USERNAME'] + proxy_password: str = config['PROXY_PASSWORD'] - except (KeyError, tomli.TOMLDecodeError) as error: - toml_errors.raise_toml_error_message(error, toml_file=Path('/etc/slpkg/slpkg.toml')) + except (KeyError, TOMLKitError) as e: + toml_errors.raise_toml_error_message(e, toml_file=Path('/etc/slpkg/slpkg.toml')) blink: str = '' bold: str = '' diff --git a/slpkg/dialog_configs.py b/slpkg/dialog_configs.py index 2c4f2f57..280ac227 100644 --- a/slpkg/dialog_configs.py +++ b/slpkg/dialog_configs.py @@ -2,11 +2,13 @@ # -*- coding: utf-8 -*- +import re import os +import ast from pathlib import Path +import tomlkit from slpkg.configs import Configs -from slpkg.utilities import Utilities from slpkg.dialog_box import DialogBox from slpkg.error_messages import Errors @@ -18,7 +20,6 @@ class FormConfigs(Configs): super(Configs).__init__() self.dialogbox = DialogBox() self.errors = Errors() - self.utils = Utilities() self.orig_configs: list = [] self.config_file: Path = Path(self.etc_path, f'{self.prog_name}.toml') @@ -40,7 +41,7 @@ class FormConfigs(Configs): title: str = ' Configuration File ' # Creating the elements for the dialog form. - for i, (key, value) in enumerate(self.configs['configs'].items(), start=1): + for i, (key, value) in enumerate(self.config.items(), start=1): if value is True: value: str = 'true' elif value is False: @@ -54,104 +55,46 @@ class FormConfigs(Configs): os.system('clear') if code == 'help': - tags = self.configs.values() self.help() - check: bool = self.check_configs(tags) - - if code == 'ok' and check: - self.write_file(tags) - - elif not check: - self.edit() + if code == 'ok': + self.write_configs(tags) def help(self) -> None: """Load the configuration file on a text box.""" self.dialogbox.textbox(str(self.config_file), 40, 60) self.edit() - def check_configs(self, tags: list) -> bool: - """Check for true of false values.""" - keys: list = [ - 'COLORS', - 'DIALOG', - 'VIEW_MISSING_DEPS', - 'PACKAGE_METHOD', - 'DOWNGRADE_PACKAGES', - 'DELETE_SOURCES', - 'SILENT_MODE', - 'ASCII_CHARACTERS', - 'ASK_QUESTION', - 'KERNEL_VERSION', - 'PARALLEL_DOWNLOADS', - 'PROGRESS_BAR', - 'SPINNING_BAR', - 'CASE_INSENSITIVE', - 'PROCESS_LOG', - 'URLLIB_RETRIES', - 'URLLIB_REDIRECT', - 'GPG_VERIFICATION', - 'CHECKSUM_MD5' - ] - values: list = ['true', 'false'] - - for key, value in zip(self.configs['configs'].keys(), tags): - - if key in keys and value not in values: - self.dialogbox.msgbox(f"\nError: Value for '{key}', it must be 'true' or 'false.'\n", - height=7, width=60) - return False - - if key in ['DOWNLOADER'] and value not in ['wget2', 'wget', 'curl', 'lftp']: - self.dialogbox.msgbox(f"\nError: Value for '{key}' not supported.\n", - height=7, width=60) - return False - - return True - def read_configs(self) -> None: """Read the original config file.""" - with open(self.config_file, 'r', encoding='utf-8') as toml_file: - self.orig_configs: list = toml_file.readlines() + with open(self.config_file, 'r', encoding='utf-8') as file: + self.toml_original = tomlkit.parse(file.read()) - def write_file(self, tags: list) -> None: - """Write the new values to the config file.""" + def write_configs(self, tags: list) -> None: + """Write new configs to the file. + + Args: + tags (list): User new configs + """ self.read_configs() - with open(self.config_file, 'w', encoding='utf-8') as patch_toml: - for line in self.orig_configs: - for key, value in zip(self.configs['configs'].keys(), tags): + for key, new in zip(self.toml_original['CONFIGS'].keys(), tags): + digit_pattern = re.compile(r"^-?\d+(\.\d+)?$") # pattern for int and float numbers. + list_pattern = re.compile(r'^\s*\[.*\]\s*$') # pattern for list. - if line.lstrip().startswith(f'{key} ='): - line = f' {key} = "{value}"\n' + if new == 'true': + new = True + elif new == 'false': + new = False + elif digit_pattern.match(new): + if new.isdigit(): + new = int(new.replace('"', '')) + else: + new = float(new.replace('"', '')) + elif list_pattern.match(new): + new = ast.literal_eval(new) - if line.lstrip().startswith( + self.toml_original['CONFIGS'][key] = new - ('COLORS =', - 'DIALOG =', - 'VIEW_MISSING_DEPS =', - 'PACKAGE_METHOD =', - 'DOWNGRADE_PACKAGES =', - 'DELETE_SOURCES =', - 'SILENT_MODE =', - 'ASCII_CHARACTERS =', - 'ASK_QUESTION =', - 'KERNEL_VERSION =', - 'PARALLEL_DOWNLOADS =', - 'MAXIMUM_PARALLEL = ', - 'PROGRESS_BAR =', - 'SPINNING_BAR =', - 'CASE_SENSITIVE =', - 'PROCESS_LOG =', - 'URLLIB_RETRIES =', - 'URLLIB_REDIRECT =', - 'URLLIB_TIMEOUT =', - 'GPG_VERIFICATION =', - 'CHECKSUM_MD5 =') - ): - line: str = line.replace('"', '') - elif line.lstrip().startswith('PACKAGE_TYPE ='): - line: str = line.replace('"[', '[') - line: str = line.replace(']"', ']') - line: str = line.replace("'", '"') - patch_toml.write(line) + with open(self.config_file, 'w', encoding='utf-8') as file: + file.write(tomlkit.dumps(self.toml_original)) diff --git a/slpkg/repositories.py b/slpkg/repositories.py index 1379e34f..6a6417b2 100644 --- a/slpkg/repositories.py +++ b/slpkg/repositories.py @@ -2,11 +2,8 @@ # -*- coding: utf-8 -*- -try: - import tomli -except ModuleNotFoundError: - import tomllib as tomli - +import tomlkit +from tomlkit.exceptions import TOMLKitError from pathlib import Path from typing import ClassVar from dataclasses import dataclass @@ -157,82 +154,80 @@ class Repositories: # pylint: disable=[R0902] try: if repositories_toml_file.is_file(): - with open(repositories_toml_file, 'rb') as repo: - repos_config = tomli.load(repo) + with open(repositories_toml_file, 'r', encoding='utf-8') as file: + repos_config = tomlkit.parse(file.read()) - repos_config = utils.convert_dict_keys_to_lower(repos_config) + default_repository: str = repos_config['DEFAULT']['REPOSITORY'].lower() - default_repository: str = repos_config['default']['repository'].lower() + new_packages = repos_config['NEW_PACKAGES']['REPOSITORIES'] + remove_packages = repos_config['REMOVE_PACKAGES']['REPOSITORIES'] - new_packages = repos_config['new_packages']['repositories'] - remove_packages = repos_config['remove_packages']['repositories'] + sbo_repo: bool = repos_config['SBO']['ENABLE'] + sbo_repo_mirror: str = repos_config['SBO']['MIRROR'] - sbo_repo: bool = repos_config['sbo']['enable'] - sbo_repo_mirror: str = repos_config['sbo']['mirror'] + ponce_repo: bool = repos_config['PONCE']['ENABLE'] + ponce_repo_mirror: str = repos_config['PONCE']['MIRROR'] - ponce_repo: bool = repos_config['ponce']['enable'] - ponce_repo_mirror: str = repos_config['ponce']['mirror'] + slack_repo: bool = repos_config['SLACK']['ENABLE'] + slack_repo_mirror: str = repos_config['SLACK']['MIRROR'] - slack_repo: bool = repos_config['slack']['enable'] - slack_repo_mirror: str = repos_config['slack']['mirror'] - - slack_extra_repo: bool = repos_config['slack_extra']['enable'] - slack_extra_repo_url: str = repos_config['slack_extra']['mirror'] + slack_extra_repo: bool = repos_config['SLACK_EXTRA']['ENABLE'] + slack_extra_repo_url: str = repos_config['SLACK_EXTRA']['MIRROR'] slack_extra_repo_mirror_packages: str = slack_extra_repo_url slack_extra_repo_mirror_changelog: str = f"{'/'.join(slack_extra_repo_url.split('/')[:-2])}/" - slack_patches_repo: bool = repos_config['slack_patches']['enable'] - slack_patches_repo_url: str = repos_config['slack_patches']['mirror'] + slack_patches_repo: bool = repos_config['SLACK_PATCHES']['ENABLE'] + slack_patches_repo_url: str = repos_config['SLACK_PATCHES']['MIRROR'] slack_patches_repo_mirror_packages: str = slack_patches_repo_url slack_patches_repo_mirror_changelog: str = f"{'/'.join(slack_patches_repo_url.split('/')[:-2])}/" - alien_repo: bool = repos_config['alien']['enable'] - alien_repo_url: str = repos_config['alien']['mirror'] + alien_repo: bool = repos_config['ALIEN']['ENABLE'] + alien_repo_url: str = repos_config['ALIEN']['MIRROR'] alien_repo_mirror_packages: str = alien_repo_url alien_repo_mirror_changelog: str = f"{'/'.join(alien_repo_url.split('/')[:-3])}/" - multilib_repo: bool = repos_config['multilib']['enable'] - multilib_repo_url: str = repos_config['multilib']['mirror'] + multilib_repo: bool = repos_config['MULTILIB']['ENABLE'] + multilib_repo_url: str = repos_config['MULTILIB']['MIRROR'] multilib_repo_mirror_packages: str = multilib_repo_url multilib_repo_mirror_changelog: str = f"{'/'.join(multilib_repo_url.split('/')[:-2])}/" - restricted_repo: bool = repos_config['restricted']['enable'] - restricted_repo_url: str = repos_config['restricted']['mirror'] + restricted_repo: bool = repos_config['RESTRICTED']['ENABLE'] + restricted_repo_url: str = repos_config['RESTRICTED']['MIRROR'] restricted_repo_mirror_packages: str = restricted_repo_url restricted_repo_mirror_changelog: str = f"{'/'.join(restricted_repo_url.split('/')[:-3])}/" - gnome_repo: bool = repos_config['gnome']['enable'] - gnome_repo_mirror: str = repos_config['gnome']['mirror'] + gnome_repo: bool = repos_config['GNOME']['ENABLE'] + gnome_repo_mirror: str = repos_config['GNOME']['MIRROR'] - msb_repo: bool = repos_config['msb']['enable'] - msb_repo_url: str = repos_config['msb']['mirror'] + msb_repo: bool = repos_config['MSB']['ENABLE'] + msb_repo_url: str = repos_config['MSB']['MIRROR'] msb_repo_mirror_packages: str = msb_repo_url msb_repo_mirror_changelog: str = f"{'/'.join(msb_repo_url.split('/')[:-4])}/" - csb_repo: bool = repos_config['csb']['enable'] - csb_repo_mirror: str = repos_config['csb']['mirror'] + csb_repo: bool = repos_config['CSB']['ENABLE'] + csb_repo_mirror: str = repos_config['CSB']['MIRROR'] - conraid_repo: bool = repos_config['conraid']['enable'] - conraid_repo_mirror: str = repos_config['conraid']['mirror'] + conraid_repo: bool = repos_config['CONRAID']['ENABLE'] + conraid_repo_mirror: str = repos_config['CONRAID']['MIRROR'] - slackonly_repo: bool = repos_config['slackonly']['enable'] - slackonly_repo_mirror: str = repos_config['slackonly']['mirror'] + slackonly_repo: bool = repos_config['SLACKONLY']['ENABLE'] + slackonly_repo_mirror: str = repos_config['SLACKONLY']['MIRROR'] - salix_repo: bool = repos_config['salix']['enable'] - salix_repo_mirror: str = repos_config['salix']['mirror'] + salix_repo: bool = repos_config['SALIX']['ENABLE'] + salix_repo_mirror: str = repos_config['SALIX']['MIRROR'] - salix_extra_repo: bool = repos_config['salix_extra']['enable'] - salix_extra_repo_mirror: str = repos_config['salix_extra']['mirror'] + salix_extra_repo: bool = repos_config['SALIX_EXTRA']['ENABLE'] + salix_extra_repo_mirror: str = repos_config['SALIX_EXTRA']['MIRROR'] - slackel_repo: bool = repos_config['slackel']['enable'] - slackel_repo_mirror: str = repos_config['slackel']['mirror'] + slackel_repo: bool = repos_config['SLACKEL']['ENABLE'] + slackel_repo_mirror: str = repos_config['SLACKEL']['MIRROR'] - slint_repo: bool = repos_config['slint']['enable'] - slint_repo_mirror: str = repos_config['slint']['mirror'] + slint_repo: bool = repos_config['SLINT']['ENABLE'] + slint_repo_mirror: str = repos_config['SLINT']['MIRROR'] - pprkut_repo: bool = repos_config['pprkut']['enable'] - pprkut_repo_mirror: str = repos_config['pprkut']['mirror'] - except (tomli.TOMLDecodeError, KeyError) as error: + pprkut_repo: bool = repos_config['PPRKUT']['ENABLE'] + pprkut_repo_mirror: str = repos_config['PPRKUT']['MIRROR'] + except (KeyError, TOMLKitError) as error: toml_errors.raise_toml_error_message(error, repositories_toml_file) # Dictionary configurations of repositories. @@ -418,7 +413,7 @@ class Repositories: # pylint: disable=[R0902] 'repo_tag': pprkut_repo_tag} } - all_repos = list(repos_config.keys()) + all_repos = [name.lower() for name in repos_config.keys()] defaults_repos = list(repositories.keys()) diff_repos = list(set(all_repos) - set(defaults_repos)) diff --git a/slpkg/utilities.py b/slpkg/utilities.py index ceb79cbf..fbb73887 100644 --- a/slpkg/utilities.py +++ b/slpkg/utilities.py @@ -306,19 +306,3 @@ class Utilities(Configs): pattern: str = '|'.join(blacklist) matching_packages: list = [pkg for pkg in packages if re.search(pattern, pkg)] return matching_packages - - def convert_dict_keys_to_lower(self, d: dict) -> dict: - """Convert dictionary keys to lower. - - Args: - d (dict): Dictionary data. - - Returns: - dict: Dictionary in lower case. - """ - new_dict = {} - for key, value in d.items(): - if isinstance(value, dict): - value = self.convert_dict_keys_to_lower(value) - new_dict[key.lower()] = value - return new_dict