From 8e06777e51e7ee6948e93ff1a58b876628be650b Mon Sep 17 00:00:00 2001 From: hygienic-books Date: Fri, 11 Mar 2022 01:36:11 +0100 Subject: [PATCH 1/3] Add basics for parsing a config file --- .gitignore | 3 +- config.ini | 9 +++-- main.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.in | 1 + requirements.txt | 12 +++++++ 5 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 requirements.in create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index bd1f177..058fa64 100644 --- a/.gitignore +++ b/.gitignore @@ -234,4 +234,5 @@ fabric.properties # ---> Custom .idea/deployment.xml .idea/misc.xml -.idea/remote-mappings.xml \ No newline at end of file +.idea/remote-mappings.xml +.idea/mvw-dl.iml \ No newline at end of file diff --git a/config.ini b/config.ini index e59260f..bee91c6 100644 --- a/config.ini +++ b/config.ini @@ -11,6 +11,11 @@ state_file_name_suffix = .log min_duration = 1200 max_duration = 2700 query = @maus-query.json -query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50} +# query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50} # state_file_name = maus -# tmp_base_dir = %(tmp_base_dir)s/maus \ No newline at end of file +# tmp_base_dir = %(tmp_base_dir)s/maus + +[test] +min_duration = 100 +max_duration = 200 +query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50} \ No newline at end of file diff --git a/main.py b/main.py index 76d0e8c..99f0ba8 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,89 @@ +import configparser +import logging +import os +import sys +from rich.logging import RichHandler + + +# Exit codes +# 1: Config file invalid, it has no sections +# 2: Config file invalid, sections must define at least CONST.CFG_MANDATORY + + +class CONST(object): + __slots__ = () + LOG_FORMAT = "%(message)s" + CFG_DEFAULT_FILENAME = "config.ini" + CFG_DEFAULT_ABS_PATH = os.path.join(os.getcwd(), CFG_DEFAULT_FILENAME) + CFG_MANDATORY = "query" + + +CONST = CONST() +logging.basicConfig( + level="NOTSET", + format=CONST.LOG_FORMAT, + datefmt="[%X]", + handlers=[RichHandler( + show_time=False if "SYSTEMD_EXEC_PID" in os.environ else True, + rich_tracebacks=True + )] +) +log = logging.getLogger("rich") +log.setLevel(logging.DEBUG) + + +config = configparser.ConfigParser() +config.read(CONST.CFG_DEFAULT_FILENAME) + + +def validate_config_defaults(config_obj: configparser.ConfigParser()) -> None: + log.debug(f"Loading config from file '{CONST.CFG_DEFAULT_ABS_PATH}' ...") + if not config_obj.sections(): + log.error(f"No config sections found in '{CONST.CFG_DEFAULT_ABS_PATH}'. Exiting 1 ...") + sys.exit(1) + if config.defaults(): + for default in config_obj.defaults(): + log.debug(f" {default} = {config_obj[config_obj.default_section][default]}") + else: + log.debug(f"No defaults defined") + + +def config_has_valid_section(config_obj: configparser.ConfigParser()) -> bool: + has_valid_section = False + for config_obj_section in config_obj.sections(): + if CONST.CFG_MANDATORY in config_obj.options(config_obj_section): + has_valid_section = True + break + return has_valid_section + + +#def option_has_default_value(config_option: configparser.ConfigParser(), section: str) -> bool: +# +#~ local option that doesn't exist in defaults section +#+ local option that overrides a default +#* global option + +def validate_config_sections(config_obj: configparser.ConfigParser()) -> None: + for section in config_obj.sections(): + log.debug(f"Loading config '[{section}]' ...") + for key in config_obj[section]: + # log.debug(config_obj.has_option(section, key)) + is_local_option = True if config_obj.has_option(section, key) else False + log.debug(f""" {key}{"/" if is_local_option else ".."} = {config_obj[section][key]}""") + + +validate_config_defaults(config) +if config_has_valid_section(config): + validate_config_sections(config) +else: + log.error(f"No valid config section found. A valid config section has at least the '{CONST.CFG_MANDATORY}' " + f"option set. Exiting 2 ...") + sys.exit(2) + + +# https://stackoverflow.com/a/2451872 +quit() + # This is a sample Python script. # Press Umschalt+F10 to execute it or replace it with your code. diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..c94be38 --- /dev/null +++ b/requirements.in @@ -0,0 +1 @@ +rich \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a3f3a84 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile +# +commonmark==0.9.1 + # via rich +pygments==2.11.2 + # via rich +rich==12.0.0 + # via -r requirements.in -- 2.47.2 From d1130baa10d3fc14f246ade6e4963ea004c093fc Mon Sep 17 00:00:00 2001 From: hygienic-books Date: Fri, 11 Mar 2022 01:42:24 +0100 Subject: [PATCH 2/3] Remove iml file --- .idea/mvw-dl.iml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .idea/mvw-dl.iml diff --git a/.idea/mvw-dl.iml b/.idea/mvw-dl.iml deleted file mode 100644 index d9e6024..0000000 --- a/.idea/mvw-dl.iml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file -- 2.47.2 From 720493d0de98d7575b8dcee18f6a34b823f5e3d2 Mon Sep 17 00:00:00 2001 From: hygienic-books Date: Sun, 13 Mar 2022 17:17:44 +0100 Subject: [PATCH 3/3] Get config loading and parsing into usable state --- main.py | 78 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/main.py b/main.py index 99f0ba8..05f4c8a 100644 --- a/main.py +++ b/main.py @@ -32,18 +32,55 @@ log = logging.getLogger("rich") log.setLevel(logging.DEBUG) -config = configparser.ConfigParser() +class ConfigParser(configparser.ConfigParser): + """Can get options() without defaults + + Taken from https://stackoverflow.com/a/12600066. + """ + + def options(self, section, no_defaults=False, **kwargs): + if no_defaults: + try: + return list(self._sections[section].keys()) + except KeyError: + raise configparser.NoSectionError(section) + else: + return super().options(section) + + +ini_defaults = [] +internal_defaults = { + "self_name": "mvw-dl", + "tmp_base_dir": "/tmp/%(self_name)s", + "state_base_dir": "/var/lib/%(self_name)s", + "state_files_dir": "%(state_base_dir)s/state", + "state_file_retention": "50", + "state_file_name_prefix": "state-", + "state_file_name_suffix": ".log" +} +config = ConfigParser(defaults=internal_defaults) config.read(CONST.CFG_DEFAULT_FILENAME) -def validate_config_defaults(config_obj: configparser.ConfigParser()) -> None: +def print_section_header(header: str) -> str: + return f"Loading config section '[{header}]' ..." + + +def validate_default_section(config_obj: configparser.ConfigParser()) -> None: log.debug(f"Loading config from file '{CONST.CFG_DEFAULT_ABS_PATH}' ...") if not config_obj.sections(): log.error(f"No config sections found in '{CONST.CFG_DEFAULT_ABS_PATH}'. Exiting 1 ...") sys.exit(1) if config.defaults(): + log.debug(f"Symbol legend:\n" + f"* Global default from section '[{config_obj.default_section}]'\n" + f"~ Local option, doesn't exist in '[{config_obj.default_section}]'\n" + f"+ Local override of a value from '[{config_obj.default_section}]'\n" + f"= Local override, same value as in '[{config_obj.default_section}]'") + log.debug(print_section_header(config_obj.default_section)) for default in config_obj.defaults(): - log.debug(f" {default} = {config_obj[config_obj.default_section][default]}") + ini_defaults.append({default: config_obj[config_obj.default_section][default]}) + log.debug(f"* {default} = {config_obj[config_obj.default_section][default]}") else: log.debug(f"No defaults defined") @@ -57,22 +94,32 @@ def config_has_valid_section(config_obj: configparser.ConfigParser()) -> bool: return has_valid_section -#def option_has_default_value(config_option: configparser.ConfigParser(), section: str) -> bool: -# -#~ local option that doesn't exist in defaults section -#+ local option that overrides a default -#* global option +def is_default(config_key: str) -> bool: + return any(config_key in ini_default for ini_default in ini_defaults) + + +def is_same_as_default(config_kv_pair: dict) -> bool: + return config_kv_pair in ini_defaults + def validate_config_sections(config_obj: configparser.ConfigParser()) -> None: for section in config_obj.sections(): - log.debug(f"Loading config '[{section}]' ...") - for key in config_obj[section]: - # log.debug(config_obj.has_option(section, key)) - is_local_option = True if config_obj.has_option(section, key) else False - log.debug(f""" {key}{"/" if is_local_option else ".."} = {config_obj[section][key]}""") + log.debug(print_section_header(section)) + if CONST.CFG_MANDATORY not in config_obj.options(section, no_defaults=True): + log.warning(f"Config section '[{section}]' does not have mandatory option '{CONST.CFG_MANDATORY}' set, " + f"skipping section ...") + config_obj.remove_section(section) + else: + for key in config_obj.options(section, no_defaults=True): + kv_prefix = "~" + if is_default(key): + kv_prefix = "+" + if is_same_as_default({key: config_obj[section][key]}): + kv_prefix = "=" + log.debug(f"{kv_prefix} {key} = {config_obj[section][key]}") -validate_config_defaults(config) +validate_default_section(config) if config_has_valid_section(config): validate_config_sections(config) else: @@ -80,10 +127,9 @@ else: f"option set. Exiting 2 ...") sys.exit(2) - -# https://stackoverflow.com/a/2451872 quit() + # This is a sample Python script. # Press Umschalt+F10 to execute it or replace it with your code. -- 2.47.2