import configparser import json import logging import os import sys import requests from rich.logging import RichHandler from rich.traceback import install from rich.console import Console from rich.table import Table import typing as t console = Console() # We use Python 3.5+ type hints; we're working with JSON objects; we're following a 2016 suggestion from # Python's "typing" GitHub issue tracker on how to create a "JSONType" hint since such a thing does not yet # officially exist: https://github.com/python/typing/issues/182#issuecomment-186684288 JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any], t.List[t.Any]] # Exit codes # 1: Config file invalid, it has no sections # 2: Config file invalid, sections must define at least CONST.CFG_MANDATORY class CONST(object): __slots__ = () LOG_FORMAT = "%(message)s" CFG_THIS_FILE_DIRNAME = os.path.dirname(__file__) CFG_DEFAULT_FILENAME = "config.ini" CFG_DEFAULT_ABS_PATH = os.path.join(CFG_THIS_FILE_DIRNAME, CFG_DEFAULT_FILENAME) CFG_KNOWN_DEFAULTS = [ {"key": "self_name", "value": "mvw-dl"}, {"key": "tmp_base_dir", "value": os.path.join(CFG_THIS_FILE_DIRNAME, "data/tmp/%(self_name)s")}, {"key": "state_base_dir", "value": os.path.join(CFG_THIS_FILE_DIRNAME, "data/var/lib/%(self_name)s")}, {"key": "state_files_dir", "value": "%(state_base_dir)s/state"}, {"key": "state_file_retention", "value": "50"}, {"key": "state_file_name_prefix", "value": "state-"}, {"key": "state_file_name_suffix", "value": ".log"}, {"key": "mvw_endpoint", "value": "http://localhost:8000/api/query"}, {"key": "title_dedup_winner", "value": "first"} ] CFG_KNOWN_SECTION = [ {"key": "min_duration", "is_mandatory": False}, {"key": "max_duration", "is_mandatory": False}, {"key": "query", "is_mandatory": True}, {"key": "dl_dir", "is_mandatory": True} ] CFG_MANDATORY = [section_cfg["key"] for section_cfg in CFG_KNOWN_SECTION if section_cfg["is_mandatory"]] CONST = CONST() logging.basicConfig( level="NOTSET", format=CONST.LOG_FORMAT, datefmt="[%X]", handlers=[RichHandler( show_time=False if "SYSTEMD_EXEC_PID" in os.environ else True, rich_tracebacks=True )] ) log = logging.getLogger("rich") log.setLevel(logging.DEBUG) install(show_locals=True) class ConfigParser(configparser.ConfigParser): """Can get options() without defaults Taken from https://stackoverflow.com/a/12600066. """ def options(self, section, no_defaults=False, **kwargs): if no_defaults: try: return list(self._sections[section].keys()) except KeyError: raise configparser.NoSectionError(section) else: return super().options(section) ini_defaults = [] internal_defaults = {default["key"]: default["value"] for default in CONST.CFG_KNOWN_DEFAULTS} config = ConfigParser(defaults=internal_defaults) config.read(CONST.CFG_DEFAULT_FILENAME) def print_section_header(header: str) -> str: return f"Loading config section '[{header}]' ..." def validate_default_section(config_obj: configparser.ConfigParser()) -> None: log.debug(f"Loading config from file '{CONST.CFG_DEFAULT_ABS_PATH}' ...") if not config_obj.sections(): log.error(f"No config sections found in '{CONST.CFG_DEFAULT_ABS_PATH}'. Exiting 1 ...") sys.exit(1) if config.defaults(): log.debug(f"Symbol legend:\n" f"* Global default from section '[{config_obj.default_section}]'\n" f"~ Local option, doesn't exist in '[{config_obj.default_section}]'\n" f"+ Local override of a value from '[{config_obj.default_section}]'\n" f"= Local override, same value as in '[{config_obj.default_section}]'") log.debug(print_section_header(config_obj.default_section)) for default in config_obj.defaults(): ini_defaults.append({default: config_obj[config_obj.default_section][default]}) log.debug(f"* {default} = {config_obj[config_obj.default_section][default]}") else: log.debug(f"No defaults defined") def config_has_valid_section(config_obj: configparser.ConfigParser()) -> bool: has_valid_section = False for config_obj_section in config_obj.sections(): if set(CONST.CFG_MANDATORY).issubset(config_obj.options(config_obj_section)): has_valid_section = True break return has_valid_section def is_default(config_key: str) -> bool: return any(config_key in ini_default for ini_default in ini_defaults) def is_same_as_default(config_kv_pair: dict) -> bool: return config_kv_pair in ini_defaults def validate_config_sections(config_obj: configparser.ConfigParser()) -> None: for this_section in config_obj.sections(): log.debug(print_section_header(this_section)) if not set(CONST.CFG_MANDATORY).issubset(config_obj.options(this_section, no_defaults=True)): log.warning(f"Config section '[{this_section}]' does not have all mandatory options " f"{CONST.CFG_MANDATORY} set, skipping section ...") config_obj.remove_section(this_section) else: for key in config_obj.options(this_section, no_defaults=True): kv_prefix = "~" if is_default(key): kv_prefix = "+" if is_same_as_default({key: config_obj[this_section][key]}): kv_prefix = "=" log.debug(f"{kv_prefix} {key} = {config_obj[this_section][key]}") def query_string_from_file(filename: str) -> str: with open(filename, "r") as jsonfile: query_string = jsonfile.read() return query_string def get_query_payload(section_name: str, config_obj: configparser.ConfigParser()) -> JSONType: log.debug(f"Generating HTTP POST JSON payload ...") query = config_obj.get(section_name, "query") if query[0] == "@": query = query.split("@", 1)[1] query = query_string_from_file(query) return json.loads(query) def get_json_response(section_name: str, config_obj: configparser.ConfigParser(), payload: JSONType) -> JSONType: log.debug(f"Downloading JSON list of Mediathek files that match search criteria") url = config_obj.get(section_name, "mvw_endpoint") req_header = {"Content-Type": "text/plain", "asdasd": "aaaaaaaaaa"} s = requests.Session() req = requests.Request("POST", url, data=json.dumps(payload), headers=req_header) prepped = req.prepare() newline = "\n" #req.method #req.url #for header, value in list(req.headers.items()): # headers_table.add_row(header, value) quit() with s.send(prepped) as s: pass # log.debug(s.content) if __name__ == '__main__': validate_default_section(config) if config_has_valid_section(config): validate_config_sections(config) else: log.error(f"No valid config section found. A valid config section has at least the mandatory options " f"{CONST.CFG_MANDATORY} set. Exiting 2 ...") sys.exit(2) log.debug(f"Iterating over config sections ...") for section in config.sections(): log.debug(f"Processing section '[{section}]' ...") query_payload = get_query_payload(section, config) json_response = get_json_response(section, config, query_payload)