Get JSON query string from config

.gitignore

@@ -235,4 +235,7 @@ fabric.properties
 .idea/deployment.xml
 .idea/misc.xml
 .idea/remote-mappings.xml
-.idea/mvw-dl.iml
+.idea/*.iml
+
+# ---> Project-specific
+data

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml

.idea/inspectionProfiles/Project_Default.xml

@@ -1,26 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <profile version="1.0">
-    <option name="myName" value="Project Default" />
-    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="ignoredPackages">
-        <value>
-          <list size="1">
-            <item index="0" class="java.lang.String" itemvalue="google" />
-          </list>
-        </value>
-      </option>
-    </inspection_tool>
-    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="ignoredIdentifiers">
-        <list>
-          <option value="str.decode" />
-        </list>
-      </option>
-    </inspection_tool>
-    <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
-      <option name="processCode" value="true" />
-      <option name="processLiterals" value="true" />
-      <option name="processComments" value="true" />
-    </inspection_tool>
-  </profile>
-</component>

config.ini

@@ -7,6 +7,7 @@ state_file_retention = 50
 state_file_name_prefix = state-
 state_file_name_suffix = .log
 mvw_endpoint = http://localhost:8000/api/query
+title_dedup_winner = first
 
 [maus]
 min_duration = 1200
@@ -15,8 +16,10 @@ query = @maus-query.json
 # query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50}
 # state_file_name = maus
 # tmp_base_dir = %(tmp_base_dir)s/maus
+dl_dir = ~/maus
 
 [test]
 min_duration = 100
 max_duration = 200
 query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50}
+dl_dir = test

main.py

@@ -1,8 +1,20 @@
 import configparser
+import json
 import logging
 import os
 import sys
+import requests
 from rich.logging import RichHandler
+from rich.traceback import install
+from rich.console import Console
+from rich.table import Table
+import typing as t
+console = Console()
+
+# We use Python 3.5+ type hints; we're working with JSON objects; we're following a 2016 suggestion from
+# Python's "typing" GitHub issue tracker on how to create a "JSONType" hint since such a thing does not yet
+# officially exist: https://github.com/python/typing/issues/182#issuecomment-186684288
+JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any], t.List[t.Any]]
 
 
 # Exit codes
@@ -13,9 +25,27 @@ from rich.logging import RichHandler
 class CONST(object):
     __slots__ = ()
     LOG_FORMAT = "%(message)s"
+    CFG_THIS_FILE_DIRNAME = os.path.dirname(__file__)
     CFG_DEFAULT_FILENAME = "config.ini"
-    CFG_DEFAULT_ABS_PATH = os.path.join(os.getcwd(), CFG_DEFAULT_FILENAME)
+    CFG_DEFAULT_ABS_PATH = os.path.join(CFG_THIS_FILE_DIRNAME, CFG_DEFAULT_FILENAME)
-    CFG_MANDATORY = "query"
+    CFG_KNOWN_DEFAULTS = [
+        {"key": "self_name", "value": "mvw-dl"},
+        {"key": "tmp_base_dir", "value": os.path.join(CFG_THIS_FILE_DIRNAME, "data/tmp/%(self_name)s")},
+        {"key": "state_base_dir", "value": os.path.join(CFG_THIS_FILE_DIRNAME, "data/var/lib/%(self_name)s")},
+        {"key": "state_files_dir", "value": "%(state_base_dir)s/state"},
+        {"key": "state_file_retention", "value": "50"},
+        {"key": "state_file_name_prefix", "value": "state-"},
+        {"key": "state_file_name_suffix", "value": ".log"},
+        {"key": "mvw_endpoint", "value": "http://localhost:8000/api/query"},
+        {"key": "title_dedup_winner", "value": "first"}
+    ]
+    CFG_KNOWN_SECTION = [
+        {"key": "min_duration", "is_mandatory": False},
+        {"key": "max_duration", "is_mandatory": False},
+        {"key": "query", "is_mandatory": True},
+        {"key": "dl_dir", "is_mandatory": True}
+    ]
+    CFG_MANDATORY = [section_cfg["key"] for section_cfg in CFG_KNOWN_SECTION if section_cfg["is_mandatory"]]
 
 
 CONST = CONST()
@@ -30,6 +60,7 @@ logging.basicConfig(
 )
 log = logging.getLogger("rich")
 log.setLevel(logging.DEBUG)
+install(show_locals=True)
 
 
 class ConfigParser(configparser.ConfigParser):
@@ -49,16 +80,7 @@ class ConfigParser(configparser.ConfigParser):
 
 
 ini_defaults = []
-internal_defaults = {
+internal_defaults = {default["key"]: default["value"] for default in CONST.CFG_KNOWN_DEFAULTS}
-    "self_name": "mvw-dl",
-    "tmp_base_dir": "/tmp/%(self_name)s",
-    "state_base_dir": "/var/lib/%(self_name)s",
-    "state_files_dir": "%(state_base_dir)s/state",
-    "state_file_retention": "50",
-    "state_file_name_prefix": "state-",
-    "state_file_name_suffix": ".log",
-    "mvw_endpoint": "http://localhost:8000/api/query"
-}
 config = ConfigParser(defaults=internal_defaults)
 config.read(CONST.CFG_DEFAULT_FILENAME)
 
@@ -89,7 +111,7 @@ def validate_default_section(config_obj: configparser.ConfigParser()) -> None:
 def config_has_valid_section(config_obj: configparser.ConfigParser()) -> bool:
     has_valid_section = False
     for config_obj_section in config_obj.sections():
-        if CONST.CFG_MANDATORY in config_obj.options(config_obj_section):
+        if set(CONST.CFG_MANDATORY).issubset(config_obj.options(config_obj_section)):
             has_valid_section = True
             break
     return has_valid_section
@@ -104,46 +126,68 @@ def is_same_as_default(config_kv_pair: dict) -> bool:
 
 
 def validate_config_sections(config_obj: configparser.ConfigParser()) -> None:
-    for section in config_obj.sections():
+    for this_section in config_obj.sections():
-        log.debug(print_section_header(section))
+        log.debug(print_section_header(this_section))
-        if CONST.CFG_MANDATORY not in config_obj.options(section, no_defaults=True):
+        if not set(CONST.CFG_MANDATORY).issubset(config_obj.options(this_section, no_defaults=True)):
-            log.warning(f"Config section '[{section}]' does not have mandatory option '{CONST.CFG_MANDATORY}' set, "
+            log.warning(f"Config section '[{this_section}]' does not have all mandatory options "
-                        f"skipping section ...")
+                        f"{CONST.CFG_MANDATORY} set, skipping section ...")
-            config_obj.remove_section(section)
+            config_obj.remove_section(this_section)
         else:
-            for key in config_obj.options(section, no_defaults=True):
+            for key in config_obj.options(this_section, no_defaults=True):
                 kv_prefix = "~"
                 if is_default(key):
                     kv_prefix = "+"
-                    if is_same_as_default({key: config_obj[section][key]}):
+                    if is_same_as_default({key: config_obj[this_section][key]}):
                         kv_prefix = "="
-                log.debug(f"{kv_prefix} {key} = {config_obj[section][key]}")
+                log.debug(f"{kv_prefix} {key} = {config_obj[this_section][key]}")
 
 
-validate_default_section(config)
+def query_string_from_file(filename: str) -> str:
-if config_has_valid_section(config):
+    with open(filename, "r") as jsonfile:
-    validate_config_sections(config)
+        query_string = jsonfile.read()
-else:
+        return query_string
-    log.error(f"No valid config section found. A valid config section has at least the '{CONST.CFG_MANDATORY}' "
-              f"option set. Exiting 2 ...")
-    sys.exit(2)
-
-quit()
 
 
-# This is a sample Python script.
+def get_query_payload(section_name: str, config_obj: configparser.ConfigParser()) -> JSONType:
-
+    log.debug(f"Generating HTTP POST JSON payload ...")
-# Press Umschalt+F10 to execute it or replace it with your code.
+    query = config_obj.get(section_name, "query")
-# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
+    if query[0] == "@":
+        query = query.split("@", 1)[1]
+        query = query_string_from_file(query)
+    return json.loads(query)
 
 
-def print_hi(name):
+def get_json_response(section_name: str, config_obj: configparser.ConfigParser(), payload: JSONType) -> JSONType:
-    # Use a breakpoint in the code line below to debug your script.
+    log.debug(f"Downloading JSON list of Mediathek files that match search criteria")
-    print(f'Hi, {name}')  # Press Strg+F8 to toggle the breakpoint.
+    url = config_obj.get(section_name, "mvw_endpoint")
+    req_header = {"Content-Type": "text/plain", "asdasd": "aaaaaaaaaa"}
+    s = requests.Session()
+    req = requests.Request("POST", url, data=json.dumps(payload), headers=req_header)
+    prepped = req.prepare()
+    newline = "\n"
+    #req.method
+    #req.url
+    #for header, value in list(req.headers.items()):
+    #    headers_table.add_row(header, value)
+
+    quit()
+    with s.send(prepped) as s:
+        pass
+        # log.debug(s.content)
 
 
-# Press the green button in the gutter to run the script.
 if __name__ == '__main__':
-    print_hi('PyCharm')
+    validate_default_section(config)
+    if config_has_valid_section(config):
+        validate_config_sections(config)
+    else:
+        log.error(f"No valid config section found. A valid config section has at least the mandatory options "
+                  f"{CONST.CFG_MANDATORY} set. Exiting 2 ...")
+        sys.exit(2)
+
+    log.debug(f"Iterating over config sections ...")
+    for section in config.sections():
+        log.debug(f"Processing section '[{section}]' ...")
+        query_payload = get_query_payload(section, config)
+        json_response = get_json_response(section, config, query_payload)
 
-# See PyCharm help at https://www.jetbrains.com/help/pycharm/

maus-query.json

@@ -0,0 +1,21 @@
+{
+    "queries": [
+        {
+            "fields": [
+                "topic"
+            ],
+            "query": "die sendung mit der maus"
+        },
+        {
+            "fields": [
+                "channel"
+            ],
+            "query": "ARD"
+        }
+    ],
+    "sortBy": "timestamp",
+    "sortOrder": "desc",
+    "future": false,
+    "offset": 0,
+    "size": 20
+}

requirements.in

@@ -1 +1,2 @@
 rich
+requests

requirements.txt

@@ -4,9 +4,19 @@
 #
 #    pip-compile
 #
+certifi==2021.10.8
+    # via requests
+charset-normalizer==2.0.12
+    # via requests
 commonmark==0.9.1
     # via rich
+idna==3.3
+    # via requests
 pygments==2.11.2
     # via rich
+requests==2.27.1
+    # via -r requirements.in
 rich==12.0.0
     # via -r requirements.in
+urllib3==1.26.8
+    # via requests