.gitignore

@@ -235,7 +235,4 @@ fabric.properties
 .idea/deployment.xml
 .idea/misc.xml
 .idea/remote-mappings.xml
-.idea/*.iml
+.idea/mvw-dl.iml
-
-# ---> Project-specific
-data

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,26 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="1">
+            <item index="0" class="java.lang.String" itemvalue="google" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredIdentifiers">
+        <list>
+          <option value="str.decode" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
+      <option name="processCode" value="true" />
+      <option name="processLiterals" value="true" />
+      <option name="processComments" value="true" />
+    </inspection_tool>
+  </profile>
+</component>

config.ini

@@ -7,7 +7,6 @@ state_file_retention = 50
 state_file_name_prefix = state-
 state_file_name_suffix = .log
 mvw_endpoint = http://localhost:8000/api/query
-title_dedup_winner = first
 
 [maus]
 min_duration = 1200
@@ -16,10 +15,8 @@ query = @maus-query.json
 # query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50}
 # state_file_name = maus
 # tmp_base_dir = %(tmp_base_dir)s/maus
-dl_dir = ~/maus
 
 [test]
 min_duration = 100
 max_duration = 200
-query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50}
+query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50}
-dl_dir = test

main.py

@@ -1,20 +1,8 @@
 import configparser
-import json
 import logging
 import os
 import sys
-import requests
 from rich.logging import RichHandler
-from rich.traceback import install
-from rich.console import Console
-from rich.table import Table
-import typing as t
-console = Console()
-
-# We use Python 3.5+ type hints; we're working with JSON objects; we're following a 2016 suggestion from
-# Python's "typing" GitHub issue tracker on how to create a "JSONType" hint since such a thing does not yet
-# officially exist: https://github.com/python/typing/issues/182#issuecomment-186684288
-JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any], t.List[t.Any]]
 
 
 # Exit codes
@@ -25,27 +13,9 @@ JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any], t.List[t.Any
 class CONST(object):
     __slots__ = ()
     LOG_FORMAT = "%(message)s"
-    CFG_THIS_FILE_DIRNAME = os.path.dirname(__file__)
     CFG_DEFAULT_FILENAME = "config.ini"
-    CFG_DEFAULT_ABS_PATH = os.path.join(CFG_THIS_FILE_DIRNAME, CFG_DEFAULT_FILENAME)
+    CFG_DEFAULT_ABS_PATH = os.path.join(os.getcwd(), CFG_DEFAULT_FILENAME)
-    CFG_KNOWN_DEFAULTS = [
+    CFG_MANDATORY = "query"
-        {"key": "self_name", "value": "mvw-dl"},
-        {"key": "tmp_base_dir", "value": os.path.join(CFG_THIS_FILE_DIRNAME, "data/tmp/%(self_name)s")},
-        {"key": "state_base_dir", "value": os.path.join(CFG_THIS_FILE_DIRNAME, "data/var/lib/%(self_name)s")},
-        {"key": "state_files_dir", "value": "%(state_base_dir)s/state"},
-        {"key": "state_file_retention", "value": "50"},
-        {"key": "state_file_name_prefix", "value": "state-"},
-        {"key": "state_file_name_suffix", "value": ".log"},
-        {"key": "mvw_endpoint", "value": "http://localhost:8000/api/query"},
-        {"key": "title_dedup_winner", "value": "first"}
-    ]
-    CFG_KNOWN_SECTION = [
-        {"key": "min_duration", "is_mandatory": False},
-        {"key": "max_duration", "is_mandatory": False},
-        {"key": "query", "is_mandatory": True},
-        {"key": "dl_dir", "is_mandatory": True}
-    ]
-    CFG_MANDATORY = [section_cfg["key"] for section_cfg in CFG_KNOWN_SECTION if section_cfg["is_mandatory"]]
 
 
 CONST = CONST()
@@ -60,7 +30,6 @@ logging.basicConfig(
 )
 log = logging.getLogger("rich")
 log.setLevel(logging.DEBUG)
-install(show_locals=True)
 
 
 class ConfigParser(configparser.ConfigParser):
@@ -80,7 +49,16 @@ class ConfigParser(configparser.ConfigParser):
 
 
 ini_defaults = []
-internal_defaults = {default["key"]: default["value"] for default in CONST.CFG_KNOWN_DEFAULTS}
+internal_defaults = {
+    "self_name": "mvw-dl",
+    "tmp_base_dir": "/tmp/%(self_name)s",
+    "state_base_dir": "/var/lib/%(self_name)s",
+    "state_files_dir": "%(state_base_dir)s/state",
+    "state_file_retention": "50",
+    "state_file_name_prefix": "state-",
+    "state_file_name_suffix": ".log",
+    "mvw_endpoint": "http://localhost:8000/api/query"
+}
 config = ConfigParser(defaults=internal_defaults)
 config.read(CONST.CFG_DEFAULT_FILENAME)
 
@@ -111,7 +89,7 @@ def validate_default_section(config_obj: configparser.ConfigParser()) -> None:
 def config_has_valid_section(config_obj: configparser.ConfigParser()) -> bool:
     has_valid_section = False
     for config_obj_section in config_obj.sections():
-        if set(CONST.CFG_MANDATORY).issubset(config_obj.options(config_obj_section)):
+        if CONST.CFG_MANDATORY in config_obj.options(config_obj_section):
             has_valid_section = True
             break
     return has_valid_section
@@ -126,68 +104,46 @@ def is_same_as_default(config_kv_pair: dict) -> bool:
 
 
 def validate_config_sections(config_obj: configparser.ConfigParser()) -> None:
-    for this_section in config_obj.sections():
+    for section in config_obj.sections():
-        log.debug(print_section_header(this_section))
+        log.debug(print_section_header(section))
-        if not set(CONST.CFG_MANDATORY).issubset(config_obj.options(this_section, no_defaults=True)):
+        if CONST.CFG_MANDATORY not in config_obj.options(section, no_defaults=True):
-            log.warning(f"Config section '[{this_section}]' does not have all mandatory options "
+            log.warning(f"Config section '[{section}]' does not have mandatory option '{CONST.CFG_MANDATORY}' set, "
-                        f"{CONST.CFG_MANDATORY} set, skipping section ...")
+                        f"skipping section ...")
-            config_obj.remove_section(this_section)
+            config_obj.remove_section(section)
         else:
-            for key in config_obj.options(this_section, no_defaults=True):
+            for key in config_obj.options(section, no_defaults=True):
                 kv_prefix = "~"
                 if is_default(key):
                     kv_prefix = "+"
-                    if is_same_as_default({key: config_obj[this_section][key]}):
+                    if is_same_as_default({key: config_obj[section][key]}):
                         kv_prefix = "="
-                log.debug(f"{kv_prefix} {key} = {config_obj[this_section][key]}")
+                log.debug(f"{kv_prefix} {key} = {config_obj[section][key]}")
 
 
-def query_string_from_file(filename: str) -> str:
+validate_default_section(config)
-    with open(filename, "r") as jsonfile:
+if config_has_valid_section(config):
-        query_string = jsonfile.read()
+    validate_config_sections(config)
-        return query_string
+else:
+    log.error(f"No valid config section found. A valid config section has at least the '{CONST.CFG_MANDATORY}' "
+              f"option set. Exiting 2 ...")
+    sys.exit(2)
+
+quit()
 
 
-def get_query_payload(section_name: str, config_obj: configparser.ConfigParser()) -> JSONType:
+# This is a sample Python script.
-    log.debug(f"Generating HTTP POST JSON payload ...")
+
-    query = config_obj.get(section_name, "query")
+# Press Umschalt+F10 to execute it or replace it with your code.
-    if query[0] == "@":
+# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
-        query = query.split("@", 1)[1]
-        query = query_string_from_file(query)
-    return json.loads(query)
 
 
-def get_json_response(section_name: str, config_obj: configparser.ConfigParser(), payload: JSONType) -> JSONType:
+def print_hi(name):
-    log.debug(f"Downloading JSON list of Mediathek files that match search criteria")
+    # Use a breakpoint in the code line below to debug your script.
-    url = config_obj.get(section_name, "mvw_endpoint")
+    print(f'Hi, {name}')  # Press Strg+F8 to toggle the breakpoint.
-    req_header = {"Content-Type": "text/plain", "asdasd": "aaaaaaaaaa"}
-    s = requests.Session()
-    req = requests.Request("POST", url, data=json.dumps(payload), headers=req_header)
-    prepped = req.prepare()
-    newline = "\n"
-    #req.method
-    #req.url
-    #for header, value in list(req.headers.items()):
-    #    headers_table.add_row(header, value)
-
-    quit()
-    with s.send(prepped) as s:
-        pass
-        # log.debug(s.content)
 
 
+# Press the green button in the gutter to run the script.
 if __name__ == '__main__':
-    validate_default_section(config)
+    print_hi('PyCharm')
-    if config_has_valid_section(config):
-        validate_config_sections(config)
-    else:
-        log.error(f"No valid config section found. A valid config section has at least the mandatory options "
-                  f"{CONST.CFG_MANDATORY} set. Exiting 2 ...")
-        sys.exit(2)
-
-    log.debug(f"Iterating over config sections ...")
-    for section in config.sections():
-        log.debug(f"Processing section '[{section}]' ...")
-        query_payload = get_query_payload(section, config)
-        json_response = get_json_response(section, config, query_payload)
 
+# See PyCharm help at https://www.jetbrains.com/help/pycharm/

maus-query.json

@@ -1,21 +0,0 @@
-{
-    "queries": [
-        {
-            "fields": [
-                "topic"
-            ],
-            "query": "die sendung mit der maus"
-        },
-        {
-            "fields": [
-                "channel"
-            ],
-            "query": "ARD"
-        }
-    ],
-    "sortBy": "timestamp",
-    "sortOrder": "desc",
-    "future": false,
-    "offset": 0,
-    "size": 20
-}

requirements.in

@@ -1,2 +1 @@
-rich
+rich
-requests

requirements.txt

@@ -4,19 +4,9 @@
 #
 #    pip-compile
 #
-certifi==2021.10.8
-    # via requests
-charset-normalizer==2.0.12
-    # via requests
 commonmark==0.9.1
     # via rich
-idna==3.3
-    # via requests
 pygments==2.11.2
     # via rich
-requests==2.27.1
-    # via -r requirements.in
 rich==12.0.0
     # via -r requirements.in
-urllib3==1.26.8
-    # via requests