From 98d307db73d9f930f554d47ef9f754194dbc99cd Mon Sep 17 00:00:00 2001 From: hygienic-books Date: Mon, 14 Mar 2022 15:06:57 +0100 Subject: [PATCH] Get JSON response body from MediathekViewWeb --- config.ini | 12 +++++------ main.py | 55 +++++++++++++++++++++++++++++++++++++----------- maus-query.json | 2 +- requirements.in | 3 ++- requirements.txt | 2 ++ 5 files changed, 54 insertions(+), 20 deletions(-) diff --git a/config.ini b/config.ini index e831e56..1e1046f 100644 --- a/config.ini +++ b/config.ini @@ -10,7 +10,7 @@ mvw_endpoint = http://localhost:8000/api/query title_dedup_winner = first [maus] -min_duration = 1200 +min_duration = 1800 max_duration = 2700 query = @maus-query.json # query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50} @@ -18,8 +18,8 @@ query = @maus-query.json # tmp_base_dir = %(tmp_base_dir)s/maus dl_dir = ~/maus -[test] -min_duration = 100 -max_duration = 200 -query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50} -dl_dir = test \ No newline at end of file +#[test] +#min_duration = 100 +#max_duration = 200 +#query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50} +#dl_dir = test \ No newline at end of file diff --git a/main.py b/main.py index 72b51bf..d7b44b0 100644 --- a/main.py +++ b/main.py @@ -4,16 +4,20 @@ import logging import os import sys import requests +import inflect from rich.logging import RichHandler from rich.traceback import install -from rich.console import Console -from rich.table import Table +from rich import print import typing as t -console = Console() +from rich.console import Console +# Without width +console = Console(width=180) +p = inflect.engine() # We use Python 3.5+ type hints; we're working with JSON objects; we're following a 2016 suggestion from # Python's "typing" GitHub issue tracker on how to create a "JSONType" hint since such a thing does not yet # officially exist: https://github.com/python/typing/issues/182#issuecomment-186684288 +# JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any], t.List[t.Any]] JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any], t.List[t.Any]] @@ -42,6 +46,7 @@ class CONST(object): CFG_KNOWN_SECTION = [ {"key": "min_duration", "is_mandatory": False}, {"key": "max_duration", "is_mandatory": False}, + {"key": "title_not_regex", "is_mandatory": False}, {"key": "query", "is_mandatory": True}, {"key": "dl_dir", "is_mandatory": True} ] @@ -50,6 +55,7 @@ class CONST(object): CONST = CONST() logging.basicConfig( + # Default for all modules in NOTSET so log everything level="NOTSET", format=CONST.LOG_FORMAT, datefmt="[%X]", @@ -59,7 +65,11 @@ logging.basicConfig( )] ) log = logging.getLogger("rich") +# Our own code logs with this level log.setLevel(logging.DEBUG) +# connectionpool logs with WARNING, we don't need its verbosity +log_connectionpool = logging.getLogger("urllib3.connectionpool") +log_connectionpool.setLevel(logging.WARNING) install(show_locals=True) @@ -159,21 +169,38 @@ def get_query_payload(section_name: str, config_obj: configparser.ConfigParser() def get_json_response(section_name: str, config_obj: configparser.ConfigParser(), payload: JSONType) -> JSONType: log.debug(f"Downloading JSON list of Mediathek files that match search criteria") + serialized_payload = json.dumps(payload) url = config_obj.get(section_name, "mvw_endpoint") - req_header = {"Content-Type": "text/plain", "asdasd": "aaaaaaaaaa"} + req_header = {"Content-Type": "text/plain"} s = requests.Session() - req = requests.Request("POST", url, data=json.dumps(payload), headers=req_header) + req = requests.Request("POST", url, data=serialized_payload, headers=req_header) prepped = req.prepare() newline = "\n" - #req.method - #req.url - #for header, value in list(req.headers.items()): - # headers_table.add_row(header, value) - - quit() + log.debug(f"Request method: {req.method}\n" + f"URL: {req.url}\n" + f"""{newline.join(f"Header '{header}': '{value}'" for header, value in list(req.headers.items()))}\n""" + f"Payload: {payload}") with s.send(prepped) as s: + return json.loads(s.content) + + +def filter_json_by_duration(section_name: str, config_obj: configparser.ConfigParser(), json_obj: JSONType) -> JSONType: + min_length = config_obj.getint(section_name, "min_duration") + if min_length >= 0: + log.debug(f"""Filtering JSON for minimum length of {min_length} {p.plural("second", min_length)}""") + # console.print(json_obj["result"]["results"][0]) + for result in json_obj["result"]["results"]: + console.log(result) + console.log(f"0000000000000000000000") + if not result["duration"] >= min_length: + pass + # json_str. + # console.log(f"{result}\n......................") + # console.log(json_obj) + # console.log(f"ssssssssss") + + # json_matches_min_length = pass - # log.debug(s.content) if __name__ == '__main__': @@ -190,4 +217,8 @@ if __name__ == '__main__': log.debug(f"Processing section '[{section}]' ...") query_payload = get_query_payload(section, config) json_response = get_json_response(section, config, query_payload) + log.debug(CONST.CFG_KNOWN_SECTION[0]) + if config.has_option(section, "min_duration") or config.has_option(section, "max_duration"): + json_matches_duration = filter_json_by_duration(section, config, json_response) + # console.log(json_response) diff --git a/maus-query.json b/maus-query.json index 7703479..2a45639 100644 --- a/maus-query.json +++ b/maus-query.json @@ -17,5 +17,5 @@ "sortOrder": "desc", "future": false, "offset": 0, - "size": 20 + "size": 8 } diff --git a/requirements.in b/requirements.in index 3bfa0c6..1ea9cba 100644 --- a/requirements.in +++ b/requirements.in @@ -1,2 +1,3 @@ rich -requests \ No newline at end of file +requests +inflect \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 954e9c2..943c542 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,8 @@ commonmark==0.9.1 # via rich idna==3.3 # via requests +inflect==5.4.0 + # via -r requirements.in pygments==2.11.2 # via rich requests==2.27.1