Compare commits
8 Commits
287a755e65
...
03b449c768
Author | SHA1 | Date | |
---|---|---|---|
03b449c768 | |||
e269a110a6 | |||
a3a375d142 | |||
81ce5812a6 | |||
83921912a4 | |||
65e3ec83b1 | |||
266d3189dc | |||
563ff4d342 |
34
config.ini
34
config.ini
@ -1,34 +0,0 @@
|
|||||||
[DEFAULT]
|
|
||||||
self_name = mvw-dl
|
|
||||||
tmp_base_dir = /tmp/%(self_name)s
|
|
||||||
state_base_dir = /var/lib/%(self_name)s
|
|
||||||
state_files_dir = %(state_base_dir)s/state
|
|
||||||
state_file_retention = 50
|
|
||||||
state_file_name_prefix = state-
|
|
||||||
state_file_name_suffix = .log
|
|
||||||
mvw_endpoint = http://localhost:8000/api/query
|
|
||||||
title_dedup_winner = first
|
|
||||||
dl_progress_update_interval = 10
|
|
||||||
dl_threads = 2
|
|
||||||
dl_filename_pattern = &(channel)s - &(publish_date)s - &(topic)s - &(title)s.&(ext)s
|
|
||||||
publish_date_srtftime_pattern = %%Y%%m%%d
|
|
||||||
dl_filename_replace_spaces_with =
|
|
||||||
dl_filename_all_lowercase = no
|
|
||||||
|
|
||||||
[maus]
|
|
||||||
min_duration = 1200
|
|
||||||
max_duration = 2700
|
|
||||||
query = @maus-query.json
|
|
||||||
title_not_regex = audiodeskription|gebärdensprache|hörfassung
|
|
||||||
# dl_filename_pattern = &(publish_date)s.&(ext)s
|
|
||||||
# publish_date_srtftime_pattern = S%%YE%%Y%%m%%d01
|
|
||||||
# query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50}
|
|
||||||
# state_file_name = maus
|
|
||||||
# tmp_base_dir = %(tmp_base_dir)s/maus
|
|
||||||
dl_dir = ~/maus
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
#min_duration = 100
|
|
||||||
#max_duration = 200
|
|
||||||
#query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50}
|
|
||||||
#dl_dir = test
|
|
25
examples/config.ini.example
Normal file
25
examples/config.ini.example
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
[DEFAULT]
|
||||||
|
self_name = mvw-dl
|
||||||
|
tmp_base_dir = /tmp/%(self_name)s
|
||||||
|
state_base_dir = /var/lib/%(self_name)s
|
||||||
|
state_files_dir = %(state_base_dir)s/state
|
||||||
|
state_file_retention = 50
|
||||||
|
state_file_name_prefix = state-
|
||||||
|
state_file_name_suffix = .log
|
||||||
|
mvw_endpoint = http://localhost:8000/api/query
|
||||||
|
title_dedup_winner = first
|
||||||
|
dl_progress_update_interval = 10
|
||||||
|
dl_threads = 2
|
||||||
|
dl_filename_pattern = &(channel)s - &(publish_date)s - &(topic)s - &(title)s.&(ext)s
|
||||||
|
publish_date_srtftime_pattern = %%Y%%m%%d
|
||||||
|
dl_filename_replace_spaces_with =
|
||||||
|
dl_filename_all_lowercase = no
|
||||||
|
|
||||||
|
[maus]
|
||||||
|
min_duration = 1200
|
||||||
|
max_duration = 3000
|
||||||
|
query = @maus-query.json.example
|
||||||
|
title_not_regex = audiodeskription|gebärdensprache|hörfassung
|
||||||
|
dl_filename_pattern = &(publish_date)s.&(ext)s
|
||||||
|
publish_date_srtftime_pattern = S%%YE%%Y%%m%%d01
|
||||||
|
dl_dir = /tmp/kodi-nfo-feeder/maus
|
@ -17,5 +17,5 @@
|
|||||||
"sortOrder": "desc",
|
"sortOrder": "desc",
|
||||||
"future": false,
|
"future": false,
|
||||||
"offset": 0,
|
"offset": 0,
|
||||||
"size": 100
|
"size": 20
|
||||||
}
|
}
|
12
examples/mvw-dl.service.example
Normal file
12
examples/mvw-dl.service.example
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=MediathekViewWeb download helper
|
||||||
|
After=multi-user.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
RemainAfterExit=no
|
||||||
|
Environment='PATH=/usr/local/sbin:/usr/local/bin:/usr/bin'
|
||||||
|
ExecStart=/opt/miniconda3/envs/mvw-dl/bin/python /opt/python/mvw-dl/dev/mvw-dl.py
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
10
examples/mvw-dl.timer.example
Normal file
10
examples/mvw-dl.timer.example
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Run MediathekViewWeb download helper
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=0/2:2
|
||||||
|
OnCalendar=0/1:20
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
57
mvw-dl.py
57
mvw-dl.py
@ -46,6 +46,11 @@ from rich.progress import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
# TODO set locale for datetime and others to globally stick to en_US
|
# TODO set locale for datetime and others to globally stick to en_US
|
||||||
|
# TODO thread log messages display timestamp in systemd journal
|
||||||
|
# TODO Increment file name suffix more than once of needed
|
||||||
|
# TODO [23:15:14] DEBUG [thread]
|
||||||
|
# TODO Clean mvw-dl.timer
|
||||||
|
# TODO Reset maus-query.json
|
||||||
download_start_time = 0
|
download_start_time = 0
|
||||||
download_last_update_time = 0
|
download_last_update_time = 0
|
||||||
total_content_length = 0
|
total_content_length = 0
|
||||||
@ -84,6 +89,7 @@ JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any], t.List[t.Any
|
|||||||
# 3: No search results to download
|
# 3: No search results to download
|
||||||
# 4: State file already exists, has more than 0 bytes size but doesn't contain usable JSON
|
# 4: State file already exists, has more than 0 bytes size but doesn't contain usable JSON
|
||||||
# 5: State file lock cannot be acquired within file_lock_timeout
|
# 5: State file lock cannot be acquired within file_lock_timeout
|
||||||
|
# 6: Unable to create state directory
|
||||||
|
|
||||||
|
|
||||||
class CONST(object):
|
class CONST(object):
|
||||||
@ -159,7 +165,7 @@ class ConfigParser(
|
|||||||
ini_defaults = []
|
ini_defaults = []
|
||||||
internal_defaults = {default["key"]: default["value"] for default in CONST.CFG_KNOWN_DEFAULTS}
|
internal_defaults = {default["key"]: default["value"] for default in CONST.CFG_KNOWN_DEFAULTS}
|
||||||
config = ConfigParser(defaults=internal_defaults)
|
config = ConfigParser(defaults=internal_defaults)
|
||||||
config.read(CONST.CFG_DEFAULT_FILENAME)
|
config.read(CONST.CFG_DEFAULT_ABS_PATH)
|
||||||
|
|
||||||
|
|
||||||
def print_section_header(
|
def print_section_header(
|
||||||
@ -227,7 +233,8 @@ def validate_config_sections(
|
|||||||
|
|
||||||
def query_string_from_file(
|
def query_string_from_file(
|
||||||
filename: str) -> str:
|
filename: str) -> str:
|
||||||
with open(filename, "r", encoding="utf-8") as jsonfile:
|
filename_abs_path = os.path.join(CONST.CFG_THIS_FILE_DIRNAME, filename)
|
||||||
|
with open(filename_abs_path, "r", encoding="utf-8") as jsonfile:
|
||||||
query_string = jsonfile.read()
|
query_string = jsonfile.read()
|
||||||
return query_string
|
return query_string
|
||||||
|
|
||||||
@ -393,8 +400,11 @@ def get_safe_filename(
|
|||||||
shorthand_uuid: str) -> str:
|
shorthand_uuid: str) -> str:
|
||||||
"""https://stackoverflow.com/a/71199182"""
|
"""https://stackoverflow.com/a/71199182"""
|
||||||
|
|
||||||
|
log.debug(f"{shorthand_uuid} Removing question marks from file name ...")
|
||||||
|
clean_filename = re.sub(r"""[?]""", "", dirty_filename)
|
||||||
|
|
||||||
log.debug(f"{shorthand_uuid} Replacing unsafe characters in filename with dashes ...")
|
log.debug(f"{shorthand_uuid} Replacing unsafe characters in filename with dashes ...")
|
||||||
clean_filename = re.sub(r"""[/\\?%*:|"<>\x7F\x00-\x1F]""", "-", dirty_filename)
|
clean_filename = re.sub(r"""[/\\?%*:|"<>\x7F\x00-\x1F]""", "-", clean_filename)
|
||||||
|
|
||||||
log.debug(f"{shorthand_uuid} New filename: '{clean_filename}'")
|
log.debug(f"{shorthand_uuid} New filename: '{clean_filename}'")
|
||||||
return clean_filename
|
return clean_filename
|
||||||
@ -456,12 +466,19 @@ def get_state_file_abs_path(
|
|||||||
config_obj: configparser.ConfigParser()) -> str:
|
config_obj: configparser.ConfigParser()) -> str:
|
||||||
|
|
||||||
state_dir = config_obj.get(section_name, "state_files_dir")
|
state_dir = config_obj.get(section_name, "state_files_dir")
|
||||||
state_file = \
|
try:
|
||||||
config_obj.get(section_name, "state_file_name_prefix") + \
|
os.makedirs(state_dir, exist_ok=True)
|
||||||
section_name + \
|
except OSError:
|
||||||
config_obj.get(section_name, "state_file_name_suffix")
|
log.error(f"Unable to create '[{section}]' state directory '{state_dir}'. "
|
||||||
state_file_abs_path = os.path.join(state_dir, state_file)
|
f"We're not going to be able to log state information. Exiting 6 ...")
|
||||||
return state_file_abs_path
|
sys.exit(6)
|
||||||
|
else:
|
||||||
|
state_file = \
|
||||||
|
config_obj.get(section_name, "state_file_name_prefix") + \
|
||||||
|
section_name + \
|
||||||
|
config_obj.get(section_name, "state_file_name_suffix")
|
||||||
|
state_file_abs_path = os.path.join(state_dir, state_file)
|
||||||
|
return state_file_abs_path
|
||||||
|
|
||||||
|
|
||||||
def state_file_none_or_valid_json(
|
def state_file_none_or_valid_json(
|
||||||
@ -529,7 +546,8 @@ def log_successful_download(
|
|||||||
|
|
||||||
with lock:
|
with lock:
|
||||||
state_file_none_or_valid_json(state_file_abs_path)
|
state_file_none_or_valid_json(state_file_abs_path)
|
||||||
with open(state_file_abs_path, "r+", encoding="utf-8") as state_file:
|
state_file_open_mode = "r+" if os.path.exists(state_file_abs_path) else "w+"
|
||||||
|
with open(state_file_abs_path, state_file_open_mode, encoding="utf-8") as state_file:
|
||||||
try:
|
try:
|
||||||
json_state = json.load(state_file)
|
json_state = json.load(state_file)
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
@ -646,14 +664,17 @@ def get_content_length(
|
|||||||
|
|
||||||
def get_json_state(
|
def get_json_state(
|
||||||
state_file_abs_path: str) -> json.loads:
|
state_file_abs_path: str) -> json.loads:
|
||||||
|
try:
|
||||||
with open(state_file_abs_path, "r", encoding="utf-8") as state_file:
|
with open(state_file_abs_path, "r", encoding="utf-8") as state_file:
|
||||||
try:
|
try:
|
||||||
json_state = json.load(state_file)
|
json_state = json.load(state_file)
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
return []
|
return []
|
||||||
else:
|
else:
|
||||||
return json_state
|
return json_state
|
||||||
|
except FileNotFoundError:
|
||||||
|
log.debug(f"State file does not exist (yet), assuming no previous downloads have ever happened ...")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
def is_already_downloaded(
|
def is_already_downloaded(
|
||||||
|
Loading…
x
Reference in New Issue
Block a user