Compare commits
8 Commits
287a755e65
...
03b449c768
Author | SHA1 | Date | |
---|---|---|---|
03b449c768 | |||
e269a110a6 | |||
a3a375d142 | |||
81ce5812a6 | |||
83921912a4 | |||
65e3ec83b1 | |||
266d3189dc | |||
563ff4d342 |
34
config.ini
34
config.ini
@ -1,34 +0,0 @@
|
||||
[DEFAULT]
|
||||
self_name = mvw-dl
|
||||
tmp_base_dir = /tmp/%(self_name)s
|
||||
state_base_dir = /var/lib/%(self_name)s
|
||||
state_files_dir = %(state_base_dir)s/state
|
||||
state_file_retention = 50
|
||||
state_file_name_prefix = state-
|
||||
state_file_name_suffix = .log
|
||||
mvw_endpoint = http://localhost:8000/api/query
|
||||
title_dedup_winner = first
|
||||
dl_progress_update_interval = 10
|
||||
dl_threads = 2
|
||||
dl_filename_pattern = &(channel)s - &(publish_date)s - &(topic)s - &(title)s.&(ext)s
|
||||
publish_date_srtftime_pattern = %%Y%%m%%d
|
||||
dl_filename_replace_spaces_with =
|
||||
dl_filename_all_lowercase = no
|
||||
|
||||
[maus]
|
||||
min_duration = 1200
|
||||
max_duration = 2700
|
||||
query = @maus-query.json
|
||||
title_not_regex = audiodeskription|gebärdensprache|hörfassung
|
||||
# dl_filename_pattern = &(publish_date)s.&(ext)s
|
||||
# publish_date_srtftime_pattern = S%%YE%%Y%%m%%d01
|
||||
# query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50}
|
||||
# state_file_name = maus
|
||||
# tmp_base_dir = %(tmp_base_dir)s/maus
|
||||
dl_dir = ~/maus
|
||||
|
||||
#[test]
|
||||
#min_duration = 100
|
||||
#max_duration = 200
|
||||
#query = {"queries":[{"fields":["topic"],"query":"die sendung mit der maus"},{"fields":["channel"],"query":"ARD"}],"sortBy":"timestamp","sortOrder":"desc","future":false,"offset":0,"size":50}
|
||||
#dl_dir = test
|
25
examples/config.ini.example
Normal file
25
examples/config.ini.example
Normal file
@ -0,0 +1,25 @@
|
||||
[DEFAULT]
|
||||
self_name = mvw-dl
|
||||
tmp_base_dir = /tmp/%(self_name)s
|
||||
state_base_dir = /var/lib/%(self_name)s
|
||||
state_files_dir = %(state_base_dir)s/state
|
||||
state_file_retention = 50
|
||||
state_file_name_prefix = state-
|
||||
state_file_name_suffix = .log
|
||||
mvw_endpoint = http://localhost:8000/api/query
|
||||
title_dedup_winner = first
|
||||
dl_progress_update_interval = 10
|
||||
dl_threads = 2
|
||||
dl_filename_pattern = &(channel)s - &(publish_date)s - &(topic)s - &(title)s.&(ext)s
|
||||
publish_date_srtftime_pattern = %%Y%%m%%d
|
||||
dl_filename_replace_spaces_with =
|
||||
dl_filename_all_lowercase = no
|
||||
|
||||
[maus]
|
||||
min_duration = 1200
|
||||
max_duration = 3000
|
||||
query = @maus-query.json.example
|
||||
title_not_regex = audiodeskription|gebärdensprache|hörfassung
|
||||
dl_filename_pattern = &(publish_date)s.&(ext)s
|
||||
publish_date_srtftime_pattern = S%%YE%%Y%%m%%d01
|
||||
dl_dir = /tmp/kodi-nfo-feeder/maus
|
@ -17,5 +17,5 @@
|
||||
"sortOrder": "desc",
|
||||
"future": false,
|
||||
"offset": 0,
|
||||
"size": 100
|
||||
"size": 20
|
||||
}
|
12
examples/mvw-dl.service.example
Normal file
12
examples/mvw-dl.service.example
Normal file
@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=MediathekViewWeb download helper
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=no
|
||||
Environment='PATH=/usr/local/sbin:/usr/local/bin:/usr/bin'
|
||||
ExecStart=/opt/miniconda3/envs/mvw-dl/bin/python /opt/python/mvw-dl/dev/mvw-dl.py
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
10
examples/mvw-dl.timer.example
Normal file
10
examples/mvw-dl.timer.example
Normal file
@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run MediathekViewWeb download helper
|
||||
|
||||
[Timer]
|
||||
OnCalendar=0/2:2
|
||||
OnCalendar=0/1:20
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
31
mvw-dl.py
31
mvw-dl.py
@ -46,6 +46,11 @@ from rich.progress import (
|
||||
)
|
||||
|
||||
# TODO set locale for datetime and others to globally stick to en_US
|
||||
# TODO thread log messages display timestamp in systemd journal
|
||||
# TODO Increment file name suffix more than once of needed
|
||||
# TODO [23:15:14] DEBUG [thread]
|
||||
# TODO Clean mvw-dl.timer
|
||||
# TODO Reset maus-query.json
|
||||
download_start_time = 0
|
||||
download_last_update_time = 0
|
||||
total_content_length = 0
|
||||
@ -84,6 +89,7 @@ JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any], t.List[t.Any
|
||||
# 3: No search results to download
|
||||
# 4: State file already exists, has more than 0 bytes size but doesn't contain usable JSON
|
||||
# 5: State file lock cannot be acquired within file_lock_timeout
|
||||
# 6: Unable to create state directory
|
||||
|
||||
|
||||
class CONST(object):
|
||||
@ -159,7 +165,7 @@ class ConfigParser(
|
||||
ini_defaults = []
|
||||
internal_defaults = {default["key"]: default["value"] for default in CONST.CFG_KNOWN_DEFAULTS}
|
||||
config = ConfigParser(defaults=internal_defaults)
|
||||
config.read(CONST.CFG_DEFAULT_FILENAME)
|
||||
config.read(CONST.CFG_DEFAULT_ABS_PATH)
|
||||
|
||||
|
||||
def print_section_header(
|
||||
@ -227,7 +233,8 @@ def validate_config_sections(
|
||||
|
||||
def query_string_from_file(
|
||||
filename: str) -> str:
|
||||
with open(filename, "r", encoding="utf-8") as jsonfile:
|
||||
filename_abs_path = os.path.join(CONST.CFG_THIS_FILE_DIRNAME, filename)
|
||||
with open(filename_abs_path, "r", encoding="utf-8") as jsonfile:
|
||||
query_string = jsonfile.read()
|
||||
return query_string
|
||||
|
||||
@ -393,8 +400,11 @@ def get_safe_filename(
|
||||
shorthand_uuid: str) -> str:
|
||||
"""https://stackoverflow.com/a/71199182"""
|
||||
|
||||
log.debug(f"{shorthand_uuid} Removing question marks from file name ...")
|
||||
clean_filename = re.sub(r"""[?]""", "", dirty_filename)
|
||||
|
||||
log.debug(f"{shorthand_uuid} Replacing unsafe characters in filename with dashes ...")
|
||||
clean_filename = re.sub(r"""[/\\?%*:|"<>\x7F\x00-\x1F]""", "-", dirty_filename)
|
||||
clean_filename = re.sub(r"""[/\\?%*:|"<>\x7F\x00-\x1F]""", "-", clean_filename)
|
||||
|
||||
log.debug(f"{shorthand_uuid} New filename: '{clean_filename}'")
|
||||
return clean_filename
|
||||
@ -456,6 +466,13 @@ def get_state_file_abs_path(
|
||||
config_obj: configparser.ConfigParser()) -> str:
|
||||
|
||||
state_dir = config_obj.get(section_name, "state_files_dir")
|
||||
try:
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
except OSError:
|
||||
log.error(f"Unable to create '[{section}]' state directory '{state_dir}'. "
|
||||
f"We're not going to be able to log state information. Exiting 6 ...")
|
||||
sys.exit(6)
|
||||
else:
|
||||
state_file = \
|
||||
config_obj.get(section_name, "state_file_name_prefix") + \
|
||||
section_name + \
|
||||
@ -529,7 +546,8 @@ def log_successful_download(
|
||||
|
||||
with lock:
|
||||
state_file_none_or_valid_json(state_file_abs_path)
|
||||
with open(state_file_abs_path, "r+", encoding="utf-8") as state_file:
|
||||
state_file_open_mode = "r+" if os.path.exists(state_file_abs_path) else "w+"
|
||||
with open(state_file_abs_path, state_file_open_mode, encoding="utf-8") as state_file:
|
||||
try:
|
||||
json_state = json.load(state_file)
|
||||
except json.JSONDecodeError:
|
||||
@ -646,7 +664,7 @@ def get_content_length(
|
||||
|
||||
def get_json_state(
|
||||
state_file_abs_path: str) -> json.loads:
|
||||
|
||||
try:
|
||||
with open(state_file_abs_path, "r", encoding="utf-8") as state_file:
|
||||
try:
|
||||
json_state = json.load(state_file)
|
||||
@ -654,6 +672,9 @@ def get_json_state(
|
||||
return []
|
||||
else:
|
||||
return json_state
|
||||
except FileNotFoundError:
|
||||
log.debug(f"State file does not exist (yet), assuming no previous downloads have ever happened ...")
|
||||
return []
|
||||
|
||||
|
||||
def is_already_downloaded(
|
||||
|
Loading…
x
Reference in New Issue
Block a user