diff --git a/main/SABnzbd.py b/main/SABnzbd.py index 0202a62..1300f3c 100755 --- a/main/SABnzbd.py +++ b/main/SABnzbd.py @@ -1149,6 +1149,8 @@ def main(): scheduler.restart() # Save config (if needed) config.save_config() + # Check the threads + sabnzbd.check_all_tasks() else: timer += 1 diff --git a/main/sabnzbd/__init__.py b/main/sabnzbd/__init__.py index 1678bd1..e56a03d 100644 --- a/main/sabnzbd/__init__.py +++ b/main/sabnzbd/__init__.py @@ -710,3 +710,28 @@ def SimpleRarExtract(rarfile, fn): """ Wrapper for call to newsunpack, required to avoid circular imports """ return sabnzbd.newsunpack.SimpleRarExtract(rarfile, fn) + + +def check_all_tasks(): + """ Check every task and restart any crashed one """ + if not sabnzbd.dirscanner.alive(): + logging.info('Restarting crashed dirscanner') + sabnzbd.dirscanner.init() + if not sabnzbd.urlgrabber.alive(): + logging.info('Restarting crashed urlgrabber') + sabnzbd.urlgrabber.init() + if not sabnzbd.newzbin.alive(): + logging.info('Restarting crashed newzbin') + sabnzbd.newzbin.init_grabber() + if not sabnzbd.postproc.alive(): + logging.info('Restarting crashed postprocessor') + sabnzbd.postproc.init() + if not sabnzbd.downloader.alive(): + logging.info('Restarting crashed downloader') + sabnzbd.downloader.init() + if not sabnzbd.assembler.alive(): + logging.info('Restarting crashed assembler') + sabnzbd.assembler.init() + if not sabnzbd.scheduler.sched_check(): + logging.info('Restarting crashed scheduler') + sabnzbd.scheduler.init() diff --git a/main/sabnzbd/assembler.py b/main/sabnzbd/assembler.py index 673777e..3451797 100644 --- a/main/sabnzbd/assembler.py +++ b/main/sabnzbd/assembler.py @@ -74,6 +74,12 @@ def stop(): except: pass +def alive(): + global __ASM + if __ASM: + return __ASM.isAlive() + else: + return False #------------------------------------------------------------------------------ class Assembler(Thread): diff --git a/main/sabnzbd/dirscanner.py b/main/sabnzbd/dirscanner.py index 08d3634..7724a0b 100644 --- a/main/sabnzbd/dirscanner.py +++ b/main/sabnzbd/dirscanner.py @@ -73,6 +73,13 @@ def save(): global __SCANNER if __SCANNER: __SCANNER.save() +def alive(): + global __SCANNER + if __SCANNER: + return __SCANNER.isAlive() + else: + return False + ################################################################################ # Body diff --git a/main/sabnzbd/downloader.py b/main/sabnzbd/downloader.py index 3875279..12256f2 100644 --- a/main/sabnzbd/downloader.py +++ b/main/sabnzbd/downloader.py @@ -85,6 +85,12 @@ def stop(): except: pass +def alive(): + global __DOWNLOADER + if __DOWNLOADER: + return __DOWNLOADER.isAlive() + else: + return False #------------------------------------------------------------------------------ diff --git a/main/sabnzbd/newzbin.py b/main/sabnzbd/newzbin.py index 48650c5..bf6dbf6 100644 --- a/main/sabnzbd/newzbin.py +++ b/main/sabnzbd/newzbin.py @@ -118,6 +118,14 @@ def grab(msgid, future_nzo): if __MSGIDGRABBER: __MSGIDGRABBER.grab(msgid, future_nzo) +def alive(): + global __MSGIDGRABBER + if __MSGIDGRABBER: + return __MSGIDGRABBER.isAlive() + else: + return False + + ################################################################################ # DirectNZB support diff --git a/main/sabnzbd/postproc.py b/main/sabnzbd/postproc.py index c31f72c..a45892b 100644 --- a/main/sabnzbd/postproc.py +++ b/main/sabnzbd/postproc.py @@ -92,6 +92,13 @@ def stop(): except: pass +def alive(): + global __POSTPROC + if __POSTPROC: + return __POSTPROC.isAlive() + else: + return False + def save(): global __POSTPROC if __POSTPROC: __POSTPROC.save() diff --git a/main/sabnzbd/scheduler.py b/main/sabnzbd/scheduler.py index 73ee3b1..5c3f08b 100644 --- a/main/sabnzbd/scheduler.py +++ b/main/sabnzbd/scheduler.py @@ -52,6 +52,7 @@ def init(): """ global __SCHED + reset_guardian() __SCHED = kronos.ThreadedScheduler() for schedule in cfg.SCHEDULES.get(): @@ -107,6 +108,10 @@ def init(): __SCHED.add_daytime_task(action, action_name, d, None, (h, m), kronos.method.sequential, arguments, None) + # Set Guardian interval to 30 seconds + __SCHED.add_interval_task(sched_guardian, "Guardian", 15, 30, + kronos.method.sequential, None, None) + # Set RSS check interval interval = cfg.RSS_RATE.get() delay = random.randint(0, interval-1) @@ -337,3 +342,30 @@ def plan_server(action, parms, interval): """ Plan to re-activate server after "interval" minutes """ __SCHED.add_single_task(action, '', interval*60, kronos.method.sequential, parms, None) + + +#------------------------------------------------------------------------------ +# Scheduler Guarding system +# Each check sets the guardian flag False +# Each succesful scheduled check sets the flag +# If 4 consequetive checks fail, the sheduler is assumed to have crashed + +__SCHED_GUARDIAN = False +__SCHED_GUARDIAN_CNT = 0 + +def reset_guardian(): + global __SCHED_GUARDIAN, __SCHED_GUARDIAN_CNT + __SCHED_GUARDIAN = False + __SCHED_GUARDIAN_CNT = 0 + +def sched_guardian(): + global __SCHED_GUARDIAN, __SCHED_GUARDIAN_CNT + __SCHED_GUARDIAN = True + +def sched_check(): + global __SCHED_GUARDIAN, __SCHED_GUARDIAN_CNT + if not __SCHED_GUARDIAN: + __SCHED_GUARDIAN_CNT += 1 + return __SCHED_GUARDIAN_CNT < 4 + reset_guardian() + return True diff --git a/main/sabnzbd/urlgrabber.py b/main/sabnzbd/urlgrabber.py index 253fbb8..c5b7dab 100644 --- a/main/sabnzbd/urlgrabber.py +++ b/main/sabnzbd/urlgrabber.py @@ -73,6 +73,14 @@ def stop(): except: pass +def alive(): + global __GRABBER + if __GRABBER: + return __GRABBER.isAlive() + else: + return False + + #------------------------------------------------------------------------------ class URLGrabber(Thread):