Perform periodic check to see if the tasks are still alive.

If not, restart the task.

main/SABnzbd.py

@@ -1149,6 +1149,8 @@ def main():
             scheduler.restart()
             # Save config (if needed)
             config.save_config()
+            # Check the threads
+            sabnzbd.check_all_tasks()
         else:
             timer += 1
 

main/sabnzbd/__init__.py

@@ -710,3 +710,28 @@ def SimpleRarExtract(rarfile, fn):
     """ Wrapper for call to newsunpack, required to avoid circular imports
     """
     return sabnzbd.newsunpack.SimpleRarExtract(rarfile, fn)
+
+
+def check_all_tasks():
+    """ Check every task and restart any crashed one """
+    if not sabnzbd.dirscanner.alive():
+        logging.info('Restarting crashed dirscanner')
+        sabnzbd.dirscanner.init()
+    if not sabnzbd.urlgrabber.alive():
+        logging.info('Restarting crashed urlgrabber')
+        sabnzbd.urlgrabber.init()
+    if not sabnzbd.newzbin.alive():
+        logging.info('Restarting crashed newzbin')
+        sabnzbd.newzbin.init_grabber()
+    if not sabnzbd.postproc.alive():
+        logging.info('Restarting crashed postprocessor')
+        sabnzbd.postproc.init()
+    if not sabnzbd.downloader.alive():
+        logging.info('Restarting crashed downloader')
+        sabnzbd.downloader.init()
+    if not sabnzbd.assembler.alive():
+        logging.info('Restarting crashed assembler')
+        sabnzbd.assembler.init()
+    if not sabnzbd.scheduler.sched_check():
+        logging.info('Restarting crashed scheduler')
+        sabnzbd.scheduler.init()

main/sabnzbd/assembler.py

@@ -74,6 +74,12 @@ def stop():
         except:
             pass
 
+def alive():
+    global __ASM
+    if __ASM:
+        return __ASM.isAlive()
+    else:
+        return False
 
 #------------------------------------------------------------------------------
 class Assembler(Thread):

main/sabnzbd/dirscanner.py

@@ -73,6 +73,13 @@ def save():
     global __SCANNER
     if __SCANNER: __SCANNER.save()
 
+def alive():
+    global __SCANNER
+    if __SCANNER:
+        return __SCANNER.isAlive()
+    else:
+        return False
+
 
 ################################################################################
 # Body

main/sabnzbd/downloader.py

@@ -85,6 +85,12 @@ def stop():
     except:
         pass
 
+def alive():
+    global __DOWNLOADER
+    if __DOWNLOADER:
+        return __DOWNLOADER.isAlive()
+    else:
+        return False
 
 #------------------------------------------------------------------------------
 

main/sabnzbd/newzbin.py

@@ -118,6 +118,14 @@ def grab(msgid, future_nzo):
     if __MSGIDGRABBER:
         __MSGIDGRABBER.grab(msgid, future_nzo)
 
+def alive():
+    global __MSGIDGRABBER
+    if __MSGIDGRABBER:
+        return __MSGIDGRABBER.isAlive()
+    else:
+        return False
+
+
 
 ################################################################################
 # DirectNZB support

main/sabnzbd/postproc.py

@@ -92,6 +92,13 @@ def stop():
         except:
             pass
 
+def alive():
+    global __POSTPROC
+    if __POSTPROC:
+        return __POSTPROC.isAlive()
+    else:
+        return False
+
 def save():
     global __POSTPROC
     if __POSTPROC: __POSTPROC.save()

main/sabnzbd/scheduler.py

@@ -52,6 +52,7 @@ def init():
     """
     global __SCHED
 
+    reset_guardian()
     __SCHED = kronos.ThreadedScheduler()
 
     for schedule in cfg.SCHEDULES.get():
@@ -107,6 +108,10 @@ def init():
         __SCHED.add_daytime_task(action, action_name, d, None, (h, m),
                              kronos.method.sequential, arguments, None)
 
+    # Set Guardian interval to 30 seconds
+    __SCHED.add_interval_task(sched_guardian, "Guardian", 15, 30,
+                                  kronos.method.sequential, None, None)
+
     # Set RSS check interval
     interval = cfg.RSS_RATE.get()
     delay = random.randint(0, interval-1)
@@ -337,3 +342,30 @@ def plan_server(action, parms, interval):
     """ Plan to re-activate server after "interval" minutes
     """
     __SCHED.add_single_task(action, '', interval*60, kronos.method.sequential, parms, None)
+
+
+#------------------------------------------------------------------------------
+# Scheduler Guarding system
+# Each check sets the guardian flag False
+# Each succesful scheduled check sets the flag
+# If 4 consequetive checks fail, the sheduler is assumed to have crashed
+
+__SCHED_GUARDIAN = False
+__SCHED_GUARDIAN_CNT = 0
+
+def reset_guardian():
+    global __SCHED_GUARDIAN, __SCHED_GUARDIAN_CNT
+    __SCHED_GUARDIAN = False
+    __SCHED_GUARDIAN_CNT = 0
+
+def sched_guardian():
+    global __SCHED_GUARDIAN, __SCHED_GUARDIAN_CNT
+    __SCHED_GUARDIAN = True
+
+def sched_check():
+    global __SCHED_GUARDIAN, __SCHED_GUARDIAN_CNT
+    if not __SCHED_GUARDIAN:
+        __SCHED_GUARDIAN_CNT += 1
+        return __SCHED_GUARDIAN_CNT < 4
+    reset_guardian()
+    return True

main/sabnzbd/urlgrabber.py

@@ -73,6 +73,14 @@ def stop():
         except:
             pass
 
+def alive():
+    global __GRABBER
+    if __GRABBER:
+        return __GRABBER.isAlive()
+    else:
+        return False
+
+
 
 #------------------------------------------------------------------------------
 class URLGrabber(Thread):