Add retention time as server parameter (in days, where 0 is infinite).

Jobs older than the retention time of a server will not be tried for that server.
The 'top_only' option is permanently disbled to prevent queue lockup by a job outside the retention time of all servers.
Some fixes and asserts to help static analysers.

interfaces/Classic/templates/config_server.tmpl

@@ -17,6 +17,7 @@
       $T('srv-password'):<br><input type="password" size="25" name="password"><br>
       $T('srv-timeout'):<br><input type="text" size="25" name="timeout" value="120"><br>
       $T('srv-connections'):<br><input type="text" size="25" name="connections"><br>
+      $T('srv-retention') ($T('days')):<br><input type="text" size="25" name="retention"><br>
       <!--#if int($have_ssl) == 0#-->
          $T('srv-ssl') $T('opt-notInstalled')
       <!--#else#-->
@@ -41,6 +42,7 @@
       $T('srv-password'):<br><input type="password" size="25" name="password" value="$servers[$server]['password']"><br>
       $T('srv-timeout'):<br><input type="text" size="25" name="timeout" value="$servers[$server]['timeout']"><br>
       $T('srv-connections'):<br><input type="text" size="25" name="connections" value="$servers[$server]['connections']"><br>
+      $T('srv-retention'):<br><input type="text" size="25" name="retention" value="$servers[$server]['retention']"><br>
       <!--#if int($have_ssl) == 0#-->
          $T('srv-ssl') $T('opt-notInstalled')
       <!--#else#-->

interfaces/Plush/templates/config_server.tmpl

@@ -23,7 +23,8 @@
 
     <table>
     <tr><td class="alignRight"><div class="sucker">#&nbsp;$T('srv-connections'):</div></td><td><input type="text" size="5" name="connections"/></td></tr>
-    <tr><td class="alignRight"><div class="sucker">$T('srv-timeout'):</div></td><td><input type="text" size="5" name="timeout" value="120">&nbsp;<small>$T('second')</small></td></tr>
+    <tr><td class="alignRight"><div class="sucker">$T('srv-retention'):</div></td><td><input type="text" size="5" name="retention"/>&nbsp;<small>$T('days')</small></td></tr>
+    <tr><td class="alignRight"><div class="sucker">$T('srv-timeout'):</div></td><td><input type="text" size="5" name="timeout" value="120">&nbsp;<small>$T('seconds')</small></td></tr>
 
     <tr><td colspan="2">&nbsp;</td></tr>
 
@@ -84,8 +85,12 @@
         <td><input type="text" size="5" name="connections" value="$servers[$server]['connections']"/></td>
     </tr>
     <tr>
+        <td class="alignRight"><div class="sucker">$T('srv-retention'):</div></td>
+        <td><input type="text" size="5" name="retention" value="$servers[$server]['retention']"/>&nbsp;<small>$T('days')</small></td>
+    </tr>
+    <tr>
         <td class="alignRight"><div class="sucker">$T('srv-timeout'):</div></td>
-        <td><input type="text" size="5" name="timeout" value="$servers[$server]['timeout']"/>&nbsp;<small>seconds</small></td>
+        <td><input type="text" size="5" name="timeout" value="$servers[$server]['timeout']"/>&nbsp;<small>$T('seconds')</small></td>
     </tr>
     <tr><td colspan="2">&nbsp;</td></tr>
     <tr>

interfaces/smpl/templates/config_server.tmpl

@@ -22,6 +22,8 @@
 			<br class="clear" />
 		<label class="label">$T('srv-connections'):</label><input type="text" size="25" name="connections" value="$servers[$server]['connections']">
 			<br class="clear" />
+		<label class="label">$T('srv-retention') ($T('days')):</label><input type="text" size="25" name="retention" value="$servers[$server]['retention']">
+			<br class="clear" />
 
 			<label><span class="label">$T('srv-ssl')<!--#if int($have_ssl) == 0 then $T('opt-notInstalled') else ""#-->:</span>
 			<input class="radio" type="checkbox" name="ssl" value="1" <!--#if int($servers[$server]['ssl']) != 0 and int($have_ssl) == 1 then "checked=1" else ""#--> <!--#if int($have_ssl) == 0 then "disabled" else ""#-->/>
@@ -85,6 +87,9 @@
 			<label class="label">$T('srv-connections'):</label><input type="text" size="25" name="connections">
 		<br class="clear" />
 
+			<label class="label">$T('srv-retention') ($T('days')):</label><input type="text" size="25" name="retention">
+		<br class="clear" />
+
 			<label><span class="label">$T('srv-ssl')<!--#if int($have_ssl) == 0 then $T('opt-notInstalled') else ""#-->:</span>
 				<input class="radio" type="checkbox" name="ssl" value="1" <!--#if int($have_ssl) == 0 then "disabled" else ""#-->></label>
  		<br class="clear" />

sabnzbd/config.py

@@ -344,6 +344,7 @@ class ConfigServer(object):
         self.ssl = OptionBool(name, 'ssl', False, add=False)
         self.enable = OptionBool(name, 'enable', True, add=False)
         self.optional = OptionBool(name, 'optional', False, add=False)
+        self.retention = OptionNumber(name, 'retention', add=False)
 
         self.set_dict(values)
         add_to_database('servers', self.__name, self)
@@ -351,7 +352,7 @@ class ConfigServer(object):
     def set_dict(self, values):
         """ Set one or more fields, passed as dictionary """
         for kw in ('host', 'port', 'timeout', 'username', 'password', 'connections',
-                   'fillserver', 'ssl', 'enable', 'optional'):
+                   'fillserver', 'ssl', 'enable', 'optional', 'retention'):
             try:
                 value = values[kw]
             except KeyError:
@@ -376,6 +377,7 @@ class ConfigServer(object):
         dict['ssl'] = self.ssl()
         dict['enable'] = self.enable()
         dict['optional'] = self.optional()
+        dict['retention'] = self.retention()
         return dict
 
     def delete(self):

sabnzbd/downloader.py

@@ -22,7 +22,6 @@ sabnzbd.downloader - download engine
 import time
 import select
 import logging
-import datetime
 from threading import Thread, RLock
 from nntplib import NNTPPermanentError
 import socket
@@ -199,7 +198,7 @@ def active_primaries():
 #------------------------------------------------------------------------------
 class Server(object):
     def __init__(self, id, host, port, timeout, threads, fillserver, ssl, username = None,
-                 password = None, optional=False):
+                 password = None, optional=False, retention=0):
         self.id = id
         self.newid = None
         self.restart = False
@@ -210,6 +209,7 @@ class Server(object):
         self.fillserver = fillserver
         self.ssl = ssl
         self.optional = optional
+        self.retention = retention
 
         self.username = username
         self.password = password
@@ -246,7 +246,7 @@ class Server(object):
 #------------------------------------------------------------------------------
 
 class Downloader(Thread):
-    def __init__(self, paused = False):
+    def __init__(self, paused=False):
         Thread.__init__(self)
 
         logging.debug("Initializing downloader/decoder")
@@ -307,6 +307,7 @@ class Downloader(Thread):
             username = srv.username()
             password = srv.password()
             optional = srv.optional()
+            retention = float(srv.retention() * 24 * 3600) # days ==> seconds
             create = True
 
         if oldserver:
@@ -321,7 +322,7 @@ class Downloader(Thread):
 
         if create and enabled and host and port and threads:
             self.servers.append(Server(newserver, host, port, timeout, threads, fillserver, ssl,
-                                            username, password, optional))
+                                            username, password, optional, retention))
 
         return primary
 
@@ -401,6 +402,7 @@ class Downloader(Thread):
 
         while 1:
             for server in self.servers:
+                assert isinstance(server, Server)
                 for nw in server.busy_threads[:]:
                     if nw.nntp.error_msg or (nw.timeout and time.time() > nw.timeout):
                         if nw.nntp.error_msg:
@@ -424,6 +426,7 @@ class Downloader(Thread):
                         # Restart pending, don't add new articles
                         continue
 
+                assert isinstance(server, Server)
                 if not server.idle_threads or server.restart or self.is_paused() or self.shutdown or self.delayed or self.postproc:
                     continue
 
@@ -431,6 +434,7 @@ class Downloader(Thread):
                     continue
 
                 for nw in server.idle_threads[:]:
+                    assert isinstance(nw, NewsWrapper)
                     if nw.timeout:
                         if time.time() < nw.timeout:
                             continue
@@ -450,26 +454,32 @@ class Downloader(Thread):
                     if not article:
                         break
 
-                    else:
-                        server.idle_threads.remove(nw)
-                        server.busy_threads.append(nw)
+                    if server.retention and article.nzf.nzo.avg_stamp < time.time() - server.retention:
+                        # Article too old for the server, treat as missing
+                        if sabnzbd.LOG_ALL:
+                            logging.debug('Article %s too old for %s:%s', article.article, server.host, server.port)
+                        self.decoder.decode(article, None)
+                        break
 
-                        nw.article = article
+                    server.idle_threads.remove(nw)
+                    server.busy_threads.append(nw)
 
-                        if nw.connected:
-                            self.__request_article(nw)
-                        else:
-                            try:
-                                logging.info("%s@%s:%s: Initiating connection",
-                                                  nw.thrdnum, server.host, server.port)
-                                nw.init_connect()
-                                self.write_fds[nw.nntp.sock.fileno()] = nw
-                            except:
-                                logging.error(Ta('Failed to initialize %s@%s:%s'),
-                                                  nw.thrdnum, server.host,
-                                                  server.port)
-                                logging.info("Traceback: ", exc_info = True)
-                                self.__reset_nw(nw, "failed to initialize")
+                    nw.article = article
+
+                    if nw.connected:
+                        self.__request_article(nw)
+                    else:
+                        try:
+                            logging.info("%s@%s:%s: Initiating connection",
+                                              nw.thrdnum, server.host, server.port)
+                            nw.init_connect()
+                            self.write_fds[nw.nntp.sock.fileno()] = nw
+                        except:
+                            logging.error(Ta('Failed to initialize %s@%s:%s'),
+                                              nw.thrdnum, server.host,
+                                              server.port)
+                            logging.info("Traceback: ", exc_info = True)
+                            self.__reset_nw(nw, "failed to initialize")
 
             # Exit-point
             if self.shutdown:

sabnzbd/nzbqueue.py

@@ -54,7 +54,7 @@ class NzbQueue(TryList):
     def __init__(self):
         TryList.__init__(self)
 
-        self.__top_only = cfg.top_only()
+        self.__top_only = False #cfg.top_only()
         self.__top_nzo = None
 
         self.__nzo_list = []
@@ -195,6 +195,7 @@ class NzbQueue(TryList):
     @synchronized(NZBQUEUE_LOCK)
     def insert_future(self, future, filename, msgid, data, pp=None, script=None, cat=None, priority=NORMAL_PRIORITY, nzbname=None, nzo_info=None):
         """ Refresh a placeholder nzo with an actual nzo """
+        assert isinstance(future, NzbObject)
         if nzo_info is None:
             nzo_info = {}
         nzo_id = future.nzo_id
@@ -276,6 +277,7 @@ class NzbQueue(TryList):
 
     @synchronized(NZBQUEUE_LOCK)
     def add(self, nzo, save=True):
+        assert isinstance(nzo, NzbObject)
         sabnzbd.QUEUECOMPLETEACTION_GO = False
 
         if not nzo.nzo_id:
@@ -729,12 +731,6 @@ class NzbQueue(TryList):
 
         ArticleCache.do.purge_articles(nzo.saved_articles)
 
-    @synchronized(NZBQUEUE_LOCK)
-    def debug(self):
-        return ([], self.__nzo_list[:],
-                self.__nzo_table.copy(), self.try_list[:])
-
-
     def get_urls(self):
         """ Return list of future-types needing URL """
         lst = []

sabnzbd/nzbstuff.py

@@ -87,6 +87,7 @@ class Article(TryList):
         if not self.fetcher and not self.server_in_try_list(server):
             self.fetcher = server
             return self
+        return None
 
     def get_art_id(self):
         """ Return unique article storage name, create if needed """
@@ -420,6 +421,7 @@ class NzbParser(xml.sax.handler.ContentHandler):
         """ End of the file """
         self.nzo.groups = self.groups
         files = max(1, self.valids)
+        self.nzo.avg_stamp = self.avg_age / files
         self.nzo.avg_date = datetime.datetime.fromtimestamp(self.avg_age / files)
         if self.skipped_files:
             logging.warning(Ta('Failed to import %s files from %s'),
@@ -524,6 +526,7 @@ class NzbObject(TryList):
             self.url = ''
         self.groups = []
         self.avg_date = datetime.datetime.fromtimestamp(0.0)
+        self.avg_stamp = 0.0        # Avg age in seconds (calculated from avg_age)
         self.dirprefix = []
 
         self.partable = {}          # Holds one parfile-name for each set
@@ -814,6 +817,7 @@ class NzbObject(TryList):
         # Flag files from NZB that already exist as finished
         for nzf in self.files[:]:
             alleged_name = nzf.filename
+            filename = alleged_name
             subject = sanitize_filename(latin1(nzf.subject))
             ready = alleged_name in files
             if not ready:
@@ -923,6 +927,7 @@ class NzbObject(TryList):
         nzf_remove_list = []
 
         for nzf in self.files:
+            assert isinstance(nzf, NzbFile)
             # Don't try to get an article if server is in try_list of nzf
             if not nzf.server_in_try_list(server):
                 if not nzf.import_finished:
@@ -1192,6 +1197,7 @@ class NzbObject(TryList):
                 # Handle new attributes
                 self.__dict__[tup[1]] = None
         self.pp_active = False
+        self.avg_stamp = time.mktime(self.avg_date.timetuple())
         TryList.__init__(self)
 
 

sabnzbd/skintext.py

@@ -69,6 +69,8 @@ SKIN_TEXT = {
     'minutes' : TT('mins'),
     'second' : TT('sec'),
     'seconds' : TT('seconds'),
+    'day' : TT('day'),
+    'days' : TT('days'),
     'month' : TT('Month'),
     'year' : TT('Year'),
     'daily' : TT('Daily'),
@@ -377,6 +379,7 @@ SKIN_TEXT = {
     'srv-password' : TT('Password'),
     'srv-timeout' : TT('Timeout'),
     'srv-connections' : TT('Connections'),
+    'srv-retention' : TT('Retention time'),
     'srv-ssl' : TT('SSL'),
     'srv-fillserver' : TT('Backup server'),
     'srv-optional' : TT('Optional'),