Merge in changes from branch sabnzbd-0.5.x Rev 3182-3192.

main/CHANGELOG.txt

@@ -1,4 +1,13 @@
 -------------------------------------------------------------------------------
+0.5.0Beta6 by The SABnzbd-Team
+-------------------------------------------------------------------------------
+Fixes:
+- Solve problems with unwilling servers and "random" server time-outs.
+- Do not generate warnings for intermittent server problems.
+- Joinable rar-sets were not always properly joined.
+- Fixed timing problems when SABnzbd is running behind an Apache server
+- Don't report failed default permissions setting when the user has set none
+-------------------------------------------------------------------------------
 0.5.0Beta5 by The SABnzbd-Team
 -------------------------------------------------------------------------------
 Fixes:

main/ISSUES.txt

@@ -8,11 +8,6 @@ So far we have too little information to reproduce these problems.
 Please report such problems on the forum with as much information
 as possibe (OS, amount of usenet servers etc.)
 
-*** (hopefully) Solved issue ***
-  We are investigating reports of downloads hanging in the
-  last bit of the actual download.
-  0.5.0 Beta5 attempts to fix this. Let us hear your experience.
-
 
 *** Known problems ***
 

main/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 1.0
 Name: SABnzbd
-Version: 0.5.0Beta5
-Summary: SABnzbd-0.5.0Beta5
+Version: 0.6.0Alpha0
+Summary: SABnzbd-0.6.0ALpha0
 Home-page: http://sourceforge.net/projects/sabnzbdplus
 Author: The SABnzbd Team
 Author-email: team@sabnzbd.org

main/README.txt

@@ -1,5 +1,5 @@
 *******************************************
-*** This is SABnzbd 0.5.0 Beta5         ***
+*** This is SABnzbd 0.6.0 Alpha0        ***
 *******************************************
 SABnzbd is an open-source cross-platform binary newsreader.
 It simplifies the process of downloading from Usenet dramatically,
@@ -16,7 +16,7 @@ http://wiki.sabnzbd.org/
 *******************************************
 *** Beta Software                       ***
 *******************************************
-This is Beta software, it´s almost mature.
+This is Beta software, it�s almost mature.
 We are quite confident, but not enough people
 have used it yet.
 

main/interfaces/Classic/templates/queue.tmpl

@@ -40,9 +40,9 @@ $T('onQueueFinish'):
 <!--#end if#-->
 </select>
 | $T('speedLimit'):
-<input id="speedbox" type="text" value="$speedlimit" onChange="location = './set_speedlimit?session=$session&value='+this.value">$T('KBs')
+<input id="speedbox" type="text" value="$speedlimit" onChange="location = './set_speedlimit?session=$session&value='+this.value">&nbsp;$T('KBs')
 | $T('pauseFor'):
-<input id="speedbox" type="text" size=5 value="" onChange="location = './set_pause?session=$session&value='+this.value">$T('minute')
+<input id="speedbox" type="text" size=5 value="" onChange="location = './set_pause?session=$session&value='+this.value">&nbsp;$T('minute')
 |
 <a href="./purge?session=$session" onclick="return confirm('$T('purgeQueueConf').replace("'","`") ');">$T('purgeQueue')</a>
 </span>

main/language/de-de.txt

@@ -164,6 +164,7 @@ clearWarnings     Leeren
 server-blocked    Freigeben
 warning           WARNUNG:
 error             FEHLER:
+server-resolving   Adresse auflösen
 
 # Configuration
 configuration     Einstellungen

main/language/fr-fr.txt

@@ -164,6 +164,7 @@ clearWarnings     effacer
 server-blocked    Débloquer
 warning           AVERTISSEMENT:
 error             ERREUR:
+server-resolving   Résoudre l'adresse
 
 
 # Configuration

main/language/nl-du.txt

@@ -160,6 +160,7 @@ clearWarnings     Wissen
 server-blocked    Deblokkeren
 warning           WAARSCHUWING:
 error             FOUT:
+server-resolving   Adres opzoeken
 
 # Configuration
 configuration     Instellingen

main/language/sv-se.txt

@@ -163,6 +163,7 @@ clearWarnings     rensa
 server-blocked    Ta bort blockering
 warning           VARNING:
 error             FEL:
+server-resolving   Lösa adress
 
 # Configuration
 configuration     Konfiguration

main/language/us-en.txt

@@ -162,6 +162,7 @@ clearWarnings     clear
 server-blocked    Unblock
 warning           WARNING:
 error             ERROR:
+server-resolving   Resolving address
 
 # Configuration
 configuration     Configuration

main/sabnzbd/database.py

@@ -378,14 +378,10 @@ def unpack_history_info(item):
 def decode_factory(text):
     '''
         Recursivly looks through the supplied argument
-        and converts and text to utf-8
+        and converts and text to Unicode
     '''
     if isinstance(text, str):
-        if sabnzbd.WIN32:
-            return text.decode('Latin-1', 'replace').encode('utf-8', 'replace')
-        else:
-            return text.decode('utf-8', 'replace')
-
+        return unicoder(text)
 
     elif isinstance(text, list):
         new_text = []

main/sabnzbd/downloader.py

@@ -29,7 +29,7 @@ from nntplib import NNTPPermanentError
 import sabnzbd
 from sabnzbd.decorators import synchronized_CV, CV
 from sabnzbd.decoder import Decoder
-from sabnzbd.newswrapper import NewsWrapper
+from sabnzbd.newswrapper import NewsWrapper, request_server_info
 from sabnzbd.utils import osx
 from sabnzbd.constants import *
 import sabnzbd.config as config
@@ -205,6 +205,9 @@ class Server:
         self.active = True
         self.bad_cons = 0
         self.errormsg = ''
+        self.warning = ''
+        self.info = None     # Will hold getaddrinfo() list
+        self.request = False # True if a getaddrinfo() request is pending
 
         for i in range(threads):
             self.idle_threads.append(NewsWrapper(self, i+1))
@@ -219,7 +222,7 @@ class Server:
                 readers.pop(fno)
             if fno and fno in writers:
                 writers.pop(fno)
-            nw.terminate()
+            nw.terminate(quit=True)
         self.idle_threads = []
 
     def __repr__(self):
@@ -367,6 +370,18 @@ class Downloader(Thread):
                 return True
         return False
 
+    def maybe_block_server(self, server):
+        if server.optional and server.active and (server.bad_cons/server.threads) > 3:
+            # Optional and active server had too many problems,
+            # disable it now and send a re-enable plan to the scheduler
+            server.bad_cons = 0
+            server.active = False
+            server.errormsg = T('warn-ignoreServer@2') % ('', _PENALTY_TIMEOUT)
+            logging.warning(Ta('warn-ignoreServer@2'), server.id, _PENALTY_TIMEOUT)
+            self.plan_server(server.id, _PENALTY_TIMEOUT)
+            sabnzbd.nzbqueue.reset_all_try_lists()
+
+
     def run(self):
         self.decoder.start()
 
@@ -379,15 +394,7 @@ class Downloader(Thread):
                         else:
                             self.__reset_nw(nw, "timed out")
                         server.bad_cons += 1
-                        if server.optional and server.active and (server.bad_cons/server.threads) > 3:
-                            # Optional and active server had too many problems,
-                            # disable it now and send a re-enable plan to the scheduler
-                            server.bad_cons = 0
-                            server.active = False
-                            server.errormsg = T('warn-ignoreServer@2') % ('', _PENALTY_TIMEOUT)
-                            logging.warning(Ta('warn-ignoreServer@2'), server.id, _PENALTY_TIMEOUT)
-                            self.plan_server(server.id, _PENALTY_TIMEOUT)
-                            sabnzbd.nzbqueue.reset_all_try_lists()
+                        self.maybe_block_server(server)
                 if server.restart:
                     if not server.busy_threads:
                         newid = server.newid
@@ -419,6 +426,11 @@ class Downloader(Thread):
                     if not server.active:
                         break
 
+                    if server.info is None:
+                        self.maybe_block_server(server)
+                        request_server_info(server)
+                        break
+
                     article = sabnzbd.nzbqueue.get_article(server)
 
                     if not article:
@@ -465,10 +477,11 @@ class Downloader(Thread):
 
             if self.force_disconnect:
                 for server in self.servers:
-                    for nw in server.idle_threads[:]:
-                        self.__reset_nw(nw, "forcing disconnect", warn=False, wait=False)
-                    for nw in server.busy_threads[:]:
-                        self.__reset_nw(nw, "forcing disconnect", warn=False, wait=False)
+                    for nw in server.idle_threads + server.busy_threads:
+                        quit = nw.connected and server.active
+                        self.__reset_nw(nw, "forcing disconnect", warn=False, wait=False, quit=quit)
+                    # Make sure server address resolution is refreshed
+                    server.info = None
 
                 self.force_disconnect = False
 
@@ -571,7 +584,7 @@ class Downloader(Thread):
                                 if server.active:
                                     server.errormsg = Ta('error-serverTooMany@2') % ('', '')
                                     logging.error(Ta('error-serverTooMany@2'), server.host, server.port)
-                                    self.__reset_nw(nw, None, warn=False, destroy=True)
+                                    self.__reset_nw(nw, None, warn=False, destroy=True, quit=True)
                                     self.plan_server(server.id, _PENALTY_TOOMANY)
                                     server.threads -= 1
                             elif ecode in ('502', '481') and clues_too_many_ip(msg):
@@ -606,7 +619,7 @@ class Downloader(Thread):
                                         logging.info('Server %s ignored for %s minutes', server.id, penalty)
                                         self.plan_server(server.id, penalty)
                                     sabnzbd.nzbqueue.reset_all_try_lists()
-                                self.__reset_nw(nw, None, warn=False)
+                                self.__reset_nw(nw, None, warn=False, quit=True)
                             continue
                         except:
                             logging.error(Ta('error-serverFailed@4'),
@@ -647,7 +660,7 @@ class Downloader(Thread):
                             self.plan_server(server.id, 0)
                             sabnzbd.nzbqueue.reset_all_try_lists()
                         msg = T('error-serverCred@1') % ('%s:%s' % (nw.server.host, nw.server.port))
-                        self.__reset_nw(nw, msg)
+                        self.__reset_nw(nw, msg, quit=True)
 
                 if done:
                     logging.info('Thread %s@%s:%s: %s done',
@@ -659,8 +672,7 @@ class Downloader(Thread):
                     server.busy_threads.remove(nw)
                     server.idle_threads.append(nw)
 
-
-    def __reset_nw(self, nw, errormsg, warn=True, wait=True, destroy=False):
+    def __reset_nw(self, nw, errormsg, warn=True, wait=True, destroy=False, quit=False):
         server = nw.server
         article = nw.article
         fileno = None
@@ -670,7 +682,8 @@ class Downloader(Thread):
             nw.nntp.error_msg = None
 
         if warn and errormsg:
-            logging.warning('Thread %s@%s:%s: ' + errormsg,
+            server.warning = errormsg
+            logging.info('Thread %s@%s:%s: ' + errormsg,
                              nw.thrdnum, server.host, server.port)
         elif errormsg:
             logging.info('Thread %s@%s:%s: ' + errormsg,
@@ -697,9 +710,9 @@ class Downloader(Thread):
             sabnzbd.nzbqueue.reset_try_lists(nzf, nzo)
 
         if destroy:
-            nw.terminate()
+            nw.terminate(quit=quit)
         else:
-            nw.hard_reset(wait)
+            nw.hard_reset(wait, quit=quit)
 
     def __request_article(self, nw):
         try:
@@ -720,7 +733,7 @@ class Downloader(Thread):
                 self.read_fds[fileno] = nw
         except:
             logging.error(Ta('error-except'))
-            self.__reset_nw(nw, "server broke off connection")
+            self.__reset_nw(nw, "server broke off connection", quit=True)
 
     #------------------------------------------------------------------------------
     # Timed restart of servers admin.

main/sabnzbd/interface.py

@@ -2479,7 +2479,13 @@ class ConnectionInfo:
                 if nw.connected:
                     connected += 1
 
+            if server.warning and not (connected or server.errormsg):
+                connected = unicoder(server.warning)
+
+            if server.request and not server.info:
+                connected = T('server-resolving')
             busy.sort()
+
             header['servers'].append((server.host, server.port, connected, busy, server.ssl,
                                       server.active, server.errormsg, server.fillserver, server.optional))
 

main/sabnzbd/newsunpack.py

@@ -661,7 +661,7 @@ def par2_repair(parfile_nzf, nzo, workdir, setname):
             nzo.set_action_line(T('msg-repair'), T('msg-startRepair'))
             logging.info('Scanning "%s"', parfile)
 
-            joinables, zips, rars, ts = build_filelists(workdir, None)
+            joinables, zips, rars, ts = build_filelists(workdir, None, check_rar=False)
 
             finished, readd, pars, datafiles, used_joinables = PAR_Verify(parfile, parfile_nzf, nzo,
                                                                           setname, joinables)
@@ -1046,7 +1046,7 @@ def par_sort(a, b):
     elif bext == 'par2':
         return 1
 
-def build_filelists(workdir, workdir_complete):
+def build_filelists(workdir, workdir_complete, check_rar=True):
     joinables, zips, rars = ([], [], [])
 
     filelist = []
@@ -1060,11 +1060,14 @@ def build_filelists(workdir, workdir_complete):
             for _file in files:
                 filelist.append(os.path.join(root, _file))
 
-    joinables = [f for f in filelist if SPLITFILE_RE.search(f) and notrar(f)]
+    if check_rar:
+        joinables = [f for f in filelist if SPLITFILE_RE.search(f) and not israr(f)]
+    else:
+        joinables = [f for f in filelist if SPLITFILE_RE.search(f)]
 
     zips = [f for f in filelist if ZIP_RE.search(f)]
 
-    rars = [f for f in filelist if RAR_RE.search(f) and f not in joinables]
+    rars = [f for f in filelist if RAR_RE.search(f) and israr(f)]
 
     ts = [f for f in filelist if TS_RE.search(f) and f not in joinables]
 
@@ -1075,7 +1078,7 @@ def build_filelists(workdir, workdir_complete):
 
     return (joinables, zips, rars, ts)
 
-def notrar(f):
+def israr(f):
     logging.debug("notrar(): testing %s", f)
     try:
         _f = open(f, 'rb')
@@ -1087,9 +1090,9 @@ def notrar(f):
 
     if header != 'Rar!':
         logging.debug("notrar(): joinable file %s", f)
-        return True
+        return False
 
-    return False
+    return True
 
 
 def QuickCheck(set, nzo):

main/sabnzbd/newswrapper.py

@@ -49,6 +49,33 @@ import select
 
 socket.setdefaulttimeout(DEF_TIMEOUT)
 
+
+#------------------------------------------------------------------------------
+# getaddrinfo() can be very slow. In some situations this can lead
+# to delayed starts and timeouts on connections.
+# Because of this, the results will be cached in the server object.
+
+def _retrieve_info(server):
+    """ Async attempt to run getaddrinfo() for specified server
+    """
+    info = GetServerParms(server.host, server.port)
+
+    if info is None:
+        server.bad_cons += server.threads
+    else:
+        server.bad_cons = 0
+    (server.info, server.request) = (info, False)
+    sabnzbd.downloader.wakeup()
+
+
+def request_server_info(server):
+    """ Launch async request to resolve server address
+    """
+    if not server.request:
+        server.request = True
+        Thread(target=_retrieve_info, args=(server,)).start()
+
+
 def GetServerParms(host, port):
     # Make sure port is numeric (unicode input not supported)
     try:
@@ -99,17 +126,16 @@ def con(sock, host, port, sslenabled, nntp):
         nntp.error(e)
 
 class NNTP:
-    def __init__(self, host, port, sslenabled, nntp, user=None, password=None, block=False):
+    def __init__(self, host, port, info, sslenabled, nntp, user=None, password=None, block=False):
         self.host = host
         self.port = port
         self.nntp = nntp
         self.blocking = block
         self.error_msg = None
-        res= GetServerParms(self.host, self.port)
-        if not res:
+        if not info:
             raise socket.error(errno.EADDRNOTAVAIL, "Address not available - Check for internet or DNS problems")
 
-        af, socktype, proto, canonname, sa = res[0]
+        af, socktype, proto, canonname, sa = info[0]
 
         if sslenabled and _ssl:
             # Some users benefit from SSLv2 not being capped.
@@ -175,7 +201,8 @@ class NNTP:
         if self.blocking:
             raise socket.error(errno.ECONNREFUSED, msg)
         else:
-            logging.error(msg)
+            logging.info(msg)
+            self.nntp.server.warning = msg
 
 class NewsWrapper:
     def __init__(self, server, thrdnum, block=False):
@@ -202,7 +229,7 @@ class NewsWrapper:
         self.pass_ok = False
 
     def init_connect(self):
-        self.nntp = NNTP(self.server.host, self.server.port, self.server.ssl, self,
+        self.nntp = NNTP(self.server.host, self.server.port, self.server.info, self.server.ssl, self,
                          self.server.username, self.server.password, self.blocking)
         self.recv = self.nntp.sock.recv
 
@@ -283,11 +310,12 @@ class NewsWrapper:
         self.data = ''
         self.lines = []
 
-    def hard_reset(self, wait=True):
+    def hard_reset(self, wait=True, quit=True):
         if self.nntp:
             try:
-                self.nntp.sock.sendall('QUIT\r\n')
-                time.sleep(0.1)
+                if quit:
+                    self.nntp.sock.sendall('QUIT\r\n')
+                    time.sleep(0.1)
                 self.nntp.sock.close()
             except:
                 pass
@@ -302,12 +330,13 @@ class NewsWrapper:
             # Reset for internal reasons, just wait 5 sec
             self.timeout = time.time() + 5
 
-    def terminate(self):
+    def terminate(self, quit=False):
         """ Close connection and remove nntp object """
         if self.nntp:
             try:
-                self.nntp.sock.sendall('QUIT\r\n')
-                time.sleep(0.1)
+                if quit:
+                    self.nntp.sock.sendall('QUIT\r\n')
+                    time.sleep(0.1)
                 self.nntp.sock.close()
             except:
                 pass

main/sabnzbd/utils/servertests.py

@@ -118,4 +118,4 @@ def test_nntp_server(host, port, username=None, password=None, ssl=None):
         return False, T('srv-generalFail@1') % xml_name(nw.lines[0])
 
     # Close the connection
-    nw.terminate()
+    nw.terminate(quit=True)