Full repair-mode "--repair-all".

Reconstruct queue from "incomplete" folder using the saved nzb file and attributes.txt in the admin folder of the jobs.

SABnzbd.py

@@ -241,6 +241,8 @@ def print_help():
     print "  -c  --clean              Remove queue, cache and logs"
     print "  -p  --pause              Start in paused mode"
     print "      --repair             Try to reconstruct the queue from the incomplete folder"
+    print "      --repair-all         Try to reconstruct the queue from the incomplete folder"
+    print "                           with full data reconstruction"
     print "      --https <port>       Port to use for HTTPS server"
 
 def print_version():
@@ -729,7 +731,7 @@ def commandline_handler(frozen=True):
                                    ['pause', 'help', 'daemon', 'nobrowser', 'clean', 'logging=',
                                     'weblogging=', 'server=', 'templates',
                                     'template2', 'browser=', 'config-file=', 'force',
-                                    'version', 'https=', 'autorestarted', 'repair',
+                                    'version', 'https=', 'autorestarted', 'repair', 'repair-all',
                                     # Below Win32 Service options
                                     'password=', 'username=', 'startup=', 'perfmonini=', 'perfmondll=',
                                     'interactive', 'wait=',
@@ -794,7 +796,7 @@ def main():
     vista_plus = False
     vista64 = False
     force_web = False
-    repair = False
+    repair = 0
     re_argv = [sys.argv[0]]
 
     service, sab_opts, serv_opts, upload_nzbs = commandline_handler()
@@ -862,7 +864,10 @@ def main():
             re_argv.append(opt)
             re_argv.append(arg)
         elif opt in ('--repair',):
-            repair = True
+            repair = 1
+            pause = True
+        elif opt in ('--repair-all',):
+            repair = 2
             pause = True
 
     sabnzbd.MY_FULLNAME = os.path.normpath(os.path.abspath(sabnzbd.MY_FULLNAME))

sabnzbd/__init__.py

@@ -157,7 +157,7 @@ def connect_db(thread_index):
 
 
 @synchronized(INIT_LOCK)
-def initialize(pause_downloader = False, clean_up = False, evalSched=False, repair=False):
+def initialize(pause_downloader = False, clean_up = False, evalSched=False, repair=0):
     global __INITIALIZED__, __SHUTTING_DOWN__,\
            LOGFILE, WEBLOGFILE, LOGHANDLER, GUIHANDLER, AMBI_LOCALHOST, WAITEXIT, \
            DAEMON, MY_NAME, MY_FULLNAME, NEW_VERSION, \
@@ -227,7 +227,8 @@ def initialize(pause_downloader = False, clean_up = False, evalSched=False, repa
     except:
         BPSMeter.do.reset()
 
-    nzbqueue.init(repair)
+    nzbqueue.init()
+    nzbqueue.read_queue(repair)
 
     PostProcessor()
 
@@ -650,13 +651,13 @@ def CheckFreeSpace():
 IO_LOCK = RLock()
 
 @synchronized(IO_LOCK)
-def get_new_id(prefix, path):
-    """ Return unique prefixed admin identifier within 'savedir/__ADMIN__'
+def get_new_id(prefix, folder):
+    """ Return unique prefixed admin identifier within folder'
     """
     try:
-        fd, tpath = tempfile.mkstemp('', 'SABnzbd_%s_' % prefix, path)
+        fd, path = tempfile.mkstemp('', 'SABnzbd_%s_' % prefix, folder)
         os.close(fd)
-        head, tail = os.path.split(tpath)
+        head, tail = os.path.split(path)
         return tail
     except:
         logging.error(Ta('error-failMkstemp'))

sabnzbd/constants.py

@@ -57,6 +57,7 @@ POSTPROC_QUEUE_FILE_NAME  = 'postproc%s.sab' % POSTPROC_QUEUE_VERSION
 RSS_FILE_NAME    = 'rss_data.sab'
 BOOKMARK_FILE_NAME = 'bookmarks.sab'
 SCAN_FILE_NAME    = 'watched_data.sab'
+JOB_ADMIN = '__ADMIN__'
 
 DB_HISTORY_VERSION = 1
 DB_QUEUE_VERSION = 1

sabnzbd/misc.py

@@ -934,11 +934,11 @@ def get_filepath(path, nzo, filename):
 
 
 def get_admin_path(newstyle, name):
-    """ Return news-style full path to __ADMIN__ folder of names job
+    """ Return news-style full path to job-admin folder of names job
         or else the old cache path
     """
     if newstyle:
-        return os.path.join(os.path.join(cfg.download_dir.get_path(), name), '__ADMIN__')
+        return os.path.join(os.path.join(cfg.download_dir.get_path(), name), JOB_ADMIN)
     else:
        return cfg.cache_dir.get_path()
 
@@ -1406,9 +1406,13 @@ def remove_dir(path):
         os.rmdir(path)
 
 
-def remove_all(path):
+def remove_all(path, pattern='*'):
     """ Remove folder its content """
     if os.path.exists(path):
-        for f in glob.glob(os.path.join(path, '*')):
+        for f in glob.glob(os.path.join(path, pattern)):
             os.remove(f)
+        try:
             os.rmdir(path)
+        except:
+            pass
+

sabnzbd/nzbqueue.py

@@ -27,9 +27,9 @@ import glob
 
 import sabnzbd
 from sabnzbd.trylist import TryList
-from sabnzbd.nzbstuff import NzbObject
+from sabnzbd.nzbstuff import NzbObject, get_attrib_file, set_attrib_file
 from sabnzbd.misc import panic_queue, exit_sab, sanitize_foldername, cat_to_opts, \
-                         get_admin_path
+                         get_admin_path, remove_all
 import sabnzbd.database as database
 from sabnzbd.decorators import *
 from sabnzbd.constants import *
@@ -39,6 +39,7 @@ import sabnzbd.downloader
 from sabnzbd.assembler import Assembler
 from sabnzbd.lang import T, Ta
 from sabnzbd.utils import osx
+from sabnzbd.dirscanner import ProcessSingleFile
 
 
 def DeleteLog(name):
@@ -52,7 +53,7 @@ def DeleteLog(name):
 #-------------------------------------------------------------------------------
 
 class NzbQueue(TryList):
-    def __init__(self, repair):
+    def __init__(self):
         TryList.__init__(self)
 
         self.__top_only = cfg.top_only()
@@ -64,19 +65,33 @@ class NzbQueue(TryList):
         self.__auto_sort = cfg.auto_sort()
 
 
-        if repair:
-            # Reconstruct the queue from the content of the "incomplete" folder
+    def read_queue(self, repair):
+        """ Read queue from disk, supporting repair modes
+        """
+        if repair == 1:
+            # Reconstruct only the main queue file from the content of the "incomplete" folder
             for item in glob.glob(os.path.join(cfg.download_dir.get_path(), '*')):
-                path = os.path.join(item, '__ADMIN__')
-                wpath = os.path.join(path, 'SABnzbd_nzo_*')
-                nzo_id = glob.glob(wpath)
+                path = os.path.join(item, JOB_ADMIN)
+                nzo_id = glob.glob(os.path.join(path, 'SABnzbd_nzo_*'))
                 if len(nzo_id) == 1:
                     nzo = sabnzbd.load_data(os.path.basename(nzo_id[0]), path)
                     if nzo:
                         self.add(nzo, save = False)
 
+        elif repair == 2:
+            # Reconstruct all queue and job admin from the content of the "incomplete" folder
+            # rebuilding all data structures from the saved NZB files
+            for item in glob.glob(os.path.join(cfg.download_dir.get_path(), '*')):
+                name = os.path.basename(item)
+                path = os.path.join(item, JOB_ADMIN)
+                remove_all(path, 'SABnzbd_*')
+                cat, pp, script, prio = get_attrib_file(path, 4)
+                filename = glob.glob(os.path.join(path, '*.gz'))
+                if len(filename) == 1:
+                    ProcessSingleFile(name, filename[0], pp=pp, script=script, cat=cat, priority=prio, keep=True)
+
         else:
-            # Read the queue from the saved file
+            # Read the queue from the saved files
             nzo_ids = []
             data = sabnzbd.load_admin(QUEUE_FILE_NAME)
             if data:
@@ -112,6 +127,7 @@ class NzbQueue(TryList):
                 nzo_ids.append(nzo.nzo_id)
             if save_nzo is None or nzo is save_nzo:
                sabnzbd.save_data(nzo, nzo.nzo_id, nzo.get_workpath())
+               nzo.save_attribs()
 
         sabnzbd.save_admin((QUEUE_VERSION, nzo_ids, []), QUEUE_FILE_NAME)
 
@@ -730,12 +746,12 @@ def sort_queue_function(nzo_list, method, reverse):
 
 __NZBQ = None  # Global pointer to NzbQueue instance
 
-def init(repair):
+def init():
     global __NZBQ
     if __NZBQ:
-        __NZBQ.__init__(repair)
+        __NZBQ.__init__()
     else:
-        __NZBQ = NzbQueue(repair)
+        __NZBQ = NzbQueue()
 
 def start():
     global __NZBQ
@@ -755,6 +771,11 @@ def debug():
     global __NZBQ
     if __NZBQ: return __NZBQ.debug()
 
+
+def read_queue(repair):
+    global __NZBQ
+    if __NZBQ: __NZBQ.read_queue(repair)
+
 def move_up_bulk(nzo_id, nzf_ids):
     global __NZBQ
     if __NZBQ: __NZBQ.move_up_bulk(nzo_id, nzf_ids)

sabnzbd/nzbstuff.py

@@ -22,6 +22,7 @@ sabnzbd.nzbstuff - misc
 import os
 import time
 import re
+import glob
 import logging
 import datetime
 import xml.sax
@@ -35,11 +36,12 @@ except ImportError:
 import sabnzbd
 from sabnzbd.constants import *
 from sabnzbd.misc import to_units, cat_to_opts, cat_convert, sanitize_foldername, \
-                         get_unique_path, get_admin_path
+                         get_unique_path, get_admin_path, remove_all, \
+                         sanitize_filename
 import sabnzbd.cfg as cfg
 from sabnzbd.trylist import TryList
 from sabnzbd.lang import T, Ta
-from sabnzbd.encoding import unicoder, platform_encode
+from sabnzbd.encoding import unicoder, platform_encode, latin1
 
 RE_NEWZBIN = re.compile(r"msgid_(\w+) (.+)(\.nzb)$", re.I)
 RE_NORMAL  = re.compile(r"(.+)(\.nzb)", re.I)
@@ -549,10 +551,17 @@ class NzbObject(TryList):
             raise TypeError
 
         # Create "incomplete" folder
-        wp = get_unique_path(os.path.join(cfg.download_dir.get_path(), self.__dirname), create_dir=True)
-        if wp:
-            os.mkdir(os.path.join(wp, '__ADMIN__'))
-        wp, self.__dirname = os.path.split(wp)
+        wdir = os.path.join(cfg.download_dir.get_path(), self.__dirname)
+        adir = os.path.join(wdir, JOB_ADMIN)
+        nzo_file = glob.glob(os.path.join(adir, 'SABnzbd_nzo_*'))
+        reuse = os.path.exists(wdir) and not nzo_file
+        if reuse:
+            remove_all(adir, 'SABnzbd_*')
+        else:
+            wdir = get_unique_path(wdir, create_dir=True)
+        if not os.path.exists(adir):
+            os.mkdir(adir)
+        dummy, self.__dirname = os.path.split(wdir)
         self.__created = True
 
         # Must create a lower level XML parser because we must
@@ -579,7 +588,10 @@ class NzbObject(TryList):
             raise ValueError
 
         sabnzbd.backup_nzb(filename, nzb)
-        sabnzbd.save_compressed(self.get_workpath(), filename, nzb)
+        sabnzbd.save_compressed(adir, filename, nzb)
+
+        if reuse:
+            self.check_existing_files(wdir)
 
         if cat is None:
             for grp in self.__group:
@@ -711,6 +723,25 @@ class NzbObject(TryList):
 
         return (file_done, post_done, reset)
 
+
+    def check_existing_files(self, wdir):
+        """ Check if downloaded files already exits, for these set NZF to complete
+        """
+        wdir = os.path.join(wdir, '*')
+        files = [os.path.basename(f) for f in glob.glob(wdir) if os.path.isfile(f)]
+        for nzf in self.__files[:]:
+            alleged_name = nzf.get_filename()
+            subject = sanitize_filename(latin1(nzf.get_subject()))
+            ready = alleged_name in files
+            if not ready:
+                for f in files:
+                    if f in subject:
+                        ready = True
+                        break
+            if ready:
+                self.remove_nzf(nzf)
+
+
     def set_opts(self, pp):
         self.__repair, self.__unpack, self.__delete = sabnzbd.pp_to_opts(pp)
 
@@ -940,7 +971,7 @@ class NzbObject(TryList):
         return self.__dirname
 
     def get_workpath(self):
-        """ Return the full path for my __ADMIN__ folder (or old style cache) """
+        """ Return the full path for my job-admin folder (or old style cache) """
         return get_admin_path(self.extra5, self.__dirname)
 
     def get_dirname_rename(self):
@@ -1116,6 +1147,9 @@ class NzbObject(TryList):
     def set_md5pack(self, name, pack):
         self.md5packs[name] = pack
 
+    def save_attribs(self):
+        set_attrib_file(self.get_workpath(), (self.__cat, self.get_pp(), self.__script, self.__priority))
+
     def __build_pos_nzf_table(self, nzf_ids):
         pos_nzf_table = {}
         for nzf_id in nzf_ids:
@@ -1256,3 +1290,35 @@ def scan_password(name):
         return m.group(1).strip('. '), m.group(2).strip()
     else:
         return name.strip('. '), None
+
+
+def get_attrib_file(path, size):
+    """ Read job's attributes from file """
+    attribs = []
+    path = os.path.join(path, 'attributes.txt')
+    try:
+        f = open(path, 'r')
+    except:
+        return [None for n in xrange(size)]
+
+    for n in xrange(size):
+        line = f.readline()
+        if line:
+            attribs.append(line.strip('\n '))
+        else:
+            attribs.append(None)
+    f.close()
+    return attribs
+
+
+def set_attrib_file(path, attribs):
+    """ Write job's attributes to file """
+    path = os.path.join(path, 'attributes.txt')
+    try:
+        f = open(path, 'w')
+    except:
+        return
+
+    for item in attribs:
+        f.write('%s\n' % item)
+    f.close()

sabnzbd/postproc.py

@@ -283,7 +283,7 @@ class PostProcessor(Thread):
                 nzo.set_status('Moving')
                 nzo.set_action_line(T('msg-moving'), '...')
                 for root, dirs, files in os.walk(workdir):
-                    if not root.endswith('__ADMIN__'):
+                    if not root.endswith(JOB_ADMIN):
                         for _file in files:
                             path = os.path.join(root, _file)
                             new_path = path.replace(workdir, tmp_workdir_complete)
@@ -430,7 +430,7 @@ class PostProcessor(Thread):
             try:
                 if os.path.exists(workdir):
                     logging.debug('Removing workdir %s', workdir)
-                    remove_all(os.path.join(workdir, '__ADMIN__'))
+                    remove_all(os.path.join(workdir, JOB_ADMIN))
                     remove_dir(workdir)
             except:
                 logging.error(Ta('error-ppDelWorkdir@1'), workdir)