Linux: compensate for par2 renaming files to CP1252 on UTF-8 file systems.

Par2-classic on Unix systems will rename files to CP1252 when running on UTF-8 file systems.
On such systems, this renaming must be undone before further processing.

sabnzbd/misc.py

@@ -42,7 +42,7 @@ from sabnzbd.decorators import synchronized
 from sabnzbd.constants import DEFAULT_PRIORITY, FUTURE_Q_FOLDER, JOB_ADMIN, GIGI, Status, MEBI
 import sabnzbd.config as config
 import sabnzbd.cfg as cfg
-from sabnzbd.encoding import unicoder, latin1
+from sabnzbd.encoding import unicoder, latin1, special_fixer, gUTF
 import sabnzbd.growler as growler
 
 RE_VERSION = re.compile('(\d+)\.(\d+)\.(\d+)([a-zA-Z]*)(\d*)')
@@ -83,7 +83,12 @@ def globber_full(path, pattern=u'*'):
     """ Return matching full file/folder names in folder `path` """
     # Cannot use glob.glob() because it doesn't support Windows long name notation
     if os.path.exists(path):
-        return [os.path.join(path, f) for f in os.listdir(path) if fnmatch.fnmatch(f, pattern)]
+        try:
+            return [os.path.join(path, f) for f in os.listdir(path) if fnmatch.fnmatch(f, pattern)]
+        except UnicodeDecodeError:
+            # This happens on Linux when names are incorrectly encoded, retry using a non-Unicode path
+            path = path.encode('utf-8')
+            return [os.path.join(path, f) for f in os.listdir(path) if fnmatch.fnmatch(f, pattern)]
     else:
         return []
 
@@ -1423,3 +1428,16 @@ def long_path(path):
             # Normal form for local paths
             path = u'\\\\?\\' + path
     return path
+
+
+def fix_unix_encoding(folder):
+    """ Fix bad name encoding for Unix systems """
+    if not sabnzbd.WIN32 and not sabnzbd.DARWIN and gUTF:
+        for root, dirs, files in os.walk(folder.encode('utf-8')):
+            for name in files:
+                new_name = special_fixer(name).encode('utf-8')
+                if name != new_name:
+                    try:
+                        os.rename(os.path.join(root, name), os.path.join(root, new_name))
+                    except:
+                        logging.info('Cannot correct name of %s', os.path.join(root, name))

sabnzbd/nzbstuff.py

@@ -47,7 +47,8 @@ from sabnzbd.constants import sample_match, GIGI, ATTRIB_FILE, JOB_ADMIN, \
 from sabnzbd.misc import to_units, cat_to_opts, cat_convert, sanitize_foldername, \
                          get_unique_path, get_admin_path, remove_all, format_source_url, \
                          sanitize_filename, globber_full, sanitize_foldername, int_conv, \
-                         set_permissions, format_time_string, long_path, trim_win_path
+                         set_permissions, format_time_string, long_path, trim_win_path, \
+                         fix_unix_encoding
 import sabnzbd.cfg as cfg
 from sabnzbd.trylist import TryList
 from sabnzbd.encoding import unicoder, platform_encode, latin1, name_fixer
@@ -913,6 +914,8 @@ class NzbObject(TryList):
     def check_existing_files(self, wdir):
         """ Check if downloaded files already exits, for these set NZF to complete
         """
+        fix_unix_encoding(wdir)
+        
         # Get a list of already present files
         files = [os.path.basename(f) for f in globber_full(wdir) if os.path.isfile(f)]
 
@@ -920,7 +923,7 @@ class NzbObject(TryList):
         renames = sabnzbd.load_data(RENAMES_FILE, self.workpath, remove=True)
         if renames:
             for name in renames:
-                if name in files:
+                if name in files or renames[name] in files:
                     files.remove(name)
                     files.append(renames[name])
 
@@ -934,7 +937,7 @@ class NzbObject(TryList):
         # Flag files from NZB that already exist as finished
         for filename in files[:]:
             for nzf in nzfs:
-                subject = sanitize_filename(latin1(nzf.subject))
+                subject = sanitize_filename(name_extractor(nzf.subject))
                 if (nzf.filename == filename) or (subject == filename) or (filename in subject):
                     nzf.filename = filename
                     nzf.completed = True

sabnzbd/postproc.py

@@ -33,7 +33,7 @@ from threading import Thread
 from sabnzbd.misc import real_path, get_unique_path, create_dirs, move_to_path, \
                          make_script_path, short_path, long_path, clip_path, \
                          on_cleanup_list, renamer, remove_dir, remove_all, globber, globber_full, \
-                         set_permissions, cleanup_empty_directories, check_win_maxpath
+                         set_permissions, cleanup_empty_directories, check_win_maxpath, fix_unix_encoding
 from sabnzbd.tvsort import Sorter
 from sabnzbd.constants import REPAIR_PRIORITY, TOP_PRIORITY, POSTPROC_QUEUE_FILE_NAME, \
      POSTPROC_QUEUE_VERSION, sample_match, JOB_ADMIN, Status, VERIFIED_FILE
@@ -295,6 +295,7 @@ def process_job(nzo):
         marker_file = None
 
         if all_ok:
+            fix_unix_encoding(workdir)
             one_folder = False
             ## Determine class directory
             if cfg.create_group_folders():
@@ -575,7 +576,7 @@ def parring(nzo, workdir):
     else:
         logging.info("No par2 sets for %s", filename)
         nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename))
-        if cfg.sfv_check():
+        if cfg.sfv_check() and not verified.get(setname, False):
             par_error = not try_sfv_check(nzo, workdir, '')
             verified[''] = not par_error