Convert to Unicode application.

This breaks compatibility with existing 0.7.x queues.
Polish and Romanian no longer need conversion to latin1.

NSIS_Installer.nsi

@@ -25,6 +25,7 @@
 !include "LogicLib.nsh"
 !include "WinVer.nsh"
 !include "WinSxSQuery.nsh"
+!include "nsProcess.nsh"
 
 ;------------------------------------------------------------------
 ;
@@ -48,6 +49,7 @@
   Delete   "${idir}\email\email-pt_BR.tmpl"
   Delete   "${idir}\email\email-sr.tmpl"
   Delete   "${idir}\email\email-ru.tmpl"
+  Delete   "${idir}\email\email-zh_CN.tmpl"
   Delete   "${idir}\email\rss-de.tmpl"
   Delete   "${idir}\email\rss-en.tmpl"
   Delete   "${idir}\email\rss-nl.tmpl"
@@ -62,6 +64,7 @@
   Delete   "${idir}\email\rss-pt_BR.tmpl"
   Delete   "${idir}\email\rss-sr.tmpl"
   Delete   "${idir}\email\rss-ru.tmpl"
+  Delete   "${idir}\email\rss-zh_CN.tmpl"
   Delete   "${idir}\email\badfetch-da.tmpl"
   Delete   "${idir}\email\badfetch-de.tmpl"
   Delete   "${idir}\email\badfetch-en.tmpl"
@@ -76,6 +79,7 @@
   Delete   "${idir}\email\badfetch-es.tmpl"
   Delete   "${idir}\email\badfetch-pt_BR.tmpl"
   Delete   "${idir}\email\badfetch-ru.tmpl"
+  Delete   "${idir}\email\badfetch-zh_CN.tmpl"
   RMDir    "${idir}\email"
   RMDir /r "${idir}\locale"
   RMDir /r "${idir}\interfaces\Classic"
@@ -222,6 +226,9 @@
   !insertmacro MUI_LANGUAGE "Romanian"
   !insertmacro MUI_LANGUAGE "Spanish"
   !insertmacro MUI_LANGUAGE "PortugueseBR"
+  !insertmacro MUI_LANGUAGE "Serbian"
+  !insertmacro MUI_LANGUAGE "Russian"
+  !insertmacro MUI_LANGUAGE "SimpChinese"
 
 
 ;------------------------------------------------------------------
@@ -279,13 +286,13 @@ runtime_loop:
 ;make sure user terminates sabnzbd.exe or else abort
 ;
 loop:
-  StrCpy $0 "SABnzbd.exe"
-  KillProc::FindProcesses
-  StrCmp $0 "0" endcheck
-  MessageBox MB_OKCANCEL|MB_ICONEXCLAMATION $(MsgCloseSab) /SD IDCANCEL IDOK loop IDCANCEL exitinstall
+  ${nsProcess::FindProcess} "SABnzbd.exe" $R0
+  StrCmp $R0 0 0 endcheck
+  MessageBox MB_OKCANCEL|MB_ICONEXCLAMATION $(MsgCloseSab) IDOK loop IDCANCEL exitinstall
   exitinstall:
+    ${nsProcess::Unload}
     Abort
-  endcheck:
+endcheck:
 
 FunctionEnd
 
@@ -311,6 +318,7 @@ File /r "dist\*"
 
 
 WriteRegStr HKEY_LOCAL_MACHINE "SOFTWARE\SABnzbd" "" "$INSTDIR"
+WriteRegStr HKEY_LOCAL_MACHINE "SOFTWARE\SABnzbd" "Installer Language" "$(MsgLangCode)"
 WriteRegStr HKEY_LOCAL_MACHINE "Software\Microsoft\Windows\CurrentVersion\Uninstall\SABnzbd" "DisplayName" "SABnzbd ${SAB_VERSION}"
 WriteRegStr HKEY_LOCAL_MACHINE "Software\Microsoft\Windows\CurrentVersion\Uninstall\SABnzbd" "UninstallString" '"$INSTDIR\uninstall.exe"'
 WriteRegStr HKEY_LOCAL_MACHINE "Software\Microsoft\Windows\CurrentVersion\Uninstall\SABnzbd" "DisplayVersion" '${SAB_VERSION}'
@@ -360,29 +368,8 @@ UninstallText $(MsgUninstall)
 
 Section "un.$(MsgDelProgram)" Uninstall
 ;make sure sabnzbd.exe isnt running..if so shut it down
-
-  StrCpy $0 "sabnzbd.exe"
-  DetailPrint "Searching for processes called '$0'"
-  KillProc::FindProcesses
-  StrCmp $1 "-1" wooops
-  DetailPrint "-> Found $0 processes"
-
-  StrCmp $0 "0" completed
-  Sleep 1500
-
-  StrCpy $0 "sabnzbd.exe"
-  DetailPrint "Killing all processes called '$0'"
-  KillProc::KillProcesses
-  StrCmp $1 "-1" wooops
-  DetailPrint "-> Killed $0 processes, failed to kill $1 processes"
-
-  Goto completed
-
-  wooops:
-  DetailPrint "-> Error: Something went wrong :-("
-  Abort
-
-  completed:
+  ${nsProcess::KillProcess} "SABnzbd.exe" $R0
+  ${nsProcess::Unload}
   DetailPrint "Process Killed"
 
 
@@ -456,6 +443,7 @@ SectionEnd
 
   LangString MsgRemoveOld2  ${LANG_ENGLISH} "Your settings and data will be preserved."
 
+  LangString MsgLangCode    ${LANG_ENGLISH} "en"
 
 Function un.onInit
   !insertmacro MUI_UNGETLANGUAGE

README.md

@@ -1,6 +1,11 @@
 SABnzbd - The automated Usenet download tool
 ============================================
 
+# WARNING
+This Unicode release is not compatible with 0.7.x queues!
+
+----
+
 SABnzbd is an Open Source Binary Newsreader written in Python.
 
 It's totally free, incredibly easy to use, and works practically everywhere.

SABnzbd.py

@@ -20,6 +20,16 @@ if sys.version_info < (2, 5):
     print "Sorry, requires Python 2.5, 2.6 or 2.7."
     sys.exit(1)
 
+# Make sure UTF-8 is default 8bit encoding
+if not hasattr(sys,"setdefaultencoding"):
+    reload(sys)
+try:
+    sys.setdefaultencoding('utf-8')
+except:
+    print 'Sorry, you MUST add the SABnzbd folder to the PYTHONPATH environment variable'
+    print 'or find another way to force Python to use UTF-8 for string encoding.'
+    sys.exit(1)
+
 import logging
 import logging.handlers
 import os
@@ -63,6 +73,14 @@ except:
         else:
             SQLITE_DLL = False
 
+import locale, __builtin__
+try:
+    locale.setlocale(locale.LC_ALL, "")
+    __builtin__.__dict__['codepage'] = locale.getlocale()[1] or 'cp1252'
+except:
+    # Work-around for Python-ports with bad "locale" support
+    __builtin__.__dict__['codepage'] = 'cp1252'
+
 import sabnzbd
 import sabnzbd.lang
 import sabnzbd.interface
@@ -78,7 +96,7 @@ import sabnzbd.scheduler as scheduler
 import sabnzbd.config as config
 import sabnzbd.cfg
 import sabnzbd.downloader
-from sabnzbd.encoding import unicoder, latin1
+from sabnzbd.encoding import unicoder, latin1, deunicode
 import sabnzbd.growler as growler
 
 from threading import Thread
@@ -364,6 +382,7 @@ def CheckColor(color, web_dir):
 #------------------------------------------------------------------------------
 def fix_webname(name):
     if name:
+        name = deunicode(name)
         xname = name.title()
     else:
         xname = ''
@@ -417,8 +436,8 @@ def GetProfileInfo(vista_plus):
 
                 try:
                     # Conversion to 8bit ASCII required for CherryPy
-                    sabnzbd.DIR_APPDATA = sabnzbd.DIR_APPDATA.encode('latin-1')
-                    sabnzbd.DIR_HOME = sabnzbd.DIR_HOME.encode('latin-1')
+                    sabnzbd.DIR_APPDATA = sabnzbd.DIR_APPDATA.encode(codepage)
+                    sabnzbd.DIR_HOME = sabnzbd.DIR_HOME.encode(codepage)
                     ok = True
                 except:
                     # If unconvertible characters exist, use MSDOS name

package.py

@@ -252,12 +252,12 @@ else:
     PanDoc = None
 
 if os.name == 'nt':
-    msg = 'Requires the standard version of NSIS'
+    msg = 'Requires the Unicode version of NSIS'
     NSIS = CheckPath('makensis')
     if NSIS:
         log = '%s.log' % NSIS
         os.system('%s >%s' % (NSIS, log))
-        if 'Unicode' not in open(log).read():
+        if 'Unicode' in open(log).read():
             msg = ''
         delete_files(log)
     if msg:

sabnzbd/__init__.py

@@ -586,7 +586,7 @@ def add_nzbfile(nzbfile, pp=None, script=None, cat=None, priority=NORMAL_PRIORIT
     if script and script.lower()=='default': script = None
     if cat and cat.lower()=='default': cat = None
 
-    if isinstance(nzbfile, str):
+    if isinstance(nzbfile, str) or isinstance(nzbfile, unicode):
         # File coming from queue repair
         filename = nzbfile
         keep = True
@@ -594,7 +594,9 @@ def add_nzbfile(nzbfile, pp=None, script=None, cat=None, priority=NORMAL_PRIORIT
         # File coming from API/TAPI
         # Consider reception of Latin-1 names for non-Windows platforms
         # When an OSX/Unix server receives a file from Windows platform
-        filename = encoding.special_fixer(nzbfile.filename)
+        # CherryPy delivers filename as UTF-8 disguised as Unicode!
+        filename = nzbfile.filename.encode('cp1252').decode('utf-8')
+        filename = encoding.special_fixer(filename)
         keep = False
 
     if not sabnzbd.WIN32:
@@ -612,7 +614,15 @@ def add_nzbfile(nzbfile, pp=None, script=None, cat=None, priority=NORMAL_PRIORIT
     else:
         try:
             f, path = tempfile.mkstemp(suffix=ext, text=False)
+            # More CherryPy madness, sometimes content is in 'value', sometimes not.
+            if nzbfile.value:
                 os.write(f, nzbfile.value)
+            elif hasattr(nzbfile, 'file'):
+                # CherryPy 3.2.2 object
+                if hasattr(nzbfile.file, 'file'):
+                    os.write(f, nzbfile.file.file.read())
+                else:
+                    os.write(f, nzbfile.file.read())
             os.close(f)
         except:
             logging.error(Ta('Cannot create temp file for %s'), filename)

sabnzbd/api.py

@@ -294,6 +294,13 @@ def _api_addfile(name, output, kwargs):
         #Side effect of next line is that attribute .value is created
         #which is needed to make add_nzbfile() work
         size = name.length
+    elif hasattr(name, 'file') and hasattr(name, 'filename') and name.filename:
+        # CherryPy 3.2.2 object
+        if hasattr(name.file, 'file'):
+            name.value = name.file.file.read()
+        else:
+            name.value = name.file.read()
+        size = len(name.value)
     elif hasattr(name, 'value'):
         size = len(name.value)
     else:

sabnzbd/config.py

@@ -20,6 +20,7 @@ sabnzbd.config - Configuration Support
 """
 
 import os
+import re
 import logging
 import threading
 import shutil
@@ -683,19 +684,41 @@ def _read_config(path, try_backup=False):
             return False, 'Cannot create INI file %s' % path
 
     try:
-        CFG = configobj.ConfigObj(path)
+        fp = open(path, 'r')
+        lines = fp.read().split('\n')
+        fp.close()
+
         try:
-            if int(CFG['__version__']) > int(CONFIG_VERSION):
-                return False, "Incorrect version number %s in %s" % (CFG['__version__'], path)
-        except (KeyError, ValueError):
-            CFG['__version__'] = CONFIG_VERSION
-    except configobj.ConfigObjError, strerror:
+            # First try UTF-8 encoding
+            CFG = configobj.ConfigObj(lines, default_encoding='utf-8', encoding='utf-8')
+        except UnicodeDecodeError:
+            # Failed, enable retry
+            CFG = {}
+
+        if not re.search(r'utf[ -]*8', CFG.get('__encoding__', ''), re.I):
+            # INI file is still in 8bit ASCII encoding, so try Latin-1 instead
+            CFG = configobj.ConfigObj(lines, default_encoding='cp1252', encoding='cp1252')
+
+    except (IOError, configobj.ConfigObjError, UnicodeEncodeError), strerror:
         if try_backup:
+            if isinstance(strerror, UnicodeEncodeError):
+                strerror = 'Character encoding of the file is inconsistent'
             return False, '"%s" is not a valid configuration file<br>Error message: %s' % (path, strerror)
         else:
+            # Try backup file
             return _read_config(path, True)
 
-    CFG['__version__'] = CONFIG_VERSION
+    try:
+        version = sabnzbd.misc.int_conv(CFG['__version__'])
+        if version > int(CONFIG_VERSION):
+            return False, "Incorrect version number %s in %s" % (version, path)
+    except (KeyError, ValueError):
+        pass
+
+    CFG.filename = path
+    CFG.encoding = 'utf-8'
+    CFG['__encoding__'] = u'utf-8'
+    CFG['__version__'] = unicode(CONFIG_VERSION)
 
     if 'misc' in CFG:
         compatibility_fix(CFG['misc'])

sabnzbd/constants.py

@@ -15,10 +15,10 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-CONFIG_VERSION = 19
+CONFIG_VERSION = 20
 
-QUEUE_VERSION = 9
-POSTPROC_QUEUE_VERSION = 1
+QUEUE_VERSION = 10
+POSTPROC_QUEUE_VERSION = 2
 
 PNFO_REPAIR_FIELD = 0
 PNFO_UNPACK_FIELD = 1
@@ -53,7 +53,7 @@ GIGI = float(2 ** 30)
 MEBI = float(2 ** 20)
 KIBI = float(2 ** 10)
 
-BYTES_FILE_NAME  = 'totals%s.sab' % QUEUE_VERSION
+BYTES_FILE_NAME  = 'totals9.sab'
 QUEUE_FILE_TMPL  = 'queue%s.sab'
 QUEUE_FILE_NAME  =  QUEUE_FILE_TMPL % QUEUE_VERSION
 POSTPROC_QUEUE_FILE_NAME  = 'postproc%s.sab' % POSTPROC_QUEUE_VERSION

sabnzbd/database.py

@@ -83,7 +83,7 @@ class HistoryDB(object):
         else:
             create_table = False
         if sabnzbd.WIN32 and isinstance(db_path, str):
-            self.con = sqlite3.connect(db_path.decode('latin-1').encode('utf-8'))
+            self.con = sqlite3.connect(db_path.decode('cp1252').encode('utf-8'))
         else:
             self.con = sqlite3.connect(db_path)
         self.con.row_factory = dict_factory

sabnzbd/encoding.py

@@ -56,34 +56,26 @@ def reliable_unpack_names():
         return gUTF
 
 def platform_encode(p):
-    """ Return the correct encoding for the platform:
-        Latin-1 for Windows/Posix-non-UTF and UTF-8 for OSX/Posix-UTF
+    """ Return Unicode name, if not already Unicode, decode with UTF-8 or latin1
     """
-    if isinstance(p, unicode):
-        if gUTF:
-            return p.encode('utf-8')
-        else:
-            return p.encode('latin-1', 'replace')
-    elif isinstance(p, basestring):
-        if gUTF:
-            try:
-                p.decode('utf-8')
+    if p is None or isinstance(p, unicode):
         return p
-            except:
-                return p.decode('latin-1').encode('utf-8')
     else:
         try:
-                return p.decode('utf-8').encode('latin-1', 'replace')
+            return p.decode('utf-8')
         except:
-                return p
-    else:
-        return p
+            return p.decode('cp1252')
 
 def name_fixer(p):
-    """ Return UTF-8 encoded string, if appropriate for the platform """
-
-    if gUTF and p:
-        return p.decode('Latin-1', 'replace').encode('utf-8', 'replace').replace('?', '_')
+    """ Return Unicode name of 8bit ASCII string, first try UTF-8, then cp1252
+    """
+    if isinstance(p, unicode):
+        return p
+    elif isinstance(p, str):
+        try:
+            return p.decode('utf-8')
+        except:
+            return p.decode('cp1252')
     else:
         return p
 
@@ -104,24 +96,17 @@ def special_fixer(p):
         Also takes care of the situation where a non-Windows/UTF-8 system
         receives a latin-1 encoded name.
     """
-    if sabnzbd.WIN32:
-        try:
-            return p.decode('utf-8').encode('latin-1', 'replace').replace('?', '_')
-        except:
+    if isinstance(p, unicode):
         return p
-    else:
-        if gUTF:
     try:
         # First see if it isn't just UTF-8
         p.decode('utf-8')
         if sabnzbd.DARWIN and '&#' in p:
             p = fixup_ff4(p)
-                return p
+        return p.decode('utf-8')
     except:
-                # Now assume it's latin-1
-                return p.decode('Latin-1').encode('utf-8')
-        else:
-            return p
+        # Now assume it's 8bit ASCII
+        return p.decode('cp1252')
 
 def unicoder(p):
     """ Make sure a Unicode string is returned """
@@ -132,8 +117,8 @@ def unicoder(p):
             try:
                 return p.decode('utf-8')
             except:
-                return p.decode('latin-1', 'replace')
-        return p.decode('latin-1', 'replace')
+                return p.decode('cp1252', 'replace')
+        return p.decode('cp1252', 'replace')
     else:
         return unicode(str(p))
 
@@ -141,11 +126,7 @@ def unicode2local(p):
     """ Convert Unicode filename to appropriate local encoding
         Leave ? characters for uncovertible characters
     """
-    if sabnzbd.WIN32:
-        return p.encode('Latin-1', 'replace')
-    else:
-        return p.encode('utf-8', 'replace')
-
+    return p
 
 def xml_name(p, keep_escape=False, encoding=None):
     """ Prepare name for use in HTML/XML contect """
@@ -158,7 +139,7 @@ def xml_name(p, keep_escape=False, encoding=None):
         elif gUTF:
             p = p.decode('utf-8', 'replace')
         else:
-            p = p.decode('Latin-1', 'replace')
+            p = p.decode(codepage, 'replace')
     else:
         p = str(p)
 
@@ -170,13 +151,7 @@ def xml_name(p, keep_escape=False, encoding=None):
 
 def latin1(txt):
     """ When Unicode or UTF-8, convert to Latin-1 """
-    if isinstance(txt, unicode):
-        return txt.encode('latin-1', 'replace').replace('?', '_')
-    elif txt and gUTF:
-        #return unicodedata.normalize('NFC', txt.decode('utf-8')).encode('latin-1', 'replace').replace('?', '_')
-        return txt.decode('utf-8').encode('latin-1', 'replace').replace('?', '_')
-    else:
-        return txt
+    return unicoder(txt)
 
 
 def encode_for_xml(ustr, encoding='ascii'):
@@ -187,7 +162,7 @@ def encode_for_xml(ustr, encoding='ascii'):
     if isinstance(ustr, unicode):
         pass
     elif isinstance(ustr, str):
-        ustr = ustr.decode('Latin-1', 'replace')
+        ustr = ustr.decode(codepage, 'replace')
     else:
         ustr = unicode(str(ustr))
     return ustr.encode(encoding, 'xmlcharrefreplace')
@@ -195,7 +170,7 @@ def encode_for_xml(ustr, encoding='ascii'):
 
 def titler(p):
     """ title() replacement
-        Python's title() fails with Latin-1, so use Unicode detour.
+        Python's title() fails with 8bit ASCII, so use Unicode detour.
     """
     if isinstance(p, unicode):
         return p.title()
@@ -203,9 +178,9 @@ def titler(p):
         try:
             return p.decode('utf-8').title().encode('utf-8')
         except:
-            return p.decode('latin-1', 'replace').title().encode('latin-1', 'replace')
+            return p.decode(codepage, 'replace').title().encode(codepage, 'replace')
     else:
-        return p.decode('latin-1', 'replace').title().encode('latin-1', 'replace')
+        return p.decode(codepage, 'replace').title().encode(codepage, 'replace')
 
 
 class LatinFilter(Filter):
@@ -216,11 +191,11 @@ class LatinFilter(Filter):
         elif isinstance(val, basestring):
             try:
                 if sabnzbd.WIN32:
-                    return val.decode('latin-1')
+                    return val.decode(codepage)
                 else:
                     return val.decode('utf-8')
             except:
-                return val.decode('latin-1', 'replace')
+                return val.decode(codepage, 'replace')
         elif val is None:
             return u''
         else:
@@ -228,7 +203,7 @@ class LatinFilter(Filter):
 
 class EmailFilter(Filter):
     """ Make sure Cheetah gets only Unicode strings
-        First try utf-8, then latin1
+        First try utf-8, then 8bit ASCII
     """
     def filter(self, val, str=str, **kw):
         if isinstance(val, unicode):
@@ -237,7 +212,7 @@ class EmailFilter(Filter):
             try:
                 return val.decode('utf-8')
             except:
-                return val.decode('latin-1', 'replace')
+                return val.decode(codepage, 'replace')
         elif val is None:
             return u''
         else:
@@ -339,4 +314,30 @@ def html_escape(txt):
     else:
         return txt
 
+
+def deunicode(p):
+    """ Return the correct 8bit ASCII encoding for the platform:
+        Latin-1 for Windows/Posix-non-UTF and UTF-8 for OSX/Posix-UTF
+    """
+    if isinstance(p, unicode):
+        if gUTF:
+            return p.encode('utf-8')
+        else:
+            return p.encode('cp1252', 'replace')
+    elif isinstance(p, basestring):
+        if gUTF:
+            try:
+                p.decode('utf-8')
+                return p
+            except:
+                return p.decode('cp1252').encode('utf-8')
+        else:
+            try:
+                return p.decode('utf-8').encode('cp1252', 'replace')
+            except:
+                return p
+    else:
+        return p
+
+
 auto_fsys()

sabnzbd/growler.py

@@ -194,9 +194,11 @@ def send_growl(title , msg, gtype):
         if _GROWL:
             assert isinstance(_GROWL, GrowlNotifier)
             _GROWL_REG = True
-            if not isinstance(msg, str) and not isinstance(msg, unicode):
+            if isinstance(msg, unicode):
+                msg = msg.decode('utf-8')
+            elif not isinstance(msg, str):
                 msg = str(msg)
-            logging.debug('Send to Growl: %s %s %s', gtype, latin1(title), latin1(msg))
+            logging.debug('Send to Growl: %s %s %s', gtype, title, msg)
             try:
                 ret = _GROWL.notify(
                     noteType = Tx(NOTIFICATION.get(gtype, 'other')),
@@ -264,7 +266,7 @@ if _HAVE_NTFOSD:
             icon = os.path.join(sabnzbd.DIR_PROG, 'sabnzbd.ico')
             _NTFOSD = _NTFOSD or pynotify.init('icon-summary-body')
             if _NTFOSD:
-                logging.info('Send to NotifyOSD: %s / %s', latin1(title), latin1(message))
+                logging.info('Send to NotifyOSD: %s / %s', title, message)
                 try:
                     note = pynotify.Notification(title, message, icon)
                     note.show()

sabnzbd/interface.py

@@ -356,7 +356,8 @@ class MainPage(object):
         if msg: return msg
 
         nzbfile = kwargs.get('nzbfile')
-        if nzbfile is not None and nzbfile.filename and nzbfile.value:
+        if nzbfile is not None and nzbfile.filename:
+            if nzbfile.value or nzbfile.file:
                 sabnzbd.add_nzbfile(nzbfile, kwargs.get('pp'), kwargs.get('script'),
                                     kwargs.get('cat'), kwargs.get('priority', NORMAL_PRIORITY))
         raise dcRaiser(self.__root, kwargs)

sabnzbd/lang.py

@@ -42,6 +42,7 @@ __all__ = ['set_locale_info', 'set_language', 'list_languages']
 
 _DOMAIN = ''        # Holds translation domain
 _LOCALEDIR = ''     # Holds path to the translation base folder
+CODEPAGE = '1252'
 
 
 def set_locale_info(domain, localedir):
@@ -55,7 +56,9 @@ def set_locale_info(domain, localedir):
 def set_language(language=None):
     """ Activate language, empty language will set default texts.
     """
+    global CODEPAGE
     if not language: language = ''
+    CODEPAGE = str(LanguageTable.get(language, (0, 0, 0))[2] or 1252)
 
     # 'codeset' will determine the output of lgettext
     lng = gettext.translation(_DOMAIN, _LOCALEDIR, [language], fallback=True, codeset='latin-1')
@@ -63,9 +66,9 @@ def set_language(language=None):
     # The unicode flag will make _() return Unicode
     lng.install(unicode=True, names=['lgettext'])
     __builtin__.__dict__['T'] = __builtin__.__dict__['_']           # Unicode
-    __builtin__.__dict__['Ta'] = __builtin__.__dict__['lgettext']   # Latin-1
+    __builtin__.__dict__['Ta'] = __builtin__.__dict__['_']          # Unicode (Used to Latin-1, compatibility support)
     __builtin__.__dict__['Tx'] = __builtin__.__dict__['_']          # Dynamic translation (unicode)
-    __builtin__.__dict__['TT'] = lambda x:x                         # Use in text tables
+    __builtin__.__dict__['TT'] = lambda x:unicode(x)                # Use in text tables
 
 
 def list_languages():
@@ -106,94 +109,97 @@ def list_languages():
 
 
 LanguageTable = {
-    'aa' : ('Afar', 'Afaraf'),
-    'af' : ('Afrikaans', 'Afrikaans'),
-    'ak' : ('Akan', 'Akan'),
-    'sq' : ('Albanian', 'Shqip'),
-    'an' : ('Aragonese', 'Aragonés'),
-    'ae' : ('Avestan', 'Avesta'),
-    'ay' : ('Aymara', 'Aymararu'),
-    'bm' : ('Bambara', 'Bamanankan'),
-    'eu' : ('Basque', 'Euskara'),
-    'bi' : ('Bislama', 'Bislama'),
-    'bs' : ('Bosnian', 'Bosanskijezik'),
-    'br' : ('Breton', 'Brezhoneg'),
-    'ca' : ('Catalan', 'Català'),
-    'ch' : ('Chamorro', 'Chamoru'),
-    'kw' : ('Cornish', 'Kernewek'),
-    'co' : ('Corsican', 'Corsu'),
-    'hr' : ('Croatian', 'Hrvatski'),
-    'cs' : ('Czech', 'Cesky, ceština'),
-    'da' : ('Danish', 'Dansk'),
-    'nl' : ('Dutch', 'Nederlands'),
-    'en' : ('English', 'English'),
-    'eo' : ('Esperanto', 'Esperanto'),
-    'et' : ('Estonian', 'Eesti'),
-    'fo' : ('Faroese', 'Føroyskt'),
-    'fj' : ('Fijian', 'Vosa Vakaviti'),
-    'fi' : ('Finnish', 'Suomi'),
-    'fr' : ('French', 'Français'),
-    'gl' : ('Galician', 'Galego'),
-    'de' : ('German', 'Deutsch'),
-    'hz' : ('Herero', 'Otjiherero'),
-    'ho' : ('Hiri Motu', 'Hiri Motu'),
-    'hu' : ('Hungarian', 'Magyar'),
-    'id' : ('Indonesian', 'Bahasa Indonesia'),
-    'ga' : ('Irish', 'Gaeilge'),
-    'io' : ('Ido', 'Ido'),
-    'is' : ('Icelandic', 'Íslenska'),
-    'it' : ('Italian', 'Italiano'),
-    'jv' : ('Javanese', 'BasaJawa'),
-    'rw' : ('Kinyarwanda', 'Ikinyarwanda'),
-    'kg' : ('Kongo', 'KiKongo'),
-    'kj' : ('Kwanyama', 'Kuanyama'),
-    'la' : ('Latin', 'Lingua latina'),
-    'lb' : ('Luxembourgish', 'Lëtzebuergesch'),
-    'lg' : ('Luganda', 'Luganda'),
-    'li' : ('Limburgish', 'Limburgs'),
-    'ln' : ('Lingala', 'Lingála'),
-    'lt' : ('Lithuanian', 'Lietuviukalba'),
-    'lv' : ('Latvian', 'Latviešuvaloda'),
-    'gv' : ('Manx', 'Gaelg'),
-    'mg' : ('Malagasy', 'Malagasy fiteny'),
-    'mt' : ('Maltese', 'Malti'),
-    'nb' : ('Norwegian Bokmål', 'Norsk bokmål'),
-    'nn' : ('Norwegian Nynorsk', 'Norsk nynorsk'),
-    'no' : ('Norwegian', 'Norsk'),
-    'oc' : ('Occitan', 'Occitan'),
-    'om' : ('Oromo', 'Afaan Oromoo'),
-    'pl' : ('Polish', 'Polski'),
-    'pt' : ('Portuguese', 'Português'),
-    'pt_BR' : ('Portuguese Brazillian', 'Português Brasileiro'),
-    'rm' : ('Romansh', 'Rumantsch grischun'),
-    'rn' : ('Kirundi', 'kiRundi'),
-    'ro' : ('Romanian', 'Româna'),
-    'sc' : ('Sardinian', 'Sardu'),
-    'se' : ('Northern Sami', 'Davvisámegiella'),
-    'sm' : ('Samoan', 'Gagana fa\'a Samoa'),
-    'gd' : ('Gaelic', 'Gàidhlig'),
-    'sn' : ('Shona', 'Chi Shona'),
-    'sk' : ('Slovak', 'Slovencina'),
-    'sl' : ('Slovene', 'Slovenšcina'),
-    'st' : ('Southern Sotho', 'Sesotho'),
-    'es' : ('Spanish Castilian', 'Español, castellano'),
-    'su' : ('Sundanese', 'Basa Sunda'),
-    'sw' : ('Swahili', 'Kiswahili'),
-    'ss' : ('Swati', 'SiSwati'),
-    'sv' : ('Swedish', 'Svenska'),
-    'tn' : ('Tswana', 'Setswana'),
-    'to' : ('Tonga (Tonga Islands)', 'faka Tonga'),
-    'tr' : ('Turkish', 'Türkçe'),
-    'ts' : ('Tsonga', 'Xitsonga'),
-    'tw' : ('Twi', 'Twi'),
-    'ty' : ('Tahitian', 'Reo Tahiti'),
-    'wa' : ('Walloon', 'Walon'),
-    'cy' : ('Welsh', 'Cymraeg'),
-    'wo' : ('Wolof', 'Wollof'),
-    'fy' : ('Western Frisian', 'Frysk'),
-    'xh' : ('Xhosa', 'isi Xhosa'),
-    'yo' : ('Yoruba', 'Yorùbá'),
-    'zu' : ('Zulu', 'isi Zulu'),
+    'aa' : ('Afar', 'Afaraf', 0),
+    'af' : ('Afrikaans', 'Afrikaans', 0),
+    'ak' : ('Akan', 'Akan', 0),
+    'sq' : ('Albanian', 'Shqip', 0),
+    'an' : ('Aragonese', 'Aragonés', 0),
+    'ae' : ('Avestan', 'Avesta', 0),
+    'ay' : ('Aymara', 'Aymararu', 0),
+    'bm' : ('Bambara', 'Bamanankan', 0),
+    'eu' : ('Basque', 'Euskara', 0),
+    'bi' : ('Bislama', 'Bislama', 0),
+    'bs' : ('Bosnian', 'Bosanskijezik', 0),
+    'br' : ('Breton', 'Brezhoneg', 0),
+    'ca' : ('Catalan', 'Català', 0),
+    'ch' : ('Chamorro', 'Chamoru', 0),
+    'kw' : ('Cornish', 'Kernewek', 0),
+    'co' : ('Corsican', 'Corsu', 0),
+    'hr' : ('Croatian', 'Hrvatski', 0),
+    'cs' : ('Czech', 'Cesky, ceština', 0),
+    'da' : ('Danish', 'Dansk', 0),
+    'nl' : ('Dutch', 'Nederlands', 0),
+    'en' : ('English', 'English', 0),
+    'eo' : ('Esperanto', 'Esperanto', 0),
+    'et' : ('Estonian', 'Eesti', 0),
+    'fo' : ('Faroese', 'Føroyskt', 0),
+    'fj' : ('Fijian', 'Vosa Vakaviti', 0),
+    'fi' : ('Finnish', 'Suomi', 0),
+    'fr' : ('French', 'Français', 0),
+    'gl' : ('Galician', 'Galego', 0),
+    'de' : ('German', 'Deutsch', 0),
+    'hz' : ('Herero', 'Otjiherero', 0),
+    'ho' : ('Hiri Motu', 'Hiri Motu', 0),
+    'hu' : ('Hungarian', 'Magyar', 0),
+    'id' : ('Indonesian', 'Bahasa Indonesia', 0),
+    'ga' : ('Irish', 'Gaeilge', 0),
+    'io' : ('Ido', 'Ido', 0),
+    'is' : ('Icelandic', 'Íslenska', 0),
+    'it' : ('Italian', 'Italiano', 0),
+    'jv' : ('Javanese', 'BasaJawa', 0),
+    'rw' : ('Kinyarwanda', 'Ikinyarwanda', 0),
+    'kg' : ('Kongo', 'KiKongo', 0),
+    'kj' : ('Kwanyama', 'Kuanyama', 0),
+    'la' : ('Latin', 'Lingua latina', 0),
+    'lb' : ('Luxembourgish', 'Lëtzebuergesch', 0),
+    'lg' : ('Luganda', 'Luganda', 0),
+    'li' : ('Limburgish', 'Limburgs', 0),
+    'ln' : ('Lingala', 'Lingála', 0),
+    'lt' : ('Lithuanian', 'Lietuviukalba', 0),
+    'lv' : ('Latvian', 'Latviešuvaloda', 0),
+    'gv' : ('Manx', 'Gaelg', 0),
+    'mg' : ('Malagasy', 'Malagasy fiteny', 0),
+    'mt' : ('Maltese', 'Malti', 0),
+    'nb' : ('Norwegian Bokmål', 'Norsk bokmål', 0),
+    'nn' : ('Norwegian Nynorsk', 'Norsk nynorsk', 0),
+    'no' : ('Norwegian', 'Norsk', 0),
+    'oc' : ('Occitan', 'Occitan', 0),
+    'om' : ('Oromo', 'Afaan Oromoo', 0),
+    'pl' : ('Polish', 'Polski', 0),
+    'pt' : ('Portuguese', 'Português', 0),
+    'pt_BR' : ('Portuguese Brazillian', 'Português Brasileiro', 0),
+    'rm' : ('Romansh', 'Rumantsch grischun', 0),
+    'rn' : ('Kirundi', 'kiRundi', 0),
+    'ro' : ('Romanian', 'Româna', 1250),
+    'sc' : ('Sardinian', 'Sardu', 0),
+    'se' : ('Northern Sami', 'Davvisámegiella', 0),
+    'sm' : ('Samoan', 'Gagana fa\'a Samoa', 0),
+    'gd' : ('Gaelic', 'Gàidhlig', 0),
+    'ru' : ('Russian', 'русский язык', 1251),
+    'sr' : ('Serbian', 'српски', 1251),
+    'sn' : ('Shona', 'Chi Shona', 0),
+    'sk' : ('Slovak', 'Slovencina', 0),
+    'sl' : ('Slovene', 'Slovenšcina', 0),
+    'st' : ('Southern Sotho', 'Sesotho', 0),
+    'es' : ('Spanish Castilian', 'Español, castellano', 0),
+    'su' : ('Sundanese', 'Basa Sunda', 0),
+    'sw' : ('Swahili', 'Kiswahili', 0),
+    'ss' : ('Swati', 'SiSwati', 0),
+    'sv' : ('Swedish', 'Svenska', 0),
+    'tn' : ('Tswana', 'Setswana', 0),
+    'to' : ('Tonga (Tonga Islands)', 'faka Tonga', 0),
+    'tr' : ('Turkish', 'Türkçe', 0),
+    'ts' : ('Tsonga', 'Xitsonga', 0),
+    'tw' : ('Twi', 'Twi', 0),
+    'ty' : ('Tahitian', 'Reo Tahiti', 0),
+    'wa' : ('Walloon', 'Walon', 0),
+    'cy' : ('Welsh', 'Cymraeg', 0),
+    'wo' : ('Wolof', 'Wollof', 0),
+    'fy' : ('Western Frisian', 'Frysk', 0),
+    'xh' : ('Xhosa', 'isi Xhosa', 0),
+    'yo' : ('Yoruba', 'Yorùbá', 0),
+    'zu' : ('Zulu', 'isi Zulu', 0),
+    'zh_CN' : ('SimpChinese', '简体中文', 936),
 }
 
 # Setup a safe null-translation

sabnzbd/misc.py

@@ -222,8 +222,8 @@ def sanitize_filename(name):
 
 FL_ILLEGAL = CH_ILLEGAL + ':\x92"'
 FL_LEGAL   = CH_LEGAL +   "-''"
-uFL_ILLEGAL = FL_ILLEGAL.decode('latin-1')
-uFL_LEGAL   = FL_LEGAL.decode('latin-1')
+uFL_ILLEGAL = FL_ILLEGAL.decode('cp1252')
+uFL_LEGAL   = FL_LEGAL.decode('cp1252')
 
 def sanitize_foldername(name):
     """ Return foldername with dodgy chars converted to safe ones
@@ -405,20 +405,7 @@ def get_user_shellfolders():
     try:
         for i in range(0, _winreg.QueryInfoKey(key)[1]):
             name, value, val_type = _winreg.EnumValue(key, i)
-            try:
-                values[name] = value.encode('latin-1')
-            except UnicodeEncodeError:
-                try:
-                    # If the path name cannot be converted to latin-1 (contains high ASCII value strings)
-                    # then try and use the short name
-                    import win32api
-                    # Need to make sure the path actually exists, otherwise ignore
-                    if os.path.exists(value):
-                        values[name] = win32api.GetShortPathName(value)
-                except:
-                    # probably a pywintypes.error error such as folder does not exist
-                    logging.error("Traceback: ", exc_info = True)
-                    values[name] = 'c:\\'
+            values[name] = value
             i += 1
         _winreg.CloseKey(key)
         _winreg.CloseKey(hive)
@@ -1039,35 +1026,30 @@ def loadavg():
 
 def format_time_string(seconds, days=0):
     """ Return a formatted and translated time string """
+
+    def unit(single, n):
+        if n == 1:
+            return sabnzbd.api.Ttemplate(single)
+        else:
+            return sabnzbd.api.Ttemplate(single + 's')
+
     seconds = int_conv(seconds)
     completestr = []
     if days:
-        completestr.append('%s %s' % (days, s_returner('day', days)))
+        completestr.append('%s %s' % (days, unit('day', days)))
     if (seconds/3600) >= 1:
-        completestr.append('%s %s' % (seconds/3600, s_returner('hour', (seconds/3600))))
+        completestr.append('%s %s' % (seconds/3600, unit('hour', (seconds/3600))))
         seconds -= (seconds/3600)*3600
     if (seconds/60) >= 1:
-        completestr.append('%s %s' % (seconds/60, s_returner('minute',(seconds/60))))
+        completestr.append('%s %s' % (seconds/60, unit('minute',(seconds/60))))
         seconds -= (seconds/60)*60
     if seconds > 0:
-        completestr.append('%s %s' % (seconds, s_returner('second', seconds)))
+        completestr.append('%s %s' % (seconds, unit('second', seconds)))
     elif not completestr:
-        completestr.append('0 %s' % s_returner('second', 0))
-
-    p = ' '.join(completestr)
-    if isinstance(p, unicode):
-        return p.encode('latin-1')
-    else:
-        return p
+        completestr.append('0 %s' % unit('second', 0))
 
+    return ' '.join(completestr)
 
-def s_returner(item, value):
-    """ Return a plural form of 'item', based on 'value' (english only)
-    """
-    if value == 1:
-        return Tx(item)
-    else:
-        return Tx(item + 's')
 
 def int_conv(value):
     """ Safe conversion to int (can handle None)

sabnzbd/newsunpack.py

@@ -29,7 +29,7 @@ import binascii
 
 import sabnzbd
 from sabnzbd.encoding import TRANS, UNTRANS, unicode2local, name_fixer, \
-     reliable_unpack_names, unicoder, latin1, platform_encode
+     reliable_unpack_names, unicoder, latin1, platform_encode, deunicode
 from sabnzbd.utils.rarfile import RarFile, is_rarfile
 from sabnzbd.misc import format_time_string, find_on_path, make_script_path, int_conv, \
                          flag_file, real_path, globber
@@ -1341,8 +1341,9 @@ def PAR_Verify(parfile, parfile_nzf, nzo, setname, joinables, classic=False):
 
             elif line.startswith('File:') and line.find('data blocks from') > 0:
                 # Find out if a joinable file has been used for joining
+                uline = unicoder(line)
                 for jn in joinables:
-                    if line.find(os.path.split(jn)[1]) > 0:
+                    if uline.find(os.path.split(jn)[1]) > 0:
                         used_joinables.append(jn)
                         break
                 # Special case of joined RAR files, the "of" and "from" must both be RAR files
@@ -1454,6 +1455,10 @@ def fix_env():
 def build_command(command):
     """ Prepare list from running an external program
     """
+    for n in xrange(len(command)):
+        if isinstance(command[n], unicode):
+            command[n] = deunicode(command[n])
+
     if not sabnzbd.WIN32:
         if IONICE_COMMAND and cfg.ionice().strip():
             lst = cfg.ionice().split()

sabnzbd/newzbin.py

@@ -106,8 +106,6 @@ class MSGIDGrabber(Thread):
 
             filename, data, newzbin_cat, nzo_info = _grabnzb(msgid)
             if filename and data:
-                filename = name_fixer(filename)
-
                 pp = nzo.pp
                 script = nzo.script
                 cat = nzo.cat

sabnzbd/nzbqueue.py

@@ -35,7 +35,7 @@ from sabnzbd.decorators import NZBQUEUE_LOCK, synchronized, synchronized_CV
 from sabnzbd.constants import QUEUE_FILE_NAME, QUEUE_VERSION, FUTURE_Q_FOLDER, JOB_ADMIN, \
                               LOW_PRIORITY, NORMAL_PRIORITY, HIGH_PRIORITY, TOP_PRIORITY, \
                               REPAIR_PRIORITY, STOP_PRIORITY, VERIFIED_FILE, \
-                              PNFO_BYTES_FIELD, PNFO_BYTES_LEFT_FIELD, Status
+                              PNFO_BYTES_FIELD, PNFO_BYTES_LEFT_FIELD, Status, QUEUE_FILE_TMPL
 import sabnzbd.cfg as cfg
 from sabnzbd.articlecache import ArticleCache
 import sabnzbd.downloader
@@ -71,7 +71,15 @@ class NzbQueue(TryList):
         if repair < 2:
             # Read the queue from the saved files
             data = sabnzbd.load_admin(QUEUE_FILE_NAME)
-            if data:
+            if not data:
+                try:
+                    # Try previous queue file
+                    queue_vers, nzo_ids, dummy = sabnzbd.load_admin(QUEUE_FILE_TMPL % '9')
+                except:
+                    nzo_ids = []
+                if nzo_ids:
+                    logging.warning(T('Old queue detected, use Status->Repair to convert the queue'))
+            else:
                 try:
                     queue_vers, nzo_ids, dummy = data
                     if not queue_vers == QUEUE_VERSION:
@@ -115,7 +123,7 @@ class NzbQueue(TryList):
 
 
     def scan_jobs(self, all=False, action=True):
-        """ Scan "incomplete" for mssing folders,
+        """ Scan "incomplete" for missing folders,
             'all' is True: Include active folders
             'action' is True, do the recovery action
             returns list of orphaned folders

sabnzbd/nzbstuff.py

@@ -41,7 +41,7 @@ from sabnzbd.constants import sample_match, GIGI, ATTRIB_FILE, JOB_ADMIN, \
 from sabnzbd.misc import to_units, cat_to_opts, cat_convert, sanitize_foldername, \
                          get_unique_path, get_admin_path, remove_all, format_source_url, \
                          sanitize_filename, globber, sanitize_foldername, int_conv, \
-                         set_permissions
+                         set_permissions, format_time_string
 import sabnzbd.cfg as cfg
 from sabnzbd.trylist import TryList
 from sabnzbd.encoding import unicoder, platform_encode, latin1, name_fixer
@@ -361,8 +361,6 @@ class NzbParser(xml.sax.handler.ContentHandler):
                 logging.info('Skipping sample file %s', subject)
             else:
                 self.in_file = True
-                if isinstance(subject, unicode):
-                    subject = subject.encode('latin-1', 'replace')
                 self.fileSubject = subject
                 try:
                     self.file_date = int(attrs.get('date'))
@@ -757,6 +755,9 @@ class NzbObject(TryList):
         # Pickup backed-up attributes when re-using
         if reuse:
             cat, pp, script, priority, name, self.url = get_attrib_file(self.workpath, 6)
+            cat = unicoder(cat)
+            script = unicoder(script)
+            name = unicoder(name)
             self.set_final_name_pw(name)
 
         # Determine category and find pp/script values
@@ -1039,9 +1040,9 @@ class NzbObject(TryList):
             if dif > 0:
                 prefix += Ta('WAIT %s sec') % dif + ' / ' #: Queue indicator for waiting URL fetch
         if self.password:
-            return '%s%s / %s' % (name_fixer(prefix), self.final_name, self.password)
+            return '%s%s / %s' % (prefix, self.final_name, self.password)
         else:
-            return '%s%s' % (name_fixer(prefix), self.final_name)
+            return '%s%s' % (prefix, self.final_name)
 
     @property
     def final_name_pw_clean(self):
@@ -1051,7 +1052,7 @@ class NzbObject(TryList):
             return self.final_name
 
     def set_final_name_pw(self, name):
-        if isinstance(name, str):
+        if isinstance(name, str) or isinstance(name, unicode):
             name, self.password = scan_password(platform_encode(name))
             self.final_name = sanitize_foldername(name)
             self.save_attribs()
@@ -1574,36 +1575,6 @@ def split_filename(name):
         return name.strip(), ""
 
 
-def format_time_string(seconds, days=0):
-    """ Given seconds and days, return formatted day/hour/min/sec string
-    """
-    def unit(n, single):
-        if n == 1:
-            return n, Tx(single)
-        else:
-            return n, Tx(single + 's')
-    try:
-        seconds = int(seconds)
-    except ValueError:
-        seconds = 0
-
-    completestr = ''
-    if days:
-        completestr += '%s %s ' % unit(days, 'day')
-    if (seconds/3600) >= 1:
-        completestr += '%s %s ' % unit(seconds/3600, 'hour')
-        seconds -= (seconds/3600)*3600
-    if (seconds/60) >= 1:
-        completestr += '%s %s ' % unit(seconds/60, 'minute')
-        seconds -= (seconds/60)*60
-    if seconds > 0:
-        completestr += '%s %s ' % unit(seconds, 'second')
-    else:
-        completestr += '%s %s' % unit(0, 'second')
-
-    return completestr.strip()
-
-
 RE_PASSWORD1 = re.compile(r'([^/\\]+)[/\\](.+)')
 RE_PASSWORD2 = re.compile(r'(.+){{([^{}]+)}}$')
 RE_PASSWORD3 = re.compile(r'(.+)\s+password\s*=\s*(.+)$', re.I)

sabnzbd/postproc.py

@@ -94,8 +94,8 @@ class PostProcessor(Thread):
         try:
             version, history_queue = data
             if POSTPROC_QUEUE_VERSION != version:
-                logging.warning(Ta('Failed to load postprocessing queue: Wrong version (need:%s, found:%s)'), POSTPROC_QUEUE_VERSION, version)
-            if isinstance(history_queue, list):
+                logging.warning(T('Old queue detected, use Status->Repair to convert the queue'))
+            elif isinstance(history_queue, list):
                 self.history_queue = [nzo for nzo in history_queue if os.path.exists(nzo.downpath)]
         except:
             logging.info('Corrupt %s file, discarding', POSTPROC_QUEUE_FILE_NAME)

sabnzbd/rss.py

@@ -374,21 +374,17 @@ class RSSQueue(object):
                     link, category = _get_link(uri, entry)
                 except (AttributeError, IndexError):
                     link = None
-                    category = ''
+                    category = u''
                     logging.info(Ta('Incompatible feed') + ' ' + uri)
                     logging.info("Traceback: ", exc_info = True)
                     return T('Incompatible feed')
-                category = latin1(category)
-                # Make sure only latin-1 encodable characters occur
-                atitle = latin1(entry.title)
-                title = unicoder(atitle)
+                title = entry.title
             else:
                 link = entry
                 category = jobs[link].get('orgcat', '')
                 if category in ('', '*'):
                     category = None
-                atitle = latin1(jobs[link].get('title', ''))
-                title = unicoder(atitle)
+                title = jobs[link].get('title', '')
 
             if link:
                 # Make sure spaces are quoted in the URL
@@ -406,7 +402,7 @@ class RSSQueue(object):
                     jobstat = 'N'
                 if jobstat in 'NGB' or (jobstat == 'X' and readout):
                     # Match this title against all filters
-                    logging.debug('Trying title %s', atitle)
+                    logging.debug('Trying title %s', title)
                     result = False
                     myCat = defCat
                     myPP = defPP
@@ -465,7 +461,7 @@ class RSSQueue(object):
 
                     if cfg.no_dupes() and dup_title(title):
                         if cfg.no_dupes() == 1:
-                            logging.info("Ignoring duplicate job %s", atitle)
+                            logging.info("Ignoring duplicate job %s", title)
                             continue
                         else:
                             myPrio = DUP_PRIORITY

sabnzbd/sabtray.py

@@ -55,18 +55,38 @@ class SABTrayThread(SysTrayIconThread):
         self.counter = 0
         text = "SABnzbd"
 
+        self.set_texts()
         menu_options = (
-            (T('Show interface'), None, self.browse),
-            (T('Open complete folder'), None, self.opencomplete),
-            (T('Troubleshoot'), None, ((T('Restart'), None, self.restart),
-                                      (T('Restart without login'), None, self.nologin),
-                                      (T('Restart') + ' - 127.0.0.1:8080', None, self.defhost))),
-            (T('Pause') + '/' + T('Resume'), None, self.pauseresume),
-            (T('Shutdown'), None, self.shutdown),
+            (self.txt_show_int, None, self.browse),
+            (self.txt_open_comp, None, self.opencomplete),
+            (self.txt_trouble, None, ((self.txt_restart, None, self.restart),
+                                      (self.txt_restart_nl, None, self.nologin),
+                                      (self.txt_restart + ' - 127.0.0.1:8080', None, self.defhost))),
+            (self.txt_pause + '/' + self.txt_resume, None, self.pauseresume),
+            (self.txt_shutdown, None, self.shutdown),
         )
 
         SysTrayIconThread.__init__(self, self.sabicons['default'], text, menu_options, None, 0, "SabTrayIcon")
 
+    def set_texts(self):
+        def fix(txt):
+            if trans:
+                return Tx(txt)
+            else:
+                return txt
+
+        trans = str(get_codepage()) == str(sabnzbd.lang.CODEPAGE)
+        self.txt_show_int = fix(TT('Show interface'))
+        self.txt_open_comp = fix(TT('Open complete folder'))
+        self.txt_trouble = fix(TT('Troubleshoot'))
+        self.txt_pause = fix(TT('Pause'))
+        self.txt_shutdown = fix(TT('Shutdown'))
+        self.txt_resume = fix(TT('Resume'))
+        self.txt_restart = fix(TT('Restart'))
+        self.txt_restart_nl = fix(TT('Restart without login'))
+        self.txt_idle = fix(TT('Idle'))
+        self.txt_paused = fix(TT('Paused'))
+        self.txt_remaining = fix(TT('Remaining'))
 
     # called every few ms by SysTrayIconThread
     def doUpdates(self):
@@ -78,13 +98,13 @@ class SABTrayThread(SysTrayIconThread):
             speed = to_units(bpsnow, dec_limit=1)
 
             if self.sabpaused:
-                self.hover_text = T('Paused')
+                self.hover_text = self.txt_paused
                 self.icon = self.sabicons['pause']
             elif bytes_left > 0:
-                self.hover_text = "%sB/s %s: %sB (%s)" % (speed, T('Remaining'), mb_left, time_left)
+                self.hover_text = "%sB/s %s: %sB (%s)" % (speed, self.txt_remaining, mb_left, time_left)
                 self.icon = self.sabicons['green']
             else:
-                self.hover_text = T('Idle')
+                self.hover_text = self.txt_idle
                 self.icon = self.sabicons['default']
 
             self.refresh_icon()
@@ -112,7 +132,7 @@ class SABTrayThread(SysTrayIconThread):
 
     # menu handler
     def restart(self, icon):
-        self.hover_text = T('Restart')
+        self.hover_text = self.txt_restart
         sabnzbd.halt()
         cherrypy.engine.restart()
 
@@ -121,7 +141,7 @@ class SABTrayThread(SysTrayIconThread):
         sabnzbd.cfg.username.set('')
         sabnzbd.cfg.password.set('')
         sabnzbd.config.save_config()
-        self.hover_text = T('Restart')
+        self.hover_text = self.txt_restart
         sabnzbd.halt()
         cherrypy.engine.restart()
 
@@ -130,13 +150,13 @@ class SABTrayThread(SysTrayIconThread):
         sabnzbd.cfg.cherryhost.set('127.0.0.1')
         sabnzbd.cfg.enable_https.set(False)
         sabnzbd.config.save_config()
-        self.hover_text = T('Restart')
+        self.hover_text = self.txt_restart
         sabnzbd.halt()
         cherrypy.engine.restart()
 
     # menu handler - adapted from interface.py
     def shutdown(self, icon):
-        self.hover_text = T('Shutdown')
+        self.hover_text = self.txt_shutdown
         sabnzbd.halt()
         cherrypy.engine.exit()
         sabnzbd.SABSTOP = True
@@ -151,3 +171,9 @@ class SABTrayThread(SysTrayIconThread):
         scheduler.plan_resume(0)
         sabnzbd.unpause_all()
 
+
+def get_codepage():
+    import locale
+    lang, code = locale.getlocale()
+    logging.debug('SysTray uses codepage %s', code)
+    return code

sabnzbd/utils/json.py

@@ -68,7 +68,8 @@ class JsonWriter(object):
                 try:
                     obj.decode('utf-8')
                 except:
-                    obj = obj.decode('latin-1').encode('utf-8', 'replace')
+                    # Should never be needed
+                    obj = obj.decode('cp1252').encode('utf-8', 'replace')
             obj = obj.replace('\\', r'\\')
             if self._escaped_forward_slash:
                 obj = obj.replace('/', r'\/')

sabnzbd/utils/rarfile.py

@@ -7,8 +7,6 @@
 #   - Improve compatibility with Python's ZipFile support:
 #       - Always use Unix separators '/' in pathnames (ascii & unicode)
 #       - Foldernames must always end with a '/' (ascii & unicode)
-#       - Use CP850 as default codepage
-#       - Convert ASCII filenames to Python's default 'latin-1' encoding
 #
 # Optimized to fit in SABnzbd:
 #   - No extract hack (not needed for just rarred NZB files).
@@ -357,8 +355,7 @@ class RarFile:
             h.filename = name[:nul]
             u = _UnicodeFilename(h.filename, name[nul + 1 : ])
             h.unicode_filename = u.decode()
-            # Remap ASCII name from CP850 to Python's default Latin-1
-            h.filename = h.filename.decode(self.charset, 'replace').encode('latin-1', 'replace')
+            h.filename = h.filename.decode(self.charset, 'replace')
         else:
             h.filename = name
             h.unicode_filename = name.decode(self.charset, 'replace')

sabnzbd/wizard.py

@@ -20,6 +20,7 @@ sabnzbd.wizard - Wizard Webinterface
 """
 
 import os
+import logging
 import cherrypy
 from Cheetah.Template import Template
 
@@ -50,10 +51,18 @@ class Wizard(object):
         info = self.info.copy()
         info['num'] = ''
         info['number'] = 0
-        info['lang'] = cfg.language()
+        lng = None
+        if sabnzbd.WIN32:
+            import util.apireg
+            lng = util.apireg.get_install_lng()
+            logging.debug('Installer language code "%s"', lng)
+        info['lang'] = lng or cfg.language()
         info['languages'] = list_languages()
         info['T'] = Ttemplate
 
+        set_language(info['lang'])
+        sabnzbd.api.clear_trans_cache()
+
         if not os.path.exists(self.__web_dir):
             # If the wizard folder does not exist, simply load the normal page
             raise cherrypy.HTTPRedirect('')

tools/extract_pot.py

@@ -184,10 +184,8 @@ dst.write('\n')
 count = 0
 for line in src:
     count += 1
-    if 'Please, first check' in line:
-        pass
     m = RE_NSIS.search(line)
-    if m:
+    if m and 'MsgLangCode' not in line:
         dst.write('#: %s:%s\n' % (NSIS, count))
         text = m.group(1).replace('$\\"', '\\"').replace('$\\', '\\\\')
         dst.write('msgid %s\n' % text)

tools/make_mo.py

@@ -105,6 +105,8 @@ LanguageTable = {
     'se' : ('Northern Sami', 'Davvisámegiella'),
     'sm' : ('Samoan', 'Gagana fa\'a Samoa'),
     'gd' : ('Gaelic', 'Gàidhlig'),
+    'ru' : ('Russian', 'русский язык'),
+    'sr' : ('Serbian', 'српски'),
     'sn' : ('Shona', 'Chi Shona'),
     'sk' : ('Slovak', 'Slovencina'),
     'sl' : ('Slovene', 'Slovenšcina'),
@@ -127,6 +129,7 @@ LanguageTable = {
     'xh' : ('Xhosa', 'isi Xhosa'),
     'yo' : ('Yoruba', 'Yorùbá'),
     'zu' : ('Zulu', 'isi Zulu'),
+    'zh_CN' : ('SimpChinese', '简体中文'),
 }
 
 # Filter for retrieving readable language from PO file
@@ -194,95 +197,11 @@ def make_templates():
                 os.remove(mo_path)
 
 
-# Convert Romanian PX files to Latin1 PO files
-ro_table = {
-    u"\u015f" : u"s", # ș
-    u"\u015e" : u"S", # Ș
-    u"\u0163" : u"t", # ț
-    u"\u0162" : u"T", # Ț
-    u"\u0103" : u"ã", # ă
-    u"\u0102" : u"Ã", # Ă
-    u'\u021b' : u"t", # ț
-    u'\u0218' : u"S", # Ș
-    u'\u0219' : u"s"  # ș
-}
-
-# Convert Polish PX files to Latin1 PO files
-pl_table = {
-    u"\u0104" : u"A", # Ą
-    u"\u0106" : u"C", # Ć
-    u"\u0118" : u"E", # Ę
-    u"\u0141" : u"L", # Ł
-    u"\u013B" : u"L", # Ł
-    u"\u0143" : u"N", # Ń
-    #u"\u00D3" : u"O", # Ó
-    u"\u015A" : u"S", # Ś
-    u"\u0179" : u"Z", # Ź
-    u"\u017B" : u"Z", # Ż
-    u"\u0105" : u"a", # ą
-    u"\u0107" : u"c", # ć
-    u"\u0119" : u"e", # ę
-    u"\u0142" : u"l", # ł
-    u"\u0144" : u"n", # ń
-    #u"\u00F3" : u"o", # ó
-    u"\u015B" : u"s", # ś
-    u"\u017A" : u"z", # ź
-    u"\u017C" : u"z"  # ż
-}
-
-def fix_ro():
-    """ Convert ro.px files to ro.po files with only Latin1
-    """
-    for section in ('main', 'email', 'nsis'):
-        f = open('po/%s/ro.px' % section, 'rb')
-        data = f.read().decode('utf-8')
-        f.close()
-
-        for ch in ro_table:
-            data = data.replace(ch, ro_table[ch])
-
-        f = open('po/%s/ro.po' % section, 'wb')
-        f.write(data.encode('utf-8'))
-        f.close()
-        try:
-            lnum = 0
-            for line in data.split('\n'):
-                lnum += 1
-                line.encode('latin-1')
-        except:
-            print line.encode('utf-8')
-            print 'WARNING: line %d in file po/%s/ro.po is not Latin-1' % (lnum, section)
-            exit(1)
-
-def fix_pl():
-    """ Convert pl.px files to pl.po files with only Latin1
-    """
-    for section in ('main', 'email', 'nsis'):
-        f = open('po/%s/pl.px' % section, 'rb')
-        data = f.read().decode('utf-8')
-        f.close()
-
-        for ch in pl_table:
-            data = data.replace(ch, pl_table[ch])
-
-        f = open('po/%s/pl.po' % section, 'wb')
-        f.write(data.encode('utf-8'))
-        f.close()
-        try:
-            lnum = 0
-            for line in data.split('\n'):
-                lnum += 1
-                line.encode('latin-1')
-        except:
-            print line.encode('utf-8')
-            print 'WARNING: line %d in file po/%s/pl.po is not Latin-1' % (lnum, section)
-            exit(1)
-
-
 def patch_nsis():
     """ Patch translation into the NSIS script
     """
     RE_NSIS = re.compile(r'^(\s*LangString\s+\w+\s+\$\{LANG_)(\w+)\}\s+(".*)', re.I)
+    RE_NSIS = re.compile(r'^(\s*LangString\s+)(\w+)(\s+\$\{LANG_)(\w+)\}\s+(".*)', re.I)
     languages = [os.path.split(path)[1] for path in glob.glob(os.path.join(MO_DIR, '*'))]
 
     src = open(NSIS, 'r')
@@ -291,8 +210,10 @@ def patch_nsis():
         m = RE_NSIS.search(line)
         if m:
             leader = m.group(1)
-            langname = m.group(2).upper()
-            text = m.group(3).strip('"\n')
+            item = m.group(2)
+            rest = m.group(3)
+            langname = m.group(4).upper()
+            text = m.group(5).strip('"\n')
             if langname == 'ENGLISH':
                 # Write back old content
                 new.append(line)
@@ -302,13 +223,16 @@ def patch_nsis():
                     lng = LanguageTable.get(lcode)
                     if lng and lcode != 'en':
                         lng = lng[0].decode('utf-8').encode('latin-1').upper()
+                        if item == 'MsgLangCode':
+                            trans = lcode
+                        else:
                             trans = gettext.translation(DOMAIN_N, MO_DIR, [lcode], fallback=False, codeset='latin-1')
                             # The unicode flag will make _() return Unicode
                             trans.install(unicode=True, names=['lgettext'])
-                        trans = lgettext(text)
+                            trans = _(text).encode('utf-8')
                             trans = trans.replace('\r', '').replace('\n', '\\r\\n')
                             trans = trans.replace('\\', '$\\').replace('"', '$\\"')
-                        line = '%s%s} "%s"\n' % (leader, lng, trans)
+                        line = '%s%s%s%s} "%s"\n' % (leader, item, rest, lng, trans)
                         new.append(line)
                     elif lng is None:
                         print 'Warning: unsupported language %s (%s), add to table in this script' % (langname, lcode)
@@ -331,10 +255,6 @@ if os.path.exists(tl):
     else:
         TOOL = '"%s"' % tl
 
-# Fix up Romanian and Polish texts
-fix_ro()
-fix_pl()
-
 if len(sys.argv) > 1 and sys.argv[1] == 'all':
     print 'NSIS MO file'
     process_po_folder(DOMAIN_N, PON_DIR)

util/apireg.py

@@ -46,7 +46,6 @@ def get_connection_info(user=True):
         key = _winreg.OpenKey(hive, keypath + r'\api')
         for i in range(0, _winreg.QueryInfoKey(key)[1]):
             name, value, val_type = _winreg.EnumValue(key, i)
-            value = value.encode('latin-1', 'replace')
             if name == 'url':
                 url = value
 
@@ -101,6 +100,26 @@ def del_connection_info(user=True):
         _winreg.CloseKey(hive)
 
 
-#print get_connection_info()
-#del_connection_info()
-#set_connection_info('localhost', '8080', 'blabla', user=False)
+def get_install_lng():
+    """ Return language-code used by the installer """
+    lng = 0
+    try:
+        hive = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE)
+        key = _winreg.OpenKey(hive, r"Software\SABnzbd")
+        for i in range(0, _winreg.QueryInfoKey(key)[1]):
+            name, value, val_type = _winreg.EnumValue(key, i)
+            if name == 'Installer Language':
+                lng = value
+        _winreg.CloseKey(key)
+    except WindowsError:
+        pass
+    finally:
+        _winreg.CloseKey(hive)
+    return lng
+
+
+if __name__ == '__main__':
+    print 'URL = %s' %get_connection_info()
+    print 'Language = %s' % get_install_lng()
+    #del_connection_info()
+    #set_connection_info('localhost', '8080', 'blabla', user=False)