sabnzbd/sabnzbd/misc.py

#!/usr/bin/python -OO
# Copyright 2008-2017 The SABnzbd-Team <team@sabnzbd.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

"""
sabnzbd.misc - misc classes
"""

import os
import sys
import logging
import urllib.request, urllib.parse, urllib.error
import re
import shutil
import threading
import subprocess
import socket
import time
import datetime
import fnmatch
import stat
import inspect
import urllib2
from urlparse import urlparse

import sabnzbd
from sabnzbd.decorators import synchronized
from sabnzbd.constants import DEFAULT_PRIORITY, FUTURE_Q_FOLDER, JOB_ADMIN, \
     GIGI, MEBI, DEF_CACHE_LIMIT
import sabnzbd.config as config
import sabnzbd.cfg as cfg
from sabnzbd.encoding import ubtou, unicoder, special_fixer, gUTF

TAB_UNITS = ('', 'K', 'M', 'G', 'T', 'P')
RE_UNITS = re.compile(r'(\d+\.*\d*)\s*([KMGTP]{0,1})', re.I)
RE_VERSION = re.compile(r'(\d+)\.(\d+)\.(\d+)([a-zA-Z]*)(\d*)')
RE_IP4 = re.compile(r'inet\s+(addr:\s*){0,1}(\d+\.\d+\.\d+\.\d+)')
RE_IP6 = re.compile(r'inet6\s+(addr:\s*){0,1}([0-9a-f:]+)', re.I)

# Check if strings are defined for AM and PM
HAVE_AMPM = bool(time.strftime('%p', time.localtime()))


def time_format(fmt):
    """ Return time-format string adjusted for 12/24 hour clock setting """
    if cfg.ampm() and HAVE_AMPM:
        return fmt.replace('%H:%M:%S', '%I:%M:%S %p').replace('%H:%M', '%I:%M %p')
    else:
        return fmt


def calc_age(date, trans=False):
    """ Calculate the age difference between now and date.
        Value is returned as either days, hours, or minutes.
        When 'trans' is True, time symbols will be translated.
    """
    if trans:
        d = T('d')  # : Single letter abbreviation of day
        h = T('h')  # : Single letter abbreviation of hour
        m = T('m')  # : Single letter abbreviation of minute
    else:
        d = 'd'
        h = 'h'
        m = 'm'
    try:
        now = datetime.datetime.now()
        # age = str(now - date).split(".")[0] #old calc_age

        # time difference
        dage = now - date
        seconds = dage.seconds
        # only one value should be returned
        # if it is less than 1 day then it returns in hours, unless it is less than one hour where it returns in minutes
        if dage.days:
            age = '%d%s' % (dage.days, d)
        elif int(seconds / 3600):
            age = '%d%s' % (seconds / 3600, h)
        else:
            age = '%d%s' % (seconds / 60, m)
    except:
        age = "-"

    return age


def monthrange(start, finish):
    """ Calculate months between 2 dates, used in the Config template """
    months = (finish.year - start.year) * 12 + finish.month + 1
    for i in range(start.month, months):
        year  = (i - 1) / 12 + start.year
        month = (i - 1) % 12 + 1
        yield datetime.date(int(year), int(month), 1)


def safe_lower(txt):
    """ Return lowercased string. Return '' for None """
    if txt:
        return txt.lower()
    else:
        return ''



def cat_to_opts(cat, pp=None, script=None, priority=None):
    """ Derive options from category, if options not already defined.
        Specified options have priority over category-options.
        If no valid category is given, special category '*' will supply default values
    """
    def_cat = config.get_categories('*')
    cat = safe_lower(cat)
    if cat in ('', 'none', 'default'):
        cat = '*'
    try:
        my_cat = config.get_categories()[cat]
    except KeyError:
        my_cat = def_cat

    if pp is None:
        pp = my_cat.pp()
        if pp == '':
            pp = def_cat.pp()

    if not script:
        script = my_cat.script()
        if safe_lower(script) in ('', 'default'):
            script = def_cat.script()

    if priority is None or priority == '' or priority == DEFAULT_PRIORITY:
        priority = my_cat.priority()
        if priority == DEFAULT_PRIORITY:
            priority = def_cat.priority()

    # logging.debug('Cat->Attrib cat=%s pp=%s script=%s prio=%s', cat, pp, script, priority)
    return cat, pp, script, priority


_wildcard_to_regex = {
    '\\': r'\\',
    '^': r'\^',
    '$': r'\$',
    '.': r'\.',
    '[': r'\[',
    ']': r'\]',
    '(': r'\(',
    ')': r'\)',
    '+': r'\+',
    '?': r'.',
    '|': r'\|',
    '{': r'\{',
    '}': r'\}',
    '*': r'.*'
}


def wildcard_to_re(text):
    """ Convert plain wildcard string (with '*' and '?') to regex. """
    return ''.join([_wildcard_to_regex.get(ch, ch) for ch in text])


def cat_convert(cat):
    """ Convert indexer's category/group-name to user categories.
        If no match found, but indexer-cat equals user-cat, then return user-cat
        If no match found, but the indexer-cat starts with the user-cat, return user-cat
        If no match found, return None
    """
    if cat and cat.lower() != 'none':
        cats = config.get_ordered_categories()
        raw_cats = config.get_categories()
        for ucat in cats:
            try:
                # Ordered cat-list has tags only as string
                indexer = raw_cats[ucat['name']].newzbin()
                if not isinstance(indexer, list):
                    indexer = [indexer]
            except:
                indexer = []
            for name in indexer:
                if re.search('^%s$' % wildcard_to_re(name), cat, re.I):
                    if '.' in name:
                        logging.debug('Convert group "%s" to user-cat "%s"', cat, ucat['name'])
                    else:
                        logging.debug('Convert index site category "%s" to user-cat "%s"', cat, ucat['name'])
                    return ucat['name']

        # Try to find full match between user category and indexer category
        for ucat in cats:
            if cat.lower() == ucat['name'].lower():
                logging.debug('Convert index site category "%s" to user-cat "%s"', cat, ucat['name'])
                return ucat['name']

        # Try to find partial match between user category and indexer category
        for ucat in cats:
            if cat.lower().startswith(ucat['name'].lower()):
                logging.debug('Convert index site category "%s" to user-cat "%s"', cat, ucat['name'])
                return ucat['name']

    return None


def windows_variant():
    """ Determine Windows variant
        Return vista_plus, x64
    """
    from win32api import GetVersionEx
    from win32con import VER_PLATFORM_WIN32_NT
    import winreg

    vista_plus = x64 = False
    maj, _minor, _buildno, plat, _csd = GetVersionEx()

    if plat == VER_PLATFORM_WIN32_NT:
        vista_plus = maj > 5
        if vista_plus:
            # Must be done the hard way, because the Python runtime lies to us.
            # This does *not* work:
            #     return os.environ['PROCESSOR_ARCHITECTURE'] == 'AMD64'
            # because the Python runtime returns 'X86' even on an x64 system!
            key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,
                    r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment")
            for n in range(winreg.QueryInfoKey(key)[1]):
                name, value, _val_type = winreg.EnumValue(key, n)
                if name == 'PROCESSOR_ARCHITECTURE':
                    x64 = value.upper() == 'AMD64'
                    break
            winreg.CloseKey(key)

    return vista_plus, x64


_SERVICE_KEY = 'SYSTEM\\CurrentControlSet\\services\\'
_SERVICE_PARM = 'CommandLine'


def get_serv_parms(service):
    """ Get the service command line parameters from Registry """
    import winreg

    value = []
    try:
        key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, _SERVICE_KEY + service)
        for n in range(winreg.QueryInfoKey(key)[1]):
            name, value, _val_type = winreg.EnumValue(key, n)
            if name == _SERVICE_PARM:
                break
        winreg.CloseKey(key)
    except WindowsError:
        pass
    for n in range(len(value)):
        value[n] = value[n]
    return value


def set_serv_parms(service, args):
    """ Set the service command line parameters in Registry """
    import winreg

    uargs = []
    for arg in args:
        uargs.append(unicoder(arg))

    try:
        key = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, _SERVICE_KEY + service)
        winreg.SetValueEx(key, _SERVICE_PARM, None, winreg.REG_MULTI_SZ, uargs)
        winreg.CloseKey(key)
    except WindowsError:
        return False
    return True


def convert_version(text):
    """ Convert version string to numerical value and a testversion indicator """
    version = 0
    test = True
    m = RE_VERSION.search(text)
    if m:
        version = int(m.group(1)) * 1000000 + int(m.group(2)) * 10000 + int(m.group(3)) * 100
        try:
            if m.group(4).lower() == 'rc':
                version = version + 80
            elif m.group(4).lower() == 'beta':
                version = version + 40
            version = version + int(m.group(5))
        except:
            version = version + 99
            test = False
    return version, test


def check_latest_version():
    """ Do an online check for the latest version

        Perform an online version check
        Syntax of online version file:
            <current-final-release>
            <url-of-current-final-release>
            <latest-alpha/beta-or-rc>
            <url-of-latest-alpha/beta/rc-release>
        The latter two lines are only present when an alpha/beta/rc is available.
        Formula for the version numbers (line 1 and 3).
            <major>.<minor>.<bugfix>[rc|beta|alpha]<cand>

        The <cand> value for a final version is assumned to be 99.
        The <cand> value for the beta/rc version is 1..98, with RC getting
        a boost of 80 and Beta of 40.
        This is done to signal alpha/beta/rc users of availability of the final
        version (which is implicitly 99).
        People will only be informed to upgrade to a higher alpha/beta/rc version, if
        they are already using an alpha/beta/rc.
        RC's are valued higher than Beta's, which are valued higher than Alpha's.
    """

    if not cfg.version_check():
        return

    current, testver = convert_version(sabnzbd.__version__)
    if not current:
        logging.debug("Unsupported release number (%s), will not check", sabnzbd.__version__)
        return

    # Using catch-all except's is poor coding practice.
    # However, the last thing you want is the app crashing due
    # to bad file content.

    try:
        fn = urllib.request.urlretrieve('https://raw.githubusercontent.com/sabnzbd/sabnzbd.github.io/master/latest.txt')[0]
        f = open(fn, 'r')
        data = f.read()
        f.close()
        os.remove(fn)
    except:
        logging.info('Cannot retrieve version information from GitHub.com')
        logging.debug('Traceback: ', exc_info=True)
        return

    try:
        latest_label = data.split()[0]
    except:
        latest_label = ''
    try:
        url = data.split()[1]
    except:
        url = ''
    try:
        latest_testlabel = data.split()[2]
    except:
        latest_testlabel = ''
    try:
        url_beta = data.split()[3]
    except:
        url_beta = url

    latest = convert_version(latest_label)[0]
    latest_test = convert_version(latest_testlabel)[0]

    logging.debug('Checked for a new release, cur= %s, latest= %s (on %s), latest_test= %s (on %s)',
                  current, latest, url, latest_test, url_beta)

    if latest_test and cfg.version_check() > 1:
        # User always wants to see the latest test release
        latest = latest_test
        latest_label = latest_testlabel
        url = url_beta

    if testver and current < latest:
        # This is a test version, but user has't seen the
        # "Final" of this one yet, so show the Final
        sabnzbd.NEW_VERSION = (latest_label, url)
    elif current < latest:
        # This one is behind, show latest final
        sabnzbd.NEW_VERSION = (latest_label, url)
    elif testver and current < latest_test:
        # This is a test version beyond the latest Final, so show latest Alpha/Beta/RC
        sabnzbd.NEW_VERSION = (latest_testlabel, url_beta)


def from_units(val):
    """ Convert K/M/G/T/P notation to float """
    val = str(val).strip().upper()
    if val == "-1":
        return val
    m = RE_UNITS.search(val)
    if m:
        if m.group(2):
            val = float(m.group(1))
            unit = m.group(2)
            n = 0
            while unit != TAB_UNITS[n]:
                val = val * 1024.0
                n = n + 1
        else:
            val = m.group(1)
        try:
            return float(val)
        except:
            return 0.0
    else:
        return 0.0


def to_units(val, spaces=0, dec_limit=2, postfix=''):
    """ Convert number to K/M/G/T/P notation
        Add "spaces" if not ending in letter
        dig_limit==1 show single decimal for M and higher
        dig_limit==2 show single decimal for G and higher
    """
    decimals = 0
    if val < 0:
        sign = '-'
    else:
        sign = ''
    val = str(abs(val)).strip()

    n = 0
    try:
        val = float(val)
    except:
        return ''
    while (val > 1023.0) and (n < 5):
        val = val / 1024.0
        n = n + 1
    unit = TAB_UNITS[n]
    if not unit:
        unit = ' ' * spaces
    if n > dec_limit:
        decimals = 1
    else:
        decimals = 0

    fmt = '%%s%%.%sf %%s%%s' % decimals
    return fmt % (sign, val, unit, postfix)


def caller_name(skip=2):
    """Get a name of a caller in the format module.method
       Originally used: https://gist.github.com/techtonik/2151727
       Adapted for speed by using sys calls directly
    """
    # Only do the tracing on Debug (function is always called)
    if cfg.log_level() != 2:
        return 'N/A'

    parentframe = sys._getframe(skip)
    function_name = parentframe.f_code.co_name

    # Modulename not available in the binaries, we can use the filename instead
    if getattr(sys, 'frozen', None):
        module_name = inspect.getfile(parentframe)
    else:
        module_name = inspect.getmodule(parentframe).__name__

    # For decorated functions we have to go deeper
    if function_name in ('call_func', 'wrap') and skip == 2:
        return caller_name(4)

    return ".".join([module_name, function_name])


def exit_sab(value):
    """ Leave the program after flushing stderr/stdout """
    sys.stderr.flush()
    sys.stdout.flush()
    if getattr(sys, 'frozen', None) == 'macosx_app':
        sabnzbd.SABSTOP = True
        from PyObjCTools import AppHelper
        AppHelper.stopEventLoop()
    sys.exit(value)


def split_host(srv):
    """ Split host:port notation, allowing for IPV6 """
    # Cannot use split, because IPV6 of "a:b:c:port" notation
    # Split on the last ':'
    mark = srv.rfind(':')
    if mark < 0:
        host = srv
    else:
        host = srv[0: mark]
        port = srv[mark + 1:]
    try:
        port = int(port)
    except:
        port = None
    return (host, port)


def get_cache_limit():
    """ Depending on OS, calculate cache limit """
    # OSX/Windows use Default value
    if sabnzbd.WIN32 or sabnzbd.DARWIN:
        return DEF_CACHE_LIMIT

    # Calculate, if possible
    try:
        # Use 1/4th of available memory
        mem_bytes = (os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES'))/4
        # Not more than the maximum we think is reasonable
        if mem_bytes > from_units(DEF_CACHE_LIMIT):
            return DEF_CACHE_LIMIT
        elif mem_bytes > from_units('32M'):
            # We make sure it's at least a valid value
            return to_units(mem_bytes)
    except:
        pass
    # If failed, leave empty so user needs to decide
    return ''


def on_cleanup_list(filename, skip_nzb=False):
    """ Return True if a filename matches the clean-up list """
    lst = cfg.cleanup_list()
    if lst:
        name, ext = os.path.splitext(filename)
        ext = ext.strip().lower()
        name = name.strip()
        for k in lst:
            item = k.strip().strip('.').lower()
            item = '.' + item
            if (item == ext or (ext == '' and item == name)) and not (skip_nzb and item == '.nzb'):
                return True
    return False


def memory_usage():
    try:
        # Probably only works on Linux because it uses /proc/<pid>/statm
        t = open('/proc/%d/statm' % os.getpid())
        v = t.read().split()
        t.close()
        virt = int(_PAGE_SIZE * int(v[0]) / MEBI)
        res = int(_PAGE_SIZE * int(v[1]) / MEBI)
        return "V=%sM R=%sM" % (virt, res)
    except IOError:
        pass
    except:
        logging.debug('Error retrieving memory usage')
        logging.info("Traceback: ", exc_info=True)
    else:
        return ''
try:
    _PAGE_SIZE = os.sysconf("SC_PAGE_SIZE")
except:
    _PAGE_SIZE = 0
_HAVE_STATM = _PAGE_SIZE and memory_usage()


def loadavg():
    """ Return 1, 5 and 15 minute load average of host or "" if not supported """
    p = ''
    if not sabnzbd.WIN32 and not sabnzbd.DARWIN:
        opt = cfg.show_sysload()
        if opt:
            try:
                p = '%.2f | %.2f | %.2f' % os.getloadavg()
            except:
                pass
            if opt > 1 and _HAVE_STATM:
                p = '%s | %s' % (p, memory_usage())
    return p


def format_time_string(seconds, days=0):
    """ Return a formatted and translated time string """

    def unit(single, n):
        if n == 1:
            return sabnzbd.api.Ttemplate(single)
        else:
            return sabnzbd.api.Ttemplate(single + 's')

    seconds = int_conv(seconds)
    completestr = []
    if days:
        completestr.append('%s %s' % (days, unit('day', days)))
    if (seconds / 3600) >= 1:
        completestr.append('%s %s' % (seconds / 3600, unit('hour', (seconds / 3600))))
        seconds -= (seconds / 3600) * 3600
    if (seconds / 60) >= 1:
        completestr.append('%s %s' % (seconds / 60, unit('minute', (seconds / 60))))
        seconds -= (seconds / 60) * 60
    if seconds > 0:
        completestr.append('%s %s' % (seconds, unit('second', seconds)))
    elif not completestr:
        completestr.append('0 %s' % unit('second', 0))

    return ' '.join(completestr)


def int_conv(value):
    """ Safe conversion to int (can handle None) """
    try:
        value = int(value)
    except:
        value = 0
    return value


def create_https_certificates(ssl_cert, ssl_key):
    """ Create self-signed HTTPS certificates and store in paths 'ssl_cert' and 'ssl_key' """
    if not sabnzbd.HAVE_CRYPTOGRAPHY:
        logging.error(T('%s missing'), 'Python Cryptography')
        return False

    # Save the key and certificate to disk
    try:
        from sabnzbd.utils.certgen import generate_key, generate_local_cert
        private_key = generate_key(key_size=2048, output_file=ssl_key)
        generate_local_cert(private_key, days_valid=3560, output_file=ssl_cert, LN='SABnzbd', ON='SABnzbd', CN='localhost')
        logging.info('Self-signed certificates generated successfully')
    except:
        logging.error(T('Error creating SSL key and certificate'))
        logging.info("Traceback: ", exc_info=True)
        return False

    return True


def get_all_passwords(nzo):
    """ Get all passwords, from the NZB, meta and password file """
    if nzo.password:
        logging.info('Found a password that was set by the user: %s', nzo.password)
        passwords = [nzo.password.strip()]
    else:
        passwords = []

    meta_passwords = nzo.meta.get('password', [])
    pw = nzo.nzo_info.get('password')
    if pw:
        meta_passwords.append(pw)

    if meta_passwords:
        if nzo.password == meta_passwords[0]:
            # this nzo.password came from meta, so don't use it twice
            passwords.extend(meta_passwords[1:])
        else:
            passwords.extend(meta_passwords)
        logging.info('Read %s passwords from meta data in NZB: %s', len(meta_passwords), meta_passwords)

    pw_file = cfg.password_file.get_path()
    if pw_file:
        try:
            with open(pw_file, 'r') as pwf:
                lines = pwf.read().split('\n')
            # Remove empty lines and space-only passwords and remove surrounding spaces
            pws = [pw.strip('\r\n ') for pw in lines if pw.strip('\r\n ')]
            logging.debug('Read these passwords from file: %s', pws)
            passwords.extend(pws)
            logging.info('Read %s passwords from file %s', len(pws), pw_file)

            # Check size
            if len(pws) > 30:
                logging.warning(T('Your password file contains more than 30 passwords, testing all these passwords takes a lot of time. Try to only list useful passwords.'))
        except:
            logging.warning('Failed to read the passwords file %s', pw_file)

    if nzo.password:
        # If an explicit password was set, add a retry without password, just in case.
        passwords.append('')
    elif not passwords or nzo.encrypted < 1:
        # If we're not sure about encryption, start with empty password
        # and make sure we have at least the empty password
        passwords.insert(0, '')
    return passwords


def find_on_path(targets):
    """ Search the PATH for a program and return full path """
    if sabnzbd.WIN32:
        paths = os.getenv('PATH').split(';')
    else:
        paths = os.getenv('PATH').split(':')

    if isinstance(targets, str):
        targets = (targets, )

    for path in paths:
        for target in targets:
            target_path = os.path.abspath(os.path.join(path, target))
            if os.path.isfile(target_path) and os.access(target_path, os.X_OK):
                return target_path
    return None


def ip_extract():
    """ Return list of IP addresses of this system """
    ips = []
    program = find_on_path('ip')
    if program:
        program = [program, 'a']
    else:
        program = find_on_path('ifconfig')
        if program:
            program = [program]

    if sabnzbd.WIN32 or not program:
        try:
            info = socket.getaddrinfo(socket.gethostname(), None)
        except:
            # Hostname does not resolve, use localhost
            info = socket.getaddrinfo('localhost', None)
        for item in info:
            ips.append(item[4][0])
    else:
        p = subprocess.Popen(program, shell=False, stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                             startupinfo=None, creationflags=0)
        output = p.stdout.read()
        p.wait()
        for line in output.split('\n'):
            m = RE_IP4.search(line)
            if not (m and m.group(2)):
                m = RE_IP6.search(line)
            if m and m.group(2):
                ips.append(m.group(2))
    return ips


def get_base_url(url):
    """ Return only the true root domain for the favicon, so api.oznzb.com -> oznzb.com
        But also api.althub.co.za -> althub.co.za
    """
    url_host = urlparse(url).hostname
    if url_host:
        url_split = url_host.split(".")
        # Exception for localhost and IPv6 addresses
        if len(url_split) < 3:
            return url_host
        return ".".join(len(url_split[-2]) < 4 and url_split[-3:] or url_split[-2:])
    else:
        return ''


def match_str(text, matches):
    """ Return first matching element of list 'matches' in 'text', otherwise None """
    for match in matches:
        if match in text:
            return match
    return None


def get_urlbase(url):
    """ Return the base URL (like http://server.domain.com/) """
    parsed_uri = urlparse(url)
    return '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)


def nntp_to_msg(text):
    """ Format raw NNTP bytes data for display """
    if isinstance(text, list):
        text = text[0]
    lines = text.split(b'\r\n')
    return ubtou(lines[0])