From 2341ea44ec467012ef81a41c37b2113cc824c729 Mon Sep 17 00:00:00 2001 From: Safihre Date: Fri, 12 Oct 2018 10:45:28 +0200 Subject: [PATCH] Py3: Fixes after merge from develop And set the shebang to Python 3 specifc --- SABHelper.py | 2 +- SABnzbd.py | 40 +++++---------------- sabnzbd/__init__.py | 51 +++++++++++++-------------- sabnzbd/api.py | 15 ++++---- sabnzbd/articlecache.py | 2 +- sabnzbd/assembler.py | 10 +++--- sabnzbd/bpsmeter.py | 2 +- sabnzbd/cfg.py | 2 +- sabnzbd/config.py | 4 +-- sabnzbd/constants.py | 2 +- sabnzbd/database.py | 13 ++----- sabnzbd/decoder.py | 6 ++-- sabnzbd/decorators.py | 2 +- sabnzbd/directunpacker.py | 9 +++-- sabnzbd/dirscanner.py | 4 +-- sabnzbd/downloader.py | 2 +- sabnzbd/emailer.py | 4 +-- sabnzbd/encoding.py | 15 +++----- sabnzbd/filesystem.py | 30 ++++++++++------ sabnzbd/getipaddress.py | 7 ++-- sabnzbd/interface.py | 25 ++++++------- sabnzbd/lang.py | 2 +- sabnzbd/misc.py | 80 ++++++++++++++++++++++++++---------------- sabnzbd/newsunpack.py | 23 ++++-------- sabnzbd/newswrapper.py | 2 +- sabnzbd/notifier.py | 4 +-- sabnzbd/nzbparser.py | 7 ++-- sabnzbd/nzbqueue.py | 6 ++-- sabnzbd/nzbstuff.py | 11 ++---- sabnzbd/osxmenu.py | 5 +-- sabnzbd/panic.py | 6 +--- sabnzbd/par2file.py | 4 +-- sabnzbd/postproc.py | 5 ++- sabnzbd/powersup.py | 2 +- sabnzbd/rating.py | 4 +-- sabnzbd/rss.py | 12 +++---- sabnzbd/sabtray.py | 2 +- sabnzbd/sabtraylinux.py | 2 +- sabnzbd/scheduler.py | 2 +- sabnzbd/skintext.py | 2 +- sabnzbd/sorting.py | 2 +- sabnzbd/urlgrabber.py | 2 +- sabnzbd/utils/certgen.py | 6 ++-- sabnzbd/utils/happyeyeballs.py | 73 +++++++++++++++++++------------------- sabnzbd/utils/kronos.py | 2 +- sabnzbd/utils/pystone.py | 12 +++---- sabnzbd/utils/servertests.py | 2 +- sabnzbd/utils/upload.py | 3 +- sabnzbd/zconfig.py | 2 +- scripts/Deobfuscate.py | 8 ++--- scripts/Sample-PostProc.py | 18 +++++----- tests/test_functional.py | 4 +-- tests/testhelper.py | 4 +-- tools/extract_pot.py | 4 +-- tools/make_mo.py | 2 +- util/apireg.py | 2 +- util/mailslot.py | 2 +- 57 files changed, 271 insertions(+), 305 deletions(-) diff --git a/SABHelper.py b/SABHelper.py index 0445ee5..b5b6d5f 100644 --- a/SABHelper.py +++ b/SABHelper.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/SABnzbd.py b/SABnzbd.py index 4f4e3b3..5794efa 100755 --- a/SABnzbd.py +++ b/SABnzbd.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python33 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -16,7 +16,6 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import sys -import imp if sys.version_info[:2] <= (3, 0): print("Sorry, requires Python 3") sys.exit(1) @@ -47,22 +46,6 @@ except: import cherrypy import portend - -SQLITE_DLL = True -try: - from sqlite3 import version as sqlite3_version -except: - try: - from pysqlite2.dbapi2 import version as sqlite3_version - except: - if os.name != 'nt': - print("Sorry, requires Python module sqlite3") - print("Try: apt-get install python-pysqlite2") - sys.exit(1) - else: - SQLITE_DLL = False - - import sabnzbd import sabnzbd.lang import sabnzbd.interface @@ -70,10 +53,9 @@ from sabnzbd.constants import * import sabnzbd.newsunpack from sabnzbd.misc import check_latest_version, exit_sab, \ split_host, create_https_certificates, windows_variant, ip_extract, \ - set_serv_parms, get_serv_parms + set_serv_parms, get_serv_parms, get_from_url from sabnzbd.filesystem import get_ext, real_path, long_path, globber_full -from sabnzbd.panic import panic_tmpl, panic_port, panic_host, \ - panic_sqlite, panic, launch_a_browser +from sabnzbd.panic import panic_tmpl, panic_port, panic_host, panic, launch_a_browser import sabnzbd.scheduler as scheduler import sabnzbd.config as config import sabnzbd.cfg @@ -227,7 +209,7 @@ def daemonize(): print("fork() failed") sys.exit(1) - dev_null = file('/dev/null', 'r') + dev_null = open('/dev/null', 'r') os.dup2(dev_null.fileno(), sys.stdin.fileno()) @@ -638,7 +620,7 @@ def is_sabnzbd_running(url): prev = sabnzbd.set_https_verification(False) ver = get_from_url(url) sabnzbd.set_https_verification(prev) - return ver and (re.search(r'\d+\.\d+\.', ver) or ver.strip() == sabnzbd.__version__) + return ver and (re.search(b'\d+\.\d+\.', ver) or ver.strip() == sabnzbd.__version__) except: return False @@ -666,7 +648,7 @@ def check_for_sabnzbd(url, upload_nzbs, allow_browser=True): # Upload any specified nzb files to the running instance if upload_nzbs: from sabnzbd.utils.upload import upload_file - prev = sabnzbd.set_https_verification(0) + prev = sabnzbd.set_https_verification(False) for f in upload_nzbs: upload_file(url, f) sabnzbd.set_https_verification(prev) @@ -930,10 +912,6 @@ def main(): vista_plus, win64 = windows_variant() sabnzbd.WIN64 = win64 - if not SQLITE_DLL: - panic_sqlite(sabnzbd.MY_FULLNAME) - exit_sab(2) - if inifile: # INI file given, simplest case inifile = evaluate_inipath(inifile) @@ -1103,7 +1081,7 @@ def main(): x = sys.stderr.fileno x = sys.stdout.fileno ol_path = os.path.join(logdir, DEF_LOG_ERRFILE) - out_log = file(ol_path, 'a+', 0) + out_log = open(ol_path, 'a+', 0) sys.stderr.flush() sys.stdout.flush() os.dup2(out_log.fileno(), sys.stderr.fileno()) @@ -1125,7 +1103,7 @@ def main(): logging.info('Console logging only') if noConsoleLoggingOSX: logging.info('Console logging for OSX App disabled') - so = file('/dev/null', 'a+') + so = open('/dev/null', 'a+') os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(so.fileno(), sys.stderr.fileno()) except AttributeError: @@ -1340,7 +1318,7 @@ def main(): wizard_static = {'tools.staticdir.on': True, 'tools.staticdir.dir': os.path.join(sabnzbd.WIZARD_DIR, 'static'), 'tools.staticdir.content_types': forced_mime_types} appconfig = {'/api': { - 'tools.basic_auth.on': False, + 'tools.auth_basic.on': False, 'tools.response_headers.on': True, 'tools.response_headers.headers': [('Access-Control-Allow-Origin', '*')] }, diff --git a/sabnzbd/__init__.py b/sabnzbd/__init__.py index f1149ac..2e6f207 100644 --- a/sabnzbd/__init__.py +++ b/sabnzbd/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -1153,32 +1153,29 @@ def test_ipv6(): def test_cert_checking(): - """ Test quality of certificate validation - On systems with at least Python > 2.7.9 - """ - if sabnzbd.HAVE_SSL_CONTEXT: - # User disabled the test, assume proper SSL certificates - if not cfg.selftest_host(): - return True - # Try a connection to our test-host - try: - import ssl - ctx = ssl.create_default_context() - base_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ssl_sock = ctx.wrap_socket(base_sock, server_hostname=cfg.selftest_host()) - ssl_sock.settimeout(2.0) - ssl_sock.connect((cfg.selftest_host(), 443)) - ssl_sock.close() - return True - except (socket.gaierror, socket.timeout): - # Non-SSL related error. - # We now assume that certificates work instead of forcing - # lower quality just because some (temporary) internet problem - logging.info('Could not determine system certificate validation quality due to connection problems') - return True - except: - # Seems something is still wrong - sabnzbd.set_https_verification(0) + """ Test quality of certificate validation """ + # User disabled the test, assume proper SSL certificates + if not cfg.selftest_host(): + return True + # Try a connection to our test-host + try: + import ssl + ctx = ssl.create_default_context() + base_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ssl_sock = ctx.wrap_socket(base_sock, server_hostname=cfg.selftest_host()) + ssl_sock.settimeout(2.0) + ssl_sock.connect((cfg.selftest_host(), 443)) + ssl_sock.close() + return True + except (socket.gaierror, socket.timeout): + # Non-SSL related error. + # We now assume that certificates work instead of forcing + # lower quality just because some (temporary) internet problem + logging.info('Could not determine system certificate validation quality due to connection problems') + return True + except: + # Seems something is still wrong + sabnzbd.set_https_verification(False) return False diff --git a/sabnzbd/api.py b/sabnzbd/api.py index 4c7698d..9490b95 100644 --- a/sabnzbd/api.py +++ b/sabnzbd/api.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -27,6 +27,7 @@ import time import json import cherrypy import locale +import html from threading import Thread @@ -54,7 +55,7 @@ from sabnzbd.misc import loadavg, to_units, int_conv, time_format, \ from sabnzbd.filesystem import diskspace, get_ext, get_filename, globber, \ globber_full, clip_path, remove_all from sabnzbd.filesystem import same_file -from sabnzbd.encoding import xml_name, unicoder, special_fixer, platform_encode, html_escape +from sabnzbd.encoding import xml_name, unicoder, special_fixer, platform_encode from sabnzbd.postproc import PostProcessor from sabnzbd.articlecache import ArticleCache from sabnzbd.utils.servertests import test_nntp_server_dict @@ -719,9 +720,9 @@ def _api_test_email(name, output, kwargs): """ API: send a test email, return result """ logging.info("Sending test email") pack = {'download': ['action 1', 'action 2'], 'unpack': ['action 1', 'action 2']} - res = sabnzbd.emailer.endjob(u'I had a d\xe8ja vu', 'unknown', True, - os.path.normpath(os.path.join(cfg.complete_dir.get_path(), u'/unknown/I had a d\xe8ja vu')), - 123 * MEBI, None, pack, 'my_script', u'Line 1\nLine 2\nLine 3\nd\xe8ja vu\n', 0, + res = sabnzbd.emailer.endjob('I had a d\xe8ja vu', 'unknown', True, + os.path.normpath(os.path.join(cfg.complete_dir.get_path(), '/unknown/I had a d\xe8ja vu')), + 123 * MEBI, None, pack, 'my_script', 'Line 1\nLine 2\nLine 3\nd\xe8ja vu\n', 0, test=kwargs) if res == 'Email succeeded': res = None @@ -1355,7 +1356,7 @@ def build_queue(start=0, limit=0, trans=False, output=None, search=None): try: datestart = datestart + datetime.timedelta(seconds=bytesleft / bytespersec) # new eta format: 16:00 Fri 07 Feb - slot['eta'] = datestart.strftime(time_format('%H:%M %a %d %b')).decode(codepage) + slot['eta'] = datestart.strftime(time_format('%H:%M %a %d %b')) except: datestart = datetime.datetime.now() slot['eta'] = 'unknown' @@ -1582,7 +1583,7 @@ def Ttemplate(txt): if txt in _SKIN_CACHE: return _SKIN_CACHE[txt] else: - tra = html_escape(Tx(SKIN_TEXT.get(txt, txt))) + tra = html.escape(Tx(SKIN_TEXT.get(txt, txt))) _SKIN_CACHE[txt] = tra return tra diff --git a/sabnzbd/articlecache.py b/sabnzbd/articlecache.py index 8fc422b..ab27a9e 100644 --- a/sabnzbd/articlecache.py +++ b/sabnzbd/articlecache.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/assembler.py b/sabnzbd/assembler.py index 50c0a0d..2af85ff 100644 --- a/sabnzbd/assembler.py +++ b/sabnzbd/assembler.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -29,8 +29,8 @@ import hashlib import sabnzbd from sabnzbd.misc import get_all_passwords -from sabnzbd.filesystem import get_filepath, sanitize_filename, get_unique_filename, \ - renamer, set_permissions, long_path, clip_path, has_win_device, diskspace, \ +from sabnzbd.filesystem import get_filepath, sanitize_filename, \ + set_permissions, long_path, clip_path, has_win_device, diskspace, \ get_filename, get_ext from sabnzbd.constants import Status, GIGI import sabnzbd.cfg as cfg @@ -109,7 +109,7 @@ class Assembler(Thread): nzf.remove_admin() # Do rar-related processing - if is_rarfile(filepath): + if rarfile.is_rarfile(filepath): # Encryption and unwanted extension detection rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(nzo, filepath) if rar_encrypted: @@ -238,7 +238,7 @@ def check_encrypted_and_unwanted_files(nzo, filepath): return encrypted, unwanted # Is it even a rarfile? - if is_rarfile(filepath): + if rarfile.is_rarfile(filepath): # Open the rar rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND zf = rarfile.RarFile(filepath, all_names=True) diff --git a/sabnzbd/bpsmeter.py b/sabnzbd/bpsmeter.py index befe4b1..a16354d 100644 --- a/sabnzbd/bpsmeter.py +++ b/sabnzbd/bpsmeter.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/cfg.py b/sabnzbd/cfg.py index 904e58d..95aa916 100644 --- a/sabnzbd/cfg.py +++ b/sabnzbd/cfg.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/config.py b/sabnzbd/config.py index a34f927..38f7853 100644 --- a/sabnzbd/config.py +++ b/sabnzbd/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -24,8 +24,6 @@ import re import logging import threading import shutil -import time -import random import uuid from urllib.parse import urlparse import sabnzbd.misc diff --git a/sabnzbd/constants.py b/sabnzbd/constants.py index 4d9eff2..88e28e3 100644 --- a/sabnzbd/constants.py +++ b/sabnzbd/constants.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/database.py b/sabnzbd/database.py index 7409459..092f900 100644 --- a/sabnzbd/database.py +++ b/sabnzbd/database.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -19,20 +19,13 @@ sabnzbd.database - Database Support """ -try: - import sqlite3 -except: - try: - import pysqlite2.dbapi2 as sqlite3 - except: - pass - import os import time import zlib import logging import sys import threading +import sqlite3 import sabnzbd import sabnzbd.cfg @@ -40,7 +33,7 @@ from sabnzbd.constants import DB_HISTORY_NAME, STAGES from sabnzbd.encoding import unicoder from sabnzbd.bpsmeter import this_week, this_month from sabnzbd.decorators import synchronized -from sabnzbd.misc import get_all_passwords, int_conv, caller_name +from sabnzbd.misc import int_conv, caller_name from sabnzbd.filesystem import remove_file DB_LOCK = threading.RLock() diff --git a/sabnzbd/decoder.py b/sabnzbd/decoder.py index 1d64518..1f60247 100644 --- a/sabnzbd/decoder.py +++ b/sabnzbd/decoder.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -19,7 +19,6 @@ sabnzbd.decoder - article decoder """ -import binascii import logging import re import hashlib @@ -27,12 +26,11 @@ from time import sleep from threading import Thread import sabnzbd -from sabnzbd.constants import Status, MAX_DECODE_QUEUE, LIMIT_DECODE_QUEUE, SABYENC_VERSION_REQUIRED +from sabnzbd.constants import MAX_DECODE_QUEUE, LIMIT_DECODE_QUEUE, SABYENC_VERSION_REQUIRED import sabnzbd.articlecache import sabnzbd.downloader import sabnzbd.nzbqueue import sabnzbd.cfg as cfg -from sabnzbd.encoding import yenc_name_fixer from sabnzbd.misc import match_str # Check for correct SABYenc version diff --git a/sabnzbd/decorators.py b/sabnzbd/decorators.py index 797716f..a02e4e7 100644 --- a/sabnzbd/decorators.py +++ b/sabnzbd/decorators.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/directunpacker.py b/sabnzbd/directunpacker.py index c03d1d3..79f2ae7 100644 --- a/sabnzbd/directunpacker.py +++ b/sabnzbd/directunpacker.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -30,8 +30,7 @@ from subprocess import Popen import sabnzbd import sabnzbd.cfg as cfg from sabnzbd.misc import int_conv, format_time_string -from sabnzbd.filesystem import clip_path, long_path, remove_all, globber, \ - has_win_device, real_path +from sabnzbd.filesystem import clip_path, long_path, remove_all, real_path, remove_file from sabnzbd.encoding import TRANS, unicoder from sabnzbd.decorators import synchronized from sabnzbd.newsunpack import build_command, EXTRACTFROM_RE, EXTRACTED_RE, rar_volumelist @@ -39,6 +38,10 @@ from sabnzbd.postproc import prepare_extraction_path from sabnzbd.utils.rarfile import RarFile from sabnzbd.utils.diskspeed import diskspeedmeasure +# Need a lock to make sure start and stop is handled correctlty +# Otherwise we could stop while the thread was still starting +START_STOP_LOCK = threading.RLock() + MAX_ACTIVE_UNPACKERS = 10 ACTIVE_UNPACKERS = [] diff --git a/sabnzbd/dirscanner.py b/sabnzbd/dirscanner.py index afd3097..f469020 100644 --- a/sabnzbd/dirscanner.py +++ b/sabnzbd/dirscanner.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -76,7 +76,7 @@ def is_archive(path): except: logging.info(T('Cannot read %s'), path, exc_info=True) return -1, None, '' - elif misc.is_rarfile(path): + elif rarfile.is_rarfile(path): try: # Set path to tool to open it rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND diff --git a/sabnzbd/downloader.py b/sabnzbd/downloader.py index 61e8d14..e6e1b32 100644 --- a/sabnzbd/downloader.py +++ b/sabnzbd/downloader.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/emailer.py b/sabnzbd/emailer.py index 2fee0e2..18fcd72 100644 --- a/sabnzbd/emailer.py +++ b/sabnzbd/emailer.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -250,7 +250,7 @@ def endjob(filename, cat, status, path, bytes, fail_msg, stages, script, script_ parm['script_ret'] = script_ret parm['cat'] = cat parm['size'] = "%sB" % to_units(bytes) - parm['end_time'] = time.strftime(time_format('%Y-%m-%d %H:%M:%S'), time.localtime(time.time())).decode(codepage) + parm['end_time'] = time.strftime(time_format('%Y-%m-%d %H:%M:%S'), time.localtime(time.time())) return send_with_template('email', parm, test) diff --git a/sabnzbd/encoding.py b/sabnzbd/encoding.py index 5202f54..9b00fe9 100644 --- a/sabnzbd/encoding.py +++ b/sabnzbd/encoding.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -246,15 +246,6 @@ _HTML_TABLE = { } -def html_escape(txt): - """ Replace HTML metacharacters with &-constructs """ - # Replacement for inefficient xml.sax.saxutils.escape function - if any(ch in txt for ch in _HTML_TABLE): - return ''.join((_HTML_TABLE.get(ch, ch) for ch in txt)) - else: - return txt - - def deunicode(p): return p """ Return the correct 8bit ASCII encoding for the platform: @@ -283,10 +274,14 @@ def deunicode(p): def utob(str_in): """ Shorthand for converting UTF-8 to bytes """ + if isinstance(str_in, bytes): + return str_in return str_in.encode('utf-8') def ubtou(str_in): """ Shorthand for converting unicode bytes to UTF-8 """ + if not isinstance(str_in, bytes): + return str_in return str_in.decode('utf-8') diff --git a/sabnzbd/filesystem.py b/sabnzbd/filesystem.py index 2198937..5c3c308 100644 --- a/sabnzbd/filesystem.py +++ b/sabnzbd/filesystem.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2008-2017 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -22,22 +22,17 @@ sabnzbd.misc - filesystem operations import os import sys import logging -import urllib.request, urllib.parse, urllib.error import re import shutil import threading -import subprocess -import socket import time -import datetime import fnmatch import stat import sabnzbd from sabnzbd.decorators import synchronized -from sabnzbd.constants import DEFAULT_PRIORITY, FUTURE_Q_FOLDER, JOB_ADMIN, \ - GIGI, MEBI -from sabnzbd.encoding import ubtou, unicoder, special_fixer, gUTF +from sabnzbd.constants import FUTURE_Q_FOLDER, JOB_ADMIN, GIGI +from sabnzbd.encoding import special_fixer, gUTF def get_ext(filename): @@ -597,6 +592,22 @@ def create_dirs(dirpath): @synchronized(DIR_LOCK) +def recursive_listdir(dir): + """ List all files in dirs and sub-dirs """ + filelist = [] + for root, dirs, files in os.walk(dir): + for file in files: + if '.AppleDouble' not in root and '.DS_Store' not in root: + try: + p = os.path.join(root, file) + filelist.append(p) + except UnicodeDecodeError: + # Just skip failing names + pass + return filelist + + +@synchronized(DIR_LOCK) def move_to_path(path, new_path): """ Move a file to a new path, optionally give unique filename Return (ok, new_path) @@ -665,10 +676,9 @@ def get_filepath(path, nzo, filename): # It does no umask setting # It uses the dir_lock for the (rare) case that the # download_dir is equal to the complete_dir. - dName = nzo.work_name if not nzo.created: for n in range(200): - dName = dirname + dName = nzo.work_name if n: dName += '.' + str(n) try: diff --git a/sabnzbd/getipaddress.py b/sabnzbd/getipaddress.py index 72bf948..344007d 100644 --- a/sabnzbd/getipaddress.py +++ b/sabnzbd/getipaddress.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -22,11 +22,13 @@ sabnzbd.getipaddress import socket import multiprocessing.pool import functools +import urllib.request import sabnzbd import sabnzbd.cfg from sabnzbd.encoding import ubtou + # decorator stuff: def timeout(max_timeout): """Timeout decorator, parameter in seconds.""" @@ -72,14 +74,13 @@ def localipv4(): def publicipv4(): # Because of dual IPv4/IPv6 clients, finding the public ipv4 needs special attention, # meaning forcing IPv4 connections, and not allowing IPv6 connections + public_ipv4 = None try: - import urllib.request, urllib.error, urllib.parse ipv4_found = False # we only want IPv4 resolving, so socket.AF_INET: result = addresslookup4(sabnzbd.cfg.selftest_host()) except: # something very bad: no urllib2, no resolving of selftest_host, no network at all - public_ipv4 = None return public_ipv4 # we got one or more IPv4 address(es), so let's connect to them diff --git a/sabnzbd/interface.py b/sabnzbd/interface.py index d22fbba..4edc320 100644 --- a/sabnzbd/interface.py +++ b/sabnzbd/interface.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -41,8 +41,9 @@ import sabnzbd.scheduler as scheduler from Cheetah.Template import Template from sabnzbd.misc import to_units, from_units, time_format, calc_age, \ - cat_to_opts, int_conv, get_base_url, probablyipv4 -from sabnzbd.filesystem import real_path, long_path, globber, globber_full, remove_all, clip_path, same_file + int_conv, get_base_url, probablyipv4, probablyipv6 +from sabnzbd.filesystem import real_path, long_path, globber, globber_full, remove_all, \ + clip_path, same_file from sabnzbd.newswrapper import GetServerParms from sabnzbd.bpsmeter import BPSMeter from sabnzbd.encoding import TRANS, xml_name, LatinFilter, unicoder, special_fixer, \ @@ -247,13 +248,13 @@ def encrypt_pwd(pwd): def set_auth(conf): """ Set the authentication for CherryPy """ if cfg.username() and cfg.password() and not cfg.html_login(): - conf.update({'tools.basic_auth.on': True, 'tools.basic_auth.realm': 'SABnzbd', - 'tools.basic_auth.users': get_users, 'tools.basic_auth.encrypt': encrypt_pwd}) - conf.update({'/api': {'tools.basic_auth.on': False}, - '%s/api' % cfg.url_base(): {'tools.basic_auth.on': False}, + conf.update({'tools.auth_basic.on': True, 'tools.auth_basic.realm': 'SABnzbd', + 'tools.auth_basic.users': get_users, 'tools.auth_basic.encrypt': encrypt_pwd}) + conf.update({'/api': {'tools.auth_basic.on': False}, + '%s/api' % cfg.url_base(): {'tools.auth_basic.on': False}, }) else: - conf.update({'tools.basic_auth.on': False}) + conf.update({'tools.auth_basic.on': False}) def check_session(kwargs): @@ -297,14 +298,14 @@ def check_apikey(kwargs, nokey=False): key = kwargs.get('session') if not key: if cfg.api_warnings(): - log_warning(T('API Key missing, please enter the api key from Config->General into your 3rd party program:')) + log_warning_and_ip(T('API Key missing, please enter the api key from Config->General into your 3rd party program:')) return report(output, 'API Key Required') elif req_access == 1 and key == cfg.nzb_key(): return None elif key == cfg.api_key(): return None else: - log_warning(T('API Key incorrect, Use the api key from Config->General in your 3rd party program:')) + log_warning_and_ip(T('API Key incorrect, Use the api key from Config->General in your 3rd party program:')) return report(output, 'API Key Incorrect') # No active APIKEY, check web credentials instead @@ -313,7 +314,7 @@ def check_apikey(kwargs, nokey=False): pass else: if cfg.api_warnings(): - log_warning(T('Authentication missing, please enter username/password from Config->General into your 3rd party program:')) + log_warning_and_ip(T('Authentication missing, please enter username/password from Config->General into your 3rd party program:')) return report(output, 'Missing authentication') return None @@ -2560,7 +2561,7 @@ def GetRssLog(feed): if job.get('time_downloaded'): job['time_downloaded_ms'] = time.mktime(job['time_downloaded']) - job['time_downloaded'] = time.strftime(time_format('%H:%M %a %d %b'), job['time_downloaded']).decode(codepage) + job['time_downloaded'] = time.strftime(time_format('%H:%M %a %d %b'), job['time_downloaded']) else: job['time_downloaded_ms'] = '' job['time_downloaded'] = '' diff --git a/sabnzbd/lang.py b/sabnzbd/lang.py index 3006861..eb33423 100644 --- a/sabnzbd/lang.py +++ b/sabnzbd/lang.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # -*- coding: utf-8 -*- # Copyright 2011-2018 The SABnzbd-Team # diff --git a/sabnzbd/misc.py b/sabnzbd/misc.py index adfb9da..ad58ff9 100644 --- a/sabnzbd/misc.py +++ b/sabnzbd/misc.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -18,30 +18,24 @@ """ sabnzbd.misc - misc classes """ - import os import sys import logging import urllib.request, urllib.parse, urllib.error import re -import ctypes -import shutil -import threading import subprocess import socket import time import datetime -import fnmatch -import stat import inspect +import ctypes import sabnzbd -from sabnzbd.decorators import synchronized -from sabnzbd.constants import DEFAULT_PRIORITY, FUTURE_Q_FOLDER, JOB_ADMIN, \ - GIGI, MEBI, DEF_ARTICLE_CACHE_DEFAULT, DEF_ARTICLE_CACHE_MAX +from sabnzbd.constants import DEFAULT_PRIORITY, \ + MEBI, DEF_ARTICLE_CACHE_DEFAULT, DEF_ARTICLE_CACHE_MAX import sabnzbd.config as config import sabnzbd.cfg as cfg -import sabnzbd.utils.rarfile as rarfile +from sabnzbd.encoding import ubtou, unicoder TAB_UNITS = ('', 'K', 'M', 'G', 'T', 'P') RE_UNITS = re.compile(r'(\d+\.*\d*)\s*([KMGTP]{0,1})', re.I) @@ -290,11 +284,20 @@ def set_serv_parms(service, args): return True +def get_from_url(url): + """ Retrieve URL and return content """ + try: + with urllib.request.urlopen(url) as response: + return response.read() + except: + return None + + def convert_version(text): """ Convert version string to numerical value and a testversion indicator """ version = 0 test = True - m = RE_VERSION.search(text) + m = RE_VERSION.search(ubtou(text)) if m: version = int(m.group(1)) * 1000000 + int(m.group(2)) * 10000 + int(m.group(3)) * 100 try: @@ -340,17 +343,9 @@ def check_latest_version(): logging.debug("Unsupported release number (%s), will not check", sabnzbd.__version__) return - # Using catch-all except's is poor coding practice. - # However, the last thing you want is the app crashing due - # to bad file content. - - try: - fn = urllib.request.urlretrieve('https://raw.githubusercontent.com/sabnzbd/sabnzbd.github.io/master/latest.txt')[0] - f = open(fn, 'r') - data = f.read() - f.close() - os.remove(fn) - except: + # Fetch version info + data = get_from_url('https://raw.githubusercontent.com/sabnzbd/sabnzbd.github.io/master/latest.txt') + if not data: logging.info('Cannot retrieve version information from GitHub.com') logging.debug('Traceback: ', exc_info=True) return @@ -543,6 +538,37 @@ def get_cache_limit(): return '' +def get_windows_memory(): + """ Use ctypes to extract available memory """ + class MEMORYSTATUSEX(ctypes.Structure): + _fields_ = [ + ("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("sullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + def __init__(self): + # have to initialize this to the size of MEMORYSTATUSEX + self.dwLength = ctypes.sizeof(self) + super(MEMORYSTATUSEX, self).__init__() + + stat = MEMORYSTATUSEX() + ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) + return stat.ullTotalPhys + + +def get_darwin_memory(): + """ Use system-call to extract total memory on macOS """ + system_output = sabnzbd.newsunpack.run_simple(['sysctl', 'hw.memsize']) + return float(system_output.split()[1]) + + def on_cleanup_list(filename, skip_nzb=False): """ Return True if a filename matches the clean-up list """ lst = cfg.cleanup_list() @@ -642,7 +668,7 @@ def create_https_certificates(ssl_cert, ssl_key): try: from sabnzbd.utils.certgen import generate_key, generate_local_cert private_key = generate_key(key_size=2048, output_file=ssl_key) - generate_local_cert(private_key, days_valid=3560, output_file=ssl_cert, LN=u'SABnzbd', ON=u'SABnzbd') + generate_local_cert(private_key, days_valid=3560, output_file=ssl_cert, LN='SABnzbd', ON='SABnzbd') logging.info('Self-signed certificates generated successfully') except: logging.error(T('Error creating SSL key and certificate')) @@ -791,12 +817,6 @@ def match_str(text, matches): return None -def get_urlbase(url): - """ Return the base URL (like http://server.domain.com/) """ - parsed_uri = urllib.parse.urlparse(url) - return '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri) - - def nntp_to_msg(text): """ Format raw NNTP bytes data for display """ if isinstance(text, list): diff --git a/sabnzbd/newsunpack.py b/sabnzbd/newsunpack.py index 5a2875b..b2c5ea2 100644 --- a/sabnzbd/newsunpack.py +++ b/sabnzbd/newsunpack.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -31,12 +31,12 @@ import functools from subprocess import Popen import sabnzbd -from sabnzbd.encoding import ubtou, TRANS, UNTRANS, unicoder, platform_encode, deunicode +from sabnzbd.encoding import ubtou, TRANS, unicoder, platform_encode, deunicode import sabnzbd.utils.rarfile as rarfile from sabnzbd.misc import format_time_string, find_on_path, int_conv, \ get_all_passwords, calc_age, cmp -from sabnzbd.filesystem import make_script_path, real_path, globber, globber_full, \ - renamer, clip_path,has_win_device, long_path +from sabnzbd.filesystem import make_script_path, real_path, globber, globber_full, \ + renamer, clip_path, long_path, remove_file, recursive_listdir from sabnzbd.sorting import SeriesSorter import sabnzbd.cfg as cfg from sabnzbd.constants import Status @@ -649,7 +649,7 @@ def rar_extract_core(rarfile_path, numrars, one_folder, nzo, setname, extraction stup, need_shell, command, creationflags = build_command(command, flatten_command=True) # Get list of all the volumes part of this set - logging.debug("Analyzing rar file ... %s found", is_rarfile(rarfile_path)) + logging.debug("Analyzing rar file ... %s found", rarfile.is_rarfile(rarfile_path)) logging.debug("Running unrar %s", command) p = Popen(command, shell=need_shell, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -2116,7 +2116,7 @@ def build_filelists(workdir, workdir_complete=None, check_both=False, check_rar= # Extra check for rar (takes CPU/disk) file_is_rar = False if check_rar: - file_is_rar = is_rarfile(file) + file_is_rar = rarfile.is_rarfile(file) # Run through all the checks if SEVENZIP_RE.search(file) or SEVENMULTI_RE.search(file): @@ -2375,17 +2375,6 @@ def list2cmdline(lst): return ' '.join(nlst) -def get_from_url(url): - """ Retrieve URL and return content - `timeout` sets non-standard timeout - """ - import urllib.request, urllib.error, urllib.parse - try: - return urllib.request.urlopen(url).read() - except: - return None - - def is_sevenfile(path): """ Return True if path has proper extension and 7Zip is installed """ return SEVEN_COMMAND and os.path.splitext(path)[1].lower() == '.7z' diff --git a/sabnzbd/newswrapper.py b/sabnzbd/newswrapper.py index 3043060..a184109 100644 --- a/sabnzbd/newswrapper.py +++ b/sabnzbd/newswrapper.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/notifier.py b/sabnzbd/notifier.py index ba1c47d..7c0ee3c 100644 --- a/sabnzbd/notifier.py +++ b/sabnzbd/notifier.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -23,10 +23,8 @@ sabnzbd.notifier - Send notifications to any notification services import os.path import logging -import socket import urllib.request, urllib.error, urllib.parse import http.client -import time import subprocess import json from threading import Thread diff --git a/sabnzbd/nzbparser.py b/sabnzbd/nzbparser.py index 060e11b..30df71d 100644 --- a/sabnzbd/nzbparser.py +++ b/sabnzbd/nzbparser.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2008-2017 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -19,7 +19,6 @@ sabnzbd.nzbparser - Parse and import NZB files """ -import io import time import logging import hashlib @@ -86,7 +85,7 @@ def nzbfile_parser(raw_data, nzo): # Dubplicate parts? if partnum in article_db: - if article_id != sarticle_db[partnum][0]: + if article_id != article_db[partnum][0]: logging.info('Duplicate part %s, but different ID-s (%s // %s)', partnum, article_db[partnum][0], article_id) nzo.increase_bad_articles_counter('duplicate_articles') else: @@ -123,5 +122,5 @@ def nzbfile_parser(raw_data, nzo): nzo.md5sum = md5sum.hexdigest() if skipped_files: - logging.warning(T('Failed to import %s files from %s'), self.skipped_files, self.nzo.filename) + logging.warning(T('Failed to import %s files from %s'), skipped_files, nzo.filename) diff --git a/sabnzbd/nzbqueue.py b/sabnzbd/nzbqueue.py index c56a766..b5e59c7 100644 --- a/sabnzbd/nzbqueue.py +++ b/sabnzbd/nzbqueue.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -27,8 +27,8 @@ import functools import sabnzbd from sabnzbd.nzbstuff import NzbObject -from sabnzbd.misc import exit_sab, cat_to_opts, int_conv, caller_name -from sabnzbd.filesystem import get_admin_path, remove_all, globber_full +from sabnzbd.misc import exit_sab, cat_to_opts, int_conv, caller_name, cmp +from sabnzbd.filesystem import get_admin_path, remove_all, globber_full, remove_file from sabnzbd.panic import panic_queue import sabnzbd.database as database from sabnzbd.decorators import NzbQueueLocker diff --git a/sabnzbd/nzbstuff.py b/sabnzbd/nzbstuff.py index aad7e8f..4305909 100644 --- a/sabnzbd/nzbstuff.py +++ b/sabnzbd/nzbstuff.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -19,17 +19,12 @@ sabnzbd.nzbstuff - misc """ -# Standard Library import os import time import re import logging import datetime import threading -import xml.sax -import xml.sax.handler -import xml.sax.xmlreader -import hashlib import functools import difflib @@ -44,11 +39,11 @@ from sabnzbd.misc import to_units, cat_to_opts, cat_convert, int_conv, \ from sabnzbd.filesystem import sanitize_foldername, get_unique_path, get_admin_path, \ remove_all, sanitize_filename, globber_full, set_permissions, long_path, \ trim_win_path, fix_unix_encoding, is_obfuscated_filename, get_ext, get_filename, \ - get_unique_filename, renamer + get_unique_filename, renamer, remove_file, remove_dir from sabnzbd.decorators import synchronized import sabnzbd.config as config import sabnzbd.cfg as cfg -from sabnzbd.encoding import utob, unicoder, platform_encode +from sabnzbd.encoding import unicoder, platform_encode import sabnzbd.nzbparser from sabnzbd.database import HistoryDB from sabnzbd.rating import Rating diff --git a/sabnzbd/osxmenu.py b/sabnzbd/osxmenu.py index 85eaaaa..92c8b69 100644 --- a/sabnzbd/osxmenu.py +++ b/sabnzbd/osxmenu.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -26,11 +26,8 @@ from PyObjCTools import AppHelper from objc import YES, NO import os - -import cherrypy import sys import time - import logging import sabnzbd diff --git a/sabnzbd/panic.py b/sabnzbd/panic.py index 3cf1da5..c95474a 100644 --- a/sabnzbd/panic.py +++ b/sabnzbd/panic.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -182,10 +182,6 @@ def panic_tmpl(name): launch_a_browser(panic_message(PANIC_TEMPL, name, 0)) -def panic_sqlite(name): - launch_a_browser(panic_message(PANIC_SQLITE, name, 0)) - - def panic(reason, remedy=""): show_error_dialog("\n%s:\n %s\n%s" % (T('Fatal error'), reason, remedy)) launch_a_browser(panic_message(PANIC_OTHER, reason, remedy)) diff --git a/sabnzbd/par2file.py b/sabnzbd/par2file.py index 3a856f3..7df29e9 100644 --- a/sabnzbd/par2file.py +++ b/sabnzbd/par2file.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -131,7 +131,7 @@ def parse_par2_file_packet(f, header): nothing = None, None, None if header != PAR_PKT_ID: - print header + print(header) return nothing # Length must be multiple of 4 and at least 20 diff --git a/sabnzbd/postproc.py b/sabnzbd/postproc.py index 2b259a3..7853178 100644 --- a/sabnzbd/postproc.py +++ b/sabnzbd/postproc.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -20,7 +20,6 @@ sabnzbd.postproc - threaded post-processing of jobs """ import os -import queue import logging import sabnzbd import xml.sax.saxutils @@ -36,7 +35,7 @@ from sabnzbd.misc import on_cleanup_list from sabnzbd.filesystem import real_path, get_unique_path, create_dirs, move_to_path, \ make_script_path, long_path, clip_path, renamer, remove_dir, remove_all, globber, \ globber_full, set_permissions, cleanup_empty_directories, fix_unix_encoding, \ - sanitize_and_trim_path, sanitize_files_in_folder + sanitize_and_trim_path, sanitize_files_in_folder, remove_file, recursive_listdir from sabnzbd.sorting import Sorter from sabnzbd.constants import REPAIR_PRIORITY, TOP_PRIORITY, POSTPROC_QUEUE_FILE_NAME, \ POSTPROC_QUEUE_VERSION, sample_match, JOB_ADMIN, Status, VERIFIED_FILE diff --git a/sabnzbd/powersup.py b/sabnzbd/powersup.py index 437d61c..1615db9 100644 --- a/sabnzbd/powersup.py +++ b/sabnzbd/powersup.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/rating.py b/sabnzbd/rating.py index 3def19b..e590b0b 100644 --- a/sabnzbd/rating.py +++ b/sabnzbd/rating.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2008-2012 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -20,12 +20,10 @@ sabnzbd.rating - Rating support functions """ import http.client -import urllib.request, urllib.parse, urllib.error import urllib.parse import time import logging import copy -import socket import queue import collections from threading import RLock, Thread diff --git a/sabnzbd/rss.py b/sabnzbd/rss.py index 84fea30..68a2ddb 100644 --- a/sabnzbd/rss.py +++ b/sabnzbd/rss.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -31,7 +31,7 @@ from sabnzbd.decorators import synchronized import sabnzbd.config as config import sabnzbd.cfg as cfg from sabnzbd.misc import cat_convert, wildcard_to_re, cat_to_opts, \ - match_str, from_units, int_conv, get_urlbase + match_str, from_units, int_conv, get_base_url import sabnzbd.emailer as emailer from sabnzbd.encoding import unicoder, xml_name @@ -298,7 +298,7 @@ class RSSQueue(object): if 'bozo_exception' in feed_parsed and not entries: msg = str(feed_parsed['bozo_exception']) if 'CERTIFICATE_VERIFY_FAILED' in msg: - msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri) + msg = T('Server %s uses an untrusted HTTPS certificate') % get_base_url(uri) msg += ' - https://sabnzbd.org/certificate-errors' logging.error(msg) elif feed_parsed['href'] != uri and 'login' in feed_parsed['href']: @@ -631,7 +631,7 @@ def _get_link(entry): """ Retrieve the post link from this entry Returns (link, category, size) """ - size = 0L + size = 0 age = datetime.datetime.now() # Try standard link and enclosures first @@ -650,7 +650,7 @@ def _get_link(entry): if entry.id and entry.id != link and entry.id.startswith('http'): infourl = entry.id - if size == 0L: + if size == 0: _RE_SIZE1 = re.compile(r'Size:\s*(\d+\.\d+\s*[KMG]{0,1})B\W*', re.I) _RE_SIZE2 = re.compile(r'\W*(\d+\.\d+\s*[KMG]{0,1})B\W*', re.I) # Try to find size in Description @@ -702,7 +702,7 @@ def _get_link(entry): return link, infourl, category, size, age, season, episode else: logging.warning(T('Empty RSS entry found (%s)'), link) - return None, None, '', 0L, None, 0, 0 + return None, None, '', 0, None, 0, 0 def special_rss_site(url): diff --git a/sabnzbd/sabtray.py b/sabnzbd/sabtray.py index 2673a73..1600084 100644 --- a/sabnzbd/sabtray.py +++ b/sabnzbd/sabtray.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/sabtraylinux.py b/sabnzbd/sabtraylinux.py index f59b632..e1d0c48 100644 --- a/sabnzbd/sabtraylinux.py +++ b/sabnzbd/sabtraylinux.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/scheduler.py b/sabnzbd/scheduler.py index 53ff338..3fae58d 100644 --- a/sabnzbd/scheduler.py +++ b/sabnzbd/scheduler.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/skintext.py b/sabnzbd/skintext.py index 743a614..36d6ba3 100644 --- a/sabnzbd/skintext.py +++ b/sabnzbd/skintext.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # -*- coding: UTF-8 -*- # Copyright 2012-2018 The SABnzbd-Team # diff --git a/sabnzbd/sorting.py b/sabnzbd/sorting.py index 30ad9b8..e38089f 100644 --- a/sabnzbd/sorting.py +++ b/sabnzbd/sorting.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/urlgrabber.py b/sabnzbd/urlgrabber.py index 7b42795..f4bd967 100644 --- a/sabnzbd/urlgrabber.py +++ b/sabnzbd/urlgrabber.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/utils/certgen.py b/sabnzbd/utils/certgen.py index 31dba9f..2058ecb 100644 --- a/sabnzbd/utils/certgen.py +++ b/sabnzbd/utils/certgen.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python33 """ Adapted from the docs of cryptography @@ -52,7 +52,7 @@ def generate_key(key_size=2048, output_file='key.pem'): # Ported from cryptography docs/x509/tutorial.rst -def generate_local_cert(private_key, days_valid=3560, output_file='cert.cert', LN=u'SABnzbd', ON=u'SABnzbd'): +def generate_local_cert(private_key, days_valid=3560, output_file='cert.cert', LN='SABnzbd', ON='SABnzbd'): # Various details about who we are. For a self-signed certificate the # subject and issuer are always the same. subject = issuer = x509.Name([ @@ -64,7 +64,7 @@ def generate_local_cert(private_key, days_valid=3560, output_file='cert.cert', L # build Subject Alternate Names (aka SAN) list # First the host names, add with x509.DNSName(): - san_list = [x509.DNSName(u"localhost"), x509.DNSName(unicode(socket.gethostname()))] + san_list = [x509.DNSName("localhost"), x509.DNSName(str(socket.gethostname()))] # Then the host IP addresses, add with x509.IPAddress() # Inside a try-except, just to be sure diff --git a/sabnzbd/utils/happyeyeballs.py b/sabnzbd/utils/happyeyeballs.py index b5c1b6b..a99a85a 100644 --- a/sabnzbd/utils/happyeyeballs.py +++ b/sabnzbd/utils/happyeyeballs.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python - +#!/usr/bin/python33 # Python implementation of RFC 6555 / Happy Eyeballs: find the quickest IPv4/IPv6 connection # See https://tools.ietf.org/html/rfc6555 # Method: Start parallel sessions using threads, and only wait for the quickest succesful socket connect @@ -11,20 +10,20 @@ from happyeyeballs import happyeyeballs print happyeyeballs('newszilla.xs4all.nl', port=119) """ # or with more logging: -''' +""" from happyeyeballs import happyeyeballs import logging logger = logging.getLogger('') logger.setLevel(logging.DEBUG) print happyeyeballs('newszilla.xs4all.nl', port=119) -''' +""" import socket import ssl -import queue import threading import time import logging +import queue DEBUG = False @@ -39,7 +38,7 @@ def do_socket_connect(queue, ip, PORT, SSL, ipv4delay): if ip.find(':') >= 0: s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) if ip.find('.') >= 0: - time.sleep(ipv4delay) # IPv4 ... so a delay for IPv4 as we prefer IPv6. Note: ipv4delay could be 0 + time.sleep(ipv4delay) # IPv4 ... so a delay for IPv4 as we prefer IPv6. Note: ipv4delay could be 0 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(3) @@ -78,11 +77,11 @@ def happyeyeballs(HOST, **kwargs): try: preferipv6 = kwargs['preferipv6'] except: - preferipv6 = True # prefer IPv6, so give IPv6 connects a head start by delaying IPv4 + preferipv6 = True # prefer IPv6, so give IPv6 connects a head start by delaying IPv4 # Find out if a cached result is available, and recent enough: - timecurrent = int(time.time()) # current time in seconds since epoch + timecurrent = int(time.time()) # current time in seconds since epoch retentionseconds = 100 hostkey = (HOST, PORT, SSL, preferipv6) # Example key: (u'ssl.astraweb.com', 563, True, True) try: @@ -102,7 +101,7 @@ def happyeyeballs(HOST, **kwargs): # we only arrive here if the entry has to be determined. So let's do that: # We have to determine the (new) best IP address - start = time.clock() + start = time.perf_counter() if DEBUG: logging.debug("\n\n%s %s %s %s", HOST, PORT, SSL, preferipv6) ipv4delay = 0 @@ -111,42 +110,42 @@ def happyeyeballs(HOST, **kwargs): socket.getaddrinfo(HOST, PORT, socket.AF_INET6, socket.SOCK_STREAM, socket.IPPROTO_IP, socket.AI_CANONNAME) if DEBUG: logging.debug("IPv6 address found for %s", HOST) if preferipv6: - ipv4delay=0.1 # preferipv6, AND at least one IPv6 found, so give IPv4 (!) a delay so that IPv6 has a head start and is preferred + ipv4delay=0.1 # preferipv6, AND at least one IPv6 found, so give IPv4 (!) a delay so that IPv6 has a head start and is preferred except: if DEBUG: logging.debug("No IPv6 address found for %s", HOST) - myqueue = queue.Queue() # queue used for threads giving back the results + myqueue = queue.Queue() # queue used for threads giving back the results try: - # Get all IP (IPv4 and IPv6) addresses: + # Get all IP (IPv4 and IPv6) addresses: allinfo = socket.getaddrinfo(HOST, PORT, 0, 0, socket.IPPROTO_TCP) for info in allinfo: address = info[4][0] thisthread = threading.Thread(target=do_socket_connect, args=(myqueue, address, PORT, SSL, ipv4delay)) thisthread.daemon = True thisthread.start() - result = None # default return value, used if none of threads says True/"OK", so no connect on any IP address + result = None # default return value, used if none of threads says True/"OK", so no connect on any IP address # start reading from the Queue for message from the threads: for i in range(len(allinfo)): - s = myqueue.get() # get a response + s = myqueue.get() # get a response if s[1] == True: result = s[0] - break # the first True/"OK" is enough, so break out of for loop + break # the first True/"OK" is enough, so break out of for loop except: if DEBUG: logging.debug("something went wrong in the try block") result = None logging.info("Quickest IP address for %s (port %s, ssl %s, preferipv6 %s) is %s", HOST, PORT, SSL, preferipv6, result) - delay = int(1000 * (time.clock() - start)) + delay = int(1000 * (time.perf_counter() - start)) logging.debug("Happy Eyeballs lookup and port connect took %s ms", delay) # We're done. Store and return the result if result: happyeyeballs.happylist[hostkey] = ( result, timecurrent ) - if DEBUG: logging.debug("Determined new result for %s with result %s", (hostkey, happyeyeballs.happylist[hostkey]) ) + if DEBUG: logging.debug("Determined new result for %s with result %s", hostkey, happyeyeballs.happylist[hostkey] ) return result -happyeyeballs.happylist = {} # The cached results. This static variable must be after the def happyeyeballs() +happyeyeballs.happylist = {} # The cached results. This static variable must be after the def happyeyeballs() @@ -156,23 +155,23 @@ if __name__ == '__main__': logger.setLevel(logging.INFO) if DEBUG: logger.setLevel(logging.DEBUG) - # plain HTTP/HTTPS sites: - print((happyeyeballs('www.google.com'))) - print((happyeyeballs('www.google.com', port=443, ssl=True))) - print((happyeyeballs('www.nu.nl'))) - - # newsservers: - print((happyeyeballs('newszilla6.xs4all.nl', port=119))) - print((happyeyeballs('newszilla.xs4all.nl', port=119))) - print((happyeyeballs('block.cheapnews.eu', port=119))) - print((happyeyeballs('block.cheapnews.eu', port=443, ssl=True))) - print((happyeyeballs('sslreader.eweka.nl', port=563, ssl=True))) - print((happyeyeballs('news.thundernews.com', port=119))) - print((happyeyeballs('news.thundernews.com', port=119, preferipv6=False))) - print((happyeyeballs('secure.eu.thundernews.com', port=563, ssl=True))) - - # Strange cases - print((happyeyeballs('does.not.resolve', port=443, ssl=True))) - print((happyeyeballs('www.google.com', port=119))) - print((happyeyeballs('216.58.211.164'))) + # plain HTTP/HTTPS sites: + print((happyeyeballs('www.google.com'))) + print((happyeyeballs('www.google.com', port=443, ssl=True))) + print((happyeyeballs('www.nu.nl'))) + + # newsservers: + print((happyeyeballs('newszilla6.xs4all.nl', port=119))) + print((happyeyeballs('newszilla.xs4all.nl', port=119))) + print((happyeyeballs('block.cheapnews.eu', port=119))) + print((happyeyeballs('block.cheapnews.eu', port=443, ssl=True))) + print((happyeyeballs('sslreader.eweka.nl', port=563, ssl=True))) + print((happyeyeballs('news.thundernews.com', port=119))) + print((happyeyeballs('news.thundernews.com', port=119, preferipv6=False))) + print((happyeyeballs('secure.eu.thundernews.com', port=563, ssl=True))) + + # Strange cases + print((happyeyeballs('does.not.resolve', port=443, ssl=True))) + print((happyeyeballs('www.google.com', port=119))) + print((happyeyeballs('216.58.211.164'))) diff --git a/sabnzbd/utils/kronos.py b/sabnzbd/utils/kronos.py index c96ab80..58acb61 100644 --- a/sabnzbd/utils/kronos.py +++ b/sabnzbd/utils/kronos.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python33 """Module that provides a cron-like task scheduler. This task scheduler is designed to be used from inside your own program. diff --git a/sabnzbd/utils/pystone.py b/sabnzbd/utils/pystone.py index 00120ca..421e7fb 100644 --- a/sabnzbd/utils/pystone.py +++ b/sabnzbd/utils/pystone.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python3 +#!/usr/bin/python33 -OO """ "PYSTONE" Benchmark Program @@ -34,7 +34,7 @@ Version History: LOOPS = 50000 -from time import clock +from time import perf_counter __version__ = "1.1" @@ -86,10 +86,10 @@ def Proc0(loops=LOOPS): global PtrGlb global PtrGlbNext - starttime = clock() + starttime = perf_counter() for i in range(loops): pass - nulltime = clock() - starttime + nulltime = perf_counter() - starttime PtrGlbNext = Record() PtrGlb = Record() @@ -101,7 +101,7 @@ def Proc0(loops=LOOPS): String1Loc = "DHRYSTONE PROGRAM, 1'ST STRING" Array2Glob[8][7] = 10 - starttime = clock() + starttime = perf_counter() for i in range(loops): Proc5() @@ -127,7 +127,7 @@ def Proc0(loops=LOOPS): IntLoc2 = 7 * (IntLoc3 - IntLoc2) - IntLoc1 IntLoc1 = Proc2(IntLoc1) - benchtime = clock() - starttime - nulltime + benchtime = perf_counter() - starttime - nulltime if benchtime == 0.0: loopsPerBenchtime = 0.0 else: diff --git a/sabnzbd/utils/servertests.py b/sabnzbd/utils/servertests.py index 7e0ae44..bee097e 100644 --- a/sabnzbd/utils/servertests.py +++ b/sabnzbd/utils/servertests.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/sabnzbd/utils/upload.py b/sabnzbd/utils/upload.py index ca1eded..cbc83e1 100644 --- a/sabnzbd/utils/upload.py +++ b/sabnzbd/utils/upload.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2009-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -28,6 +28,7 @@ from sabnzbd.filesystem import get_ext, get_filename from sabnzbd.constants import VALID_ARCHIVES, VALID_NZB_FILES from sabnzbd.dirscanner import ProcessArchiveFile, ProcessSingleFile +from sabnzbd.misc import get_from_url def upload_file(url, fp): diff --git a/sabnzbd/zconfig.py b/sabnzbd/zconfig.py index 51b07b7..4ca5921 100644 --- a/sabnzbd/zconfig.py +++ b/sabnzbd/zconfig.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/scripts/Deobfuscate.py b/scripts/Deobfuscate.py index 13c0800..ac90837 100644 --- a/scripts/Deobfuscate.py +++ b/scripts/Deobfuscate.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -113,17 +113,17 @@ def decodePar(parfile): # file already exists, skip it if path.exists(targetPath): - print "File already exists: " + targetName + print("File already exists: " + targetName) continue # find and rename file srcPath = findFile(dir, filelength, hash16k) if srcPath is not None: os.rename(srcPath, targetPath) - print "Renamed file from " + path.basename(srcPath) + " to " + targetName + print("Renamed file from " + path.basename(srcPath) + " to " + targetName) result = True else: - print "No match found for: " + targetName + print("No match found for: " + targetName) return result diff --git a/scripts/Sample-PostProc.py b/scripts/Sample-PostProc.py index 2f4a678..eb5eaf8 100755 --- a/scripts/Sample-PostProc.py +++ b/scripts/Sample-PostProc.py @@ -10,35 +10,35 @@ import sys, os # Raw parsing of input parameters en SABnzbd environment variables counter = 0 -print "\nINPUT from argv:\n" +print("\nINPUT from argv:\n") for item in sys.argv: - print "Argument", counter, ":", item + print("Argument", counter, ":", item) counter += 1 -print "\nINPUT from environment variables (only SAB specifics):\n" +print("\nINPUT from environment variables (only SAB specifics):\n") for item in os.environ: if item.find("SAB_") == 0: - print item, os.environ[item] + print(item, os.environ[item]) # More intelligent parsing: try: (scriptname,directory,orgnzbname,jobname,reportnumber,category,group,postprocstatus,url) = sys.argv except: - print "No SAB compliant number of commandline parameters found (should be 8):", len(sys.argv)-1 + print("No SAB compliant number of commandline parameters found (should be 8):", len(sys.argv)-1) sys.exit(1) # non-zero return code # Some examples: -print "\nExamples of some specific values:\n" -print "jobname is", jobname +print("\nExamples of some specific values:\n") +print("jobname is", jobname) try: sabversion = os.environ['SAB_VERSION'] - print "sabversion is", sabversion + print("sabversion is", sabversion) except: pass ''' your code here ''' # We're done: -print "\nScript done. All OK." # the last line will appear in the SABnzb History GUI +print("\nScript done. All OK.") # the last line will appear in the SABnzb History GUI sys.exit(0) # The result code towards SABnzbd diff --git a/tests/test_functional.py b/tests/test_functional.py index 8a82932..3a2b8b3 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -93,7 +93,7 @@ class SABnzbdDownloadFlow(SABnzbdBaseTest): self.add_nzb_from_url("http://sabnzbd.org/tests/basic_rar5.nzb", "testfile.bin") # Unicode test - self.add_nzb_from_url("http://sabnzbd.org/tests/unicode_rar.nzb", u"\u4f60\u597d\u4e16\u754c.bin") + self.add_nzb_from_url("http://sabnzbd.org/tests/unicode_rar.nzb", "\u4f60\u597d\u4e16\u754c.bin") # Unicode test with a missing article #self.add_nzb_from_url("http://sabnzbd.org/tests/unicode_rar_broken.nzb", u"\u4f60\u597d\u4e16\u754c.bin") diff --git a/tests/testhelper.py b/tests/testhelper.py index 8d0940f..b41ab50 100644 --- a/tests/testhelper.py +++ b/tests/testhelper.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -99,5 +99,5 @@ def tearDownModule(): shutil.rmtree(SAB_CACHE_DIR) break except OSError: - print "Unable to remove cache dir (try %d)" % x + print("Unable to remove cache dir (try %d)" % x) time.sleep(1) diff --git a/tools/extract_pot.py b/tools/extract_pot.py index 9b9ce9b..6f5310b 100755 --- a/tools/extract_pot.py +++ b/tools/extract_pot.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2011-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or @@ -27,7 +27,7 @@ import re f = open('sabnzbd/version.py') code = f.read() f.close() -exec code +exec(code) # Fixed information for the POT header HEADER = r'''# diff --git a/tools/make_mo.py b/tools/make_mo.py index f28c041..0cd1f2f 100755 --- a/tools/make_mo.py +++ b/tools/make_mo.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # -*- coding: utf-8 -*- # Copyright 2010-2018 The SABnzbd-Team # diff --git a/util/apireg.py b/util/apireg.py index 05b6017..8a79767 100644 --- a/util/apireg.py +++ b/util/apireg.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2012-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or diff --git a/util/mailslot.py b/util/mailslot.py index 2df389b..64d5145 100644 --- a/util/mailslot.py +++ b/util/mailslot.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -OO +#!/usr/bin/python3 -OO # Copyright 2007-2018 The SABnzbd-Team # # This program is free software; you can redistribute it and/or