diff --git a/CHANGES.md b/CHANGES.md index 025f2c4..6286739 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,6 +31,11 @@ * Update Tornado_py3 Web Server 6.0.3 (ff985fe) to 6.1.dev1 (18b653c) +### 0.21.20 (2019-03-11 18:35:00 UTC) + +* Fix timezone handling on Windows to correct timestamps related to file system and db episode management + + ### 0.21.19 (2019-03-08 15:45:00 UTC) * Change update provider TL from v4/classic to V5 diff --git a/_cleaner.py b/_cleaner.py index 0e363fa..4aaa1de 100644 --- a/_cleaner.py +++ b/_cleaner.py @@ -46,8 +46,8 @@ if old_magic != magic_number: # skip cleaned005 as used during dev by testers cleanups = [ - ['.cleaned006.tmp', ('lib', 'bs4', 'builder'), [ - ('lib', 'boto'), ('lib', 'bs4', 'builder'), ('lib', 'growl'), + ['.cleaned006.tmp', ('lib', 'boto'), [ + ('lib', 'boto'), ('lib', 'growl'), ('lib', 'hachoir', 'core'), ('lib', 'hachoir', 'field'), ('lib', 'hachoir', 'metadata'), ('lib', 'hachoir', 'parser', 'archive'), ('lib', 'hachoir', 'parser', 'audio'), ('lib', 'hachoir', 'parser', 'common'), ('lib', 'hachoir', 'parser', 'container'), @@ -84,8 +84,8 @@ for cleaned_path, test_path, dir_list in cleanups: except (BaseException, Exception): pass - with open(cleaned_file, 'wb') as fp: - fp.write('This file exists to prevent a rerun delete of *.pyc, *.pyo files') + with io.open(cleaned_file, 'w+', encoding='utf-8') as fp: + fp.write(u'This file exists to prevent a rerun delete of *.pyc, *.pyo files') fp.flush() os.fsync(fp.fileno()) @@ -126,12 +126,12 @@ if not os.path.isfile(cleaned_file) or os.path.exists(test): swap_name = cleaned_file cleaned_file = danger_output danger_output = swap_name - msg = 'Failed (permissions?) to delete file(s). You must manually delete:\r\n%s' % '\r\n'.join(bad_files) + msg = u'Failed (permissions?) to delete file(s). You must manually delete:\r\n%s' % '\r\n'.join(bad_files) print(msg) else: - msg = 'This file exists to prevent a rerun delete of dead lib/html5lib files' + msg = u'This file exists to prevent a rerun delete of dead lib/html5lib files' - with open(cleaned_file, 'wb') as fp: + with io.open(cleaned_file, 'w+', encoding='utf-8') as fp: fp.write(msg) fp.flush() os.fsync(fp.fileno()) diff --git a/lib/dateutil/tz/win.py b/lib/dateutil/tz/win.py index 52a65b7..6d6c004 100644 --- a/lib/dateutil/tz/win.py +++ b/lib/dateutil/tz/win.py @@ -9,6 +9,7 @@ Attempting to import this module on a non-Windows platform will raise an # This code was originally contributed by Jeffrey Harris. import datetime import struct +import sys from six.moves import winreg from six import text_type @@ -278,7 +279,8 @@ class tzwinlocal(tzwinbase): with winreg.OpenKey(handle, TZLOCALKEYNAME) as tzlocalkey: keydict = valuestodict(tzlocalkey) - self._std_abbr = keydict["StandardName"] + # only windows 7+ has the "TimeZoneKeyName" reg key + self._std_abbr = keydict["TimeZoneKeyName" if (6, 1) <= sys.getwindowsversion()[:2] else "StandardName"] self._dst_abbr = keydict["DaylightName"] try: diff --git a/lib/dateutil/zoneinfo/.gitignore b/lib/dateutil/zoneinfo/.gitignore deleted file mode 100644 index 335ec95..0000000 --- a/lib/dateutil/zoneinfo/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.tar.gz diff --git a/lib/dateutil/zoneinfo/Greenwich b/lib/dateutil/zoneinfo/Greenwich new file mode 100644 index 0000000..c05e45f Binary files /dev/null and b/lib/dateutil/zoneinfo/Greenwich differ diff --git a/lib/dateutil/zoneinfo/__init__.py b/lib/dateutil/zoneinfo/__init__.py index 6d43031..8607269 100644 --- a/lib/dateutil/zoneinfo/__init__.py +++ b/lib/dateutil/zoneinfo/__init__.py @@ -4,7 +4,7 @@ import json import os from tarfile import TarFile -from pkgutil import get_data +# from pkgutil import get_data from io import BytesIO from dateutil.tz import tzfile as _tzfile @@ -27,7 +27,11 @@ class tzfile(_tzfile): def getzoneinfofile_stream(): try: # return BytesIO(get_data(__name__, ZONEFILENAME)) - with open(ek.ek(os.path.join, sickbeard.ZONEINFO_DIR, ZONEFILENAME), 'rb') as f: + zonefile = ek.ek(os.path.join, sickbeard.ZONEINFO_DIR, ZONEFILENAME) + if not ek.ek(os.path.isfile, zonefile): + warnings.warn('Falling back to included zoneinfo file') + zonefile = ek.ek(os.path.join, ek.ek(os.path.dirname, __file__), ZONEFILENAME) + with open(zonefile, 'rb') as f: return BytesIO(f.read()) except IOError as e: # TODO switch to FileNotFoundError? warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror)) diff --git a/lib/dateutil/zoneinfo/dateutil-zoneinfo.tar.gz b/lib/dateutil/zoneinfo/dateutil-zoneinfo.tar.gz new file mode 100644 index 0000000..5e8c11a Binary files /dev/null and b/lib/dateutil/zoneinfo/dateutil-zoneinfo.tar.gz differ diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index 23eb44f..e8de0cb 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -1659,8 +1659,8 @@ def datetime_to_epoch(dt): """ """ can raise an error with dates pre 1970-1-1 """ if not isinstance(getattr(dt, 'tzinfo'), datetime.tzinfo): - from sickbeard.network_timezones import sb_timezone - dt = dt.replace(tzinfo=sb_timezone) + from sickbeard.network_timezones import SG_TIMEZONE + dt = dt.replace(tzinfo=SG_TIMEZONE) utc_naive = dt.replace(tzinfo=None) - dt.utcoffset() return int((utc_naive - datetime.datetime(1970, 1, 1)).total_seconds()) diff --git a/sickbeard/network_timezones.py b/sickbeard/network_timezones.py index eb9de0e..18d5d9e 100644 --- a/sickbeard/network_timezones.py +++ b/sickbeard/network_timezones.py @@ -21,6 +21,7 @@ from os.path import basename, join, isfile import datetime import os import re +import sys import sickbeard from . import db, helpers, logger @@ -45,9 +46,10 @@ pm_regex = re.compile(r'(P[. ]? ?M)', flags=re.I) network_dict = None network_dupes = None -last_failure = {'datetime': datetime.datetime.fromordinal(1), 'count': 0} +last_failure = {'datetime': datetime.datetime.fromordinal(1), 'count': 0} # type: dict max_retry_time = 900 max_retry_count = 3 +is_win = 'win32' == sys.platform country_timezones = { 'AU': 'Australia/Sydney', 'AR': 'America/Buenos_Aires', 'AUSTRALIA': 'Australia/Sydney', 'BR': 'America/Sao_Paulo', @@ -58,18 +60,25 @@ country_timezones = { 'PT': 'Europe/Lisbon', 'RU': 'Europe/Kaliningrad', 'SE': 'Europe/Stockholm', 'SG': 'Asia/Singapore', 'TW': 'Asia/Taipei', 'UK': 'Europe/London', 'US': 'US/Eastern', 'ZA': 'Africa/Johannesburg'} +EPOCH_START = None # type: Optional[datetime.datetime] +EPOCH_START_WIN = None # type: Optional[datetime.datetime] +SG_TIMEZONE = None # type: Optional[datetime.tzinfo] + def reset_last_retry(): + # type: (...) -> None global last_failure last_failure = {'datetime': datetime.datetime.fromordinal(1), 'count': 0} def update_last_retry(): + # type: (...) -> None global last_failure last_failure = {'datetime': datetime.datetime.now(), 'count': last_failure.get('count', 0) + 1} def should_try_loading(): + # type: (...) -> bool global last_failure if last_failure.get('count', 0) >= max_retry_count and \ (datetime.datetime.now() - last_failure.get('datetime', @@ -79,10 +88,16 @@ def should_try_loading(): def tz_fallback(t): - return t if isinstance(t, datetime.tzinfo) else tz.tzlocal() + # type: (...) -> datetime.tzinfo + if isinstance(t, datetime.tzinfo): + return t + if is_win: + return tz.tzwinlocal() + return tz.tzlocal() def get_tz(): + # type: (...) -> datetime.tzinfo t = None try: t = get_localzone() @@ -99,7 +114,36 @@ def get_tz(): return t -sb_timezone = get_tz() +def get_utc(): + # type: (...) -> Optional[datetime.tzinfo] + if hasattr(sickbeard, 'ZONEINFO_DIR'): + utc = None + try: + utc = tz.gettz('GMT', zoneinfo_priority=True) + except (BaseException, Exception): + pass + if isinstance(utc, datetime.tzinfo): + return utc + tz_utc_file = ek.ek(os.path.join, ek.ek(os.path.dirname, zoneinfo.__file__), 'Greenwich') + if ek.ek(os.path.isfile, tz_utc_file): + return tz.tzfile(tz_utc_file) + return None + + +def set_vars(): + # type: (...) -> None + global EPOCH_START, EPOCH_START_WIN, SG_TIMEZONE + SG_TIMEZONE = get_tz() + params = dict(year=1970, month=1, day=1) + EPOCH_START_WIN = EPOCH_START = datetime.datetime(tzinfo=get_utc(), **params) + if is_win: + try: + EPOCH_START_WIN = datetime.datetime(tzinfo=tz.win.tzwin('UTC'), **params) + except (BaseException, Exception): + pass + + +set_vars() def _remove_zoneinfo_failed(filename): @@ -127,7 +171,7 @@ def _remove_old_zoneinfo(): for entry in chain.from_iterable( [ek.ek(scandir, helpers.real_path(_dir)) - for _dir in (sickbeard.ZONEINFO_DIR, ek.ek(os.path.dirname, zoneinfo.__file__))]): + for _dir in (sickbeard.ZONEINFO_DIR, )]): if entry.is_file(follow_symlinks=False): if entry.name.endswith('.tar.gz'): if entry.path != cur_file: @@ -146,8 +190,7 @@ def _update_zoneinfo(): if not should_try_loading(): return - global sb_timezone - sb_timezone = get_tz() + set_vars() # now check if the zoneinfo needs update url_zv = 'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/zoneinfo.txt' @@ -221,7 +264,7 @@ def _update_zoneinfo(): except AttributeError: pass - sb_timezone = get_tz() + set_vars() except (BaseException, Exception): _remove_zoneinfo_failed(zonefile_tmp) return @@ -345,7 +388,7 @@ def get_network_timezone(network, return_name=False): :return: timezone info or tuple of timezone info, timezone name """ if None is network: - return sb_timezone + return SG_TIMEZONE timezone = None timezone_name = None @@ -370,9 +413,11 @@ def get_network_timezone(network, return_name=False): except (BaseException, Exception): pass + if isinstance(timezone, datetime.tzinfo): + return timezone if return_name: - return timezone if isinstance(timezone, datetime.tzinfo) else sb_timezone, timezone_name - return timezone if isinstance(timezone, datetime.tzinfo) else sb_timezone + return SG_TIMEZONE, timezone_name + return SG_TIMEZONE def parse_time(t): @@ -438,7 +483,7 @@ def parse_date_time(d, t, network): foreign_naive = datetime.datetime(te.year, te.month, te.day, hr, m, tzinfo=foreign_timezone) return foreign_naive except (BaseException, Exception): - return datetime.datetime(te.year, te.month, te.day, hr, m, tzinfo=sb_timezone) + return datetime.datetime(te.year, te.month, te.day, hr, m, tzinfo=SG_TIMEZONE) def test_timeformat(t): diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py index 7199831..d5f9c5c 100755 --- a/sickbeard/providers/newznab.py +++ b/sickbeard/providers/newznab.py @@ -32,7 +32,7 @@ from ..classes import SearchResult from ..common import NeededQualities, Quality from ..helpers import remove_non_release_groups, try_int from ..indexers.indexer_config import * -from ..network_timezones import sb_timezone +from ..network_timezones import SG_TIMEZONE from ..sgdatetime import SGDatetime from ..search import get_aired_in_season, get_wanted_qualities from ..show_name_helpers import get_show_names @@ -781,7 +781,7 @@ class NewznabProvider(generic.NZBProvider): if p: p = parser.parse(p, fuzzy=True) try: - p = p.astimezone(sb_timezone) + p = p.astimezone(SG_TIMEZONE) except (BaseException, Exception): pass if isinstance(p, datetime.datetime): diff --git a/sickbeard/search_queue.py b/sickbeard/search_queue.py index 1135c63..2ec381d 100644 --- a/sickbeard/search_queue.py +++ b/sickbeard/search_queue.py @@ -360,7 +360,7 @@ class RecentSearchQueueItem(generic_queue.QueueItem): else: cur_date = (datetime.date.today() - datetime.timedelta(days=2)).toordinal() - cur_time = datetime.datetime.now(network_timezones.sb_timezone) + cur_time = datetime.datetime.now(network_timezones.SG_TIMEZONE) my_db = db.DBConnection() sql_result = my_db.select( diff --git a/sickbeard/sgdatetime.py b/sickbeard/sgdatetime.py index 4b0438a..3288cf8 100644 --- a/sickbeard/sgdatetime.py +++ b/sickbeard/sgdatetime.py @@ -20,12 +20,12 @@ import datetime import functools import locale import re -import time +import sys import sickbeard -from .network_timezones import sb_timezone +from dateutil import tz -from six import string_types +from six import integer_types, string_types # noinspection PyUnreachableCode if False: @@ -96,6 +96,8 @@ time_presets = ('%I:%M:%S %p', '%I:%M:%S %P', '%H:%M:%S') +is_win = 'win32' == sys.platform + # helper decorator class # noinspection PyPep8Naming @@ -122,10 +124,11 @@ class SGDatetime(datetime.datetime): @static_or_instance def convert_to_setting(self, dt=None, force_local=False): # type: (Optional[datetime.datetime, SGDatetime], bool) -> Union[SGDatetime, datetime.datetime] - obj = (dt, self)[self is not None] + obj = (dt, self)[self is not None] # type: datetime.datetime try: if force_local or 'local' == sickbeard.TIMEZONE_DISPLAY: - return obj.astimezone(sb_timezone) + from sickbeard.network_timezones import SG_TIMEZONE + return obj.astimezone(SG_TIMEZONE) except (BaseException, Exception): pass @@ -150,7 +153,7 @@ class SGDatetime(datetime.datetime): strt = '' - obj = (dt, self)[self is not None] + obj = (dt, self)[self is not None] # type: datetime.datetime if None is not obj: tmpl = (((sickbeard.TIME_PRESET, sickbeard.TIME_PRESET_W_SECONDS)[show_seconds]), t_preset)[None is not t_preset] @@ -192,7 +195,7 @@ class SGDatetime(datetime.datetime): strd = '' try: - obj = (dt, self)[self is not None] + obj = (dt, self)[self is not None] # type: datetime.datetime if None is not obj: strd = SGDatetime.sbstrftime(obj, (sickbeard.DATE_PRESET, d_preset)[None is not d_preset]) @@ -207,7 +210,7 @@ class SGDatetime(datetime.datetime): SGDatetime.setlocale() strd = '' - obj = (dt, self)[self is not None] + obj = (dt, self)[self is not None] # type: datetime.datetime try: if None is not obj: strd = u'%s, %s' % ( @@ -229,10 +232,43 @@ class SGDatetime(datetime.datetime): @static_or_instance def totimestamp(self, dt=None, default=None): - # type: (Optional[SGDatetime, datetime.datetime], Optional[float]) -> float - obj = (dt, self)[self is not None] + # type: (Optional[Union[SGDatetime, datetime.datetime]], Optional[float]) -> Union[float, integer_types] + """ Calculate timestamp from datetime.datetime obj + """ + obj = (dt, self)[self is not None] # type: datetime.datetime + if not isinstance(getattr(obj, 'tzinfo', None), datetime.tzinfo): + from sickbeard.network_timezones import SG_TIMEZONE + obj = obj.replace(tzinfo=SG_TIMEZONE) + from .network_timezones import EPOCH_START timestamp = default try: - timestamp = time.mktime(obj.timetuple()) + timestamp = (obj - EPOCH_START).total_seconds() finally: - return (default, timestamp)[isinstance(timestamp, float)] + return (default, timestamp)[isinstance(timestamp, (float, integer_types))] + + @staticmethod + def from_timestamp(ts, local_time=True): + # type: (Union[integer_types, float], bool) -> datetime.datetime + """ + convert timestamp to datetime.datetime obj + :param ts: timestamp integer, float + :param local_time: return as local timezone (SG_TIMEZONE) + """ + from .network_timezones import EPOCH_START, SG_TIMEZONE + if local_time and SG_TIMEZONE: + return (EPOCH_START + datetime.timedelta(seconds=ts)).astimezone(SG_TIMEZONE) + return EPOCH_START + datetime.timedelta(seconds=ts) + + @static_or_instance + def to_file_timestamp(self, dt=None): + # type: (Optional[SGDatetime, datetime.datetime]) -> Union[float, integer_types] + """ + convert datetime to filetime + special handling for windows filetime issues + for pre Windows 7 this can result in an exception for pre 1970 dates + """ + obj = (dt, self)[self is not None] # type: datetime.datetime + if is_win: + from .network_timezones import EPOCH_START_WIN + return (obj.replace(tzinfo=tz.tzwinlocal()) - EPOCH_START_WIN).total_seconds() + return self.totimestamp(obj) diff --git a/sickbeard/tv.py b/sickbeard/tv.py index 8ddfe55..54d4363 100644 --- a/sickbeard/tv.py +++ b/sickbeard/tv.py @@ -2471,7 +2471,7 @@ class TVEpisode(TVEpisodeBase): self.show_obj.airs, self.show_obj.network) show_length = datetime.timedelta(minutes=helpers.try_int(self.show_obj.runtime, 60)) - tz_now = datetime.datetime.now(network_timezones.sb_timezone) + tz_now = datetime.datetime.now(network_timezones.SG_TIMEZONE) future_airtime = (self.airdate > (today + delta) or (not self.airdate < (today - delta) and ((show_time + show_length) > tz_now))) @@ -3333,7 +3333,8 @@ class TVEpisode(TVEpisodeBase): aired_dt = datetime.datetime.combine(self.airdate, airtime) try: - aired_epoch = helpers.datetime_to_epoch(aired_dt) + # aired_epoch = helpers.datetime_to_epoch(aired_dt) + aired_epoch = SGDatetime.to_file_timestamp(aired_dt) filemtime = int(ek.ek(os.path.getmtime, self.location)) except (BaseException, Exception): return diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 703791d..c38ec99 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -1088,8 +1088,8 @@ class MainHandler(WebHandler): sql_result.sort(key=sorts[sickbeard.EPISODE_VIEW_SORT]) - t.next_week = datetime.datetime.combine(next_week_dt, datetime.time(tzinfo=network_timezones.sb_timezone)) - t.today = datetime.datetime.now(network_timezones.sb_timezone) + t.next_week = datetime.datetime.combine(next_week_dt, datetime.time(tzinfo=network_timezones.SG_TIMEZONE)) + t.today = datetime.datetime.now(network_timezones.SG_TIMEZONE) t.sql_results = sql_result return t.respond() diff --git a/tests/network_timezone_tests.py b/tests/network_timezone_tests.py index 70566e0..a9e4899 100644 --- a/tests/network_timezone_tests.py +++ b/tests/network_timezone_tests.py @@ -44,7 +44,7 @@ class NetworkTimezoneTests(test.SickbeardTestDBCase): def test_timezone(self): network_timezones.update_network_dict() - network_timezones.sb_timezone = tz.gettz('CET', zoneinfo_priority=True) + network_timezones.SG_TIMEZONE = tz.gettz('CET', zoneinfo_priority=True) d = datetime.date(2018, 9, 2).toordinal() t = 'Monday 9:00 PM' network = 'NBC' diff --git a/tests/newznab_tests.py b/tests/newznab_tests.py index 059ecf8..687d6e8 100644 --- a/tests/newznab_tests.py +++ b/tests/newznab_tests.py @@ -18,7 +18,7 @@ sys.path.insert(1, os.path.abspath('../lib')) from lxml_etree import etree from lib.dateutil import parser from sickbeard.indexers.indexer_config import * -from sickbeard.network_timezones import sb_timezone +from sickbeard.network_timezones import SG_TIMEZONE from sickbeard.providers import newznab from six import iteritems, iterkeys @@ -298,7 +298,7 @@ class BasicTests(test.SickbeardTestDBCase): if date_str: p = parser.parse(date_str, fuzzy=True) try: - p = p.astimezone(sb_timezone) + p = p.astimezone(SG_TIMEZONE) except (BaseException, Exception): pass if isinstance(p, datetime.datetime):