Fix timezone handling on Windows to correct timestamps related to file system and db episode management.

Fix loading timezone on Windows 7+.
Add fallback zoneinfo file.
Use tzwinlocal as fallback timezone on Windows.
Add epoch_start to network_timezones.
Fix issue with getting tzwinlocal on Windows (non English system language).
Use new totimestamp, from_timestamp.
Change aired modify time to to_file_timestamp.
Fix _cleaner.py for py3.
Change don't delete folder bs4/builder.

CHANGES.md

@@ -1,4 +1,9 @@
-### 0.21.19 (2019-03-08 15:45:00 UTC)
+### 0.21.20 (2019-03-11 18:35:00 UTC)
+
+* Fix timezone handling on Windows to correct timestamps related to file system and db episode management
+
+
+### 0.21.19 (2019-03-08 15:45:00 UTC)
 
 * Change update provider TL from v4/classic to V5
 * Fix webapi (add show) wrong error message if show is not at info source

_cleaner.py

@@ -46,8 +46,8 @@ if old_magic != magic_number:
 
 # skip cleaned005 as used during dev by testers
 cleanups = [
-    ['.cleaned006.tmp', ('lib', 'bs4', 'builder'), [
+    ['.cleaned006.tmp', ('lib', 'boto'), [
-        ('lib', 'boto'), ('lib', 'bs4', 'builder'), ('lib', 'growl'),
+        ('lib', 'boto'), ('lib', 'growl'),
         ('lib', 'hachoir', 'core'), ('lib', 'hachoir', 'field'), ('lib', 'hachoir', 'metadata'),
         ('lib', 'hachoir', 'parser', 'archive'), ('lib', 'hachoir', 'parser', 'audio'),
         ('lib', 'hachoir', 'parser', 'common'), ('lib', 'hachoir', 'parser', 'container'),
@@ -84,8 +84,8 @@ for cleaned_path, test_path, dir_list in cleanups:
                 except (BaseException, Exception):
                     pass
 
-        with open(cleaned_file, 'wb') as fp:
+        with io.open(cleaned_file, 'w+', encoding='utf-8') as fp:
-            fp.write('This file exists to prevent a rerun delete of *.pyc, *.pyo files')
+            fp.write(u'This file exists to prevent a rerun delete of *.pyc, *.pyo files')
             fp.flush()
             os.fsync(fp.fileno())
 
@@ -126,12 +126,12 @@ if not os.path.isfile(cleaned_file) or os.path.exists(test):
         swap_name = cleaned_file
         cleaned_file = danger_output
         danger_output = swap_name
-        msg = 'Failed (permissions?) to delete file(s). You must manually delete:\r\n%s' % '\r\n'.join(bad_files)
+        msg = u'Failed (permissions?) to delete file(s). You must manually delete:\r\n%s' % '\r\n'.join(bad_files)
         print(msg)
     else:
-        msg = 'This file exists to prevent a rerun delete of dead lib/html5lib files'
+        msg = u'This file exists to prevent a rerun delete of dead lib/html5lib files'
 
-    with open(cleaned_file, 'wb') as fp:
+    with io.open(cleaned_file, 'w+', encoding='utf-8') as fp:
         fp.write(msg)
         fp.flush()
         os.fsync(fp.fileno())

lib/dateutil/tz/win.py

@@ -9,6 +9,7 @@ Attempting to import this module on a non-Windows platform will raise an
 # This code was originally contributed by Jeffrey Harris.
 import datetime
 import struct
+import sys
 
 from six.moves import winreg
 from six import text_type
@@ -278,7 +279,8 @@ class tzwinlocal(tzwinbase):
             with winreg.OpenKey(handle, TZLOCALKEYNAME) as tzlocalkey:
                 keydict = valuestodict(tzlocalkey)
 
-            self._std_abbr = keydict["StandardName"]
+            # only windows 7+ has the "TimeZoneKeyName" reg key
+            self._std_abbr = keydict["TimeZoneKeyName" if (6, 1) <= sys.getwindowsversion()[:2] else "StandardName"]
             self._dst_abbr = keydict["DaylightName"]
 
             try:

lib/dateutil/zoneinfo/.gitignore

@@ -1 +0,0 @@
-*.tar.gz

lib/dateutil/zoneinfo/__init__.py

@@ -4,7 +4,7 @@ import json
 import os
 
 from tarfile import TarFile
-from pkgutil import get_data
+# from pkgutil import get_data
 from io import BytesIO
 
 from dateutil.tz import tzfile as _tzfile
@@ -27,7 +27,11 @@ class tzfile(_tzfile):
 def getzoneinfofile_stream():
     try:
         # return BytesIO(get_data(__name__, ZONEFILENAME))
-        with open(ek.ek(os.path.join, sickbeard.ZONEINFO_DIR, ZONEFILENAME), 'rb') as f:
+        zonefile = ek.ek(os.path.join, sickbeard.ZONEINFO_DIR, ZONEFILENAME)
+        if not ek.ek(os.path.isfile, zonefile):
+            warnings.warn('Falling back to included zoneinfo file')
+            zonefile = ek.ek(os.path.join, ek.ek(os.path.dirname, __file__), ZONEFILENAME)
+        with open(zonefile, 'rb') as f:
             return BytesIO(f.read())
     except IOError as e:  # TODO  switch to FileNotFoundError?
         warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))

sickbeard/helpers.py

@@ -1669,8 +1669,8 @@ def datetime_to_epoch(dt):
     """
     """ can raise an error with dates pre 1970-1-1 """
     if not isinstance(getattr(dt, 'tzinfo'), datetime.tzinfo):
-        from sickbeard.network_timezones import sb_timezone
+        from sickbeard.network_timezones import SG_TIMEZONE
-        dt = dt.replace(tzinfo=sb_timezone)
+        dt = dt.replace(tzinfo=SG_TIMEZONE)
     utc_naive = dt.replace(tzinfo=None) - dt.utcoffset()
     return int((utc_naive - datetime.datetime(1970, 1, 1)).total_seconds())
 

sickbeard/network_timezones.py

@@ -21,6 +21,7 @@ from os.path import basename, join, isfile
 import datetime
 import os
 import re
+import sys
 
 import sickbeard
 from . import db, helpers, logger
@@ -43,9 +44,10 @@ pm_regex = re.compile(r'(P[. ]? ?M)', flags=re.I)
 
 network_dict = None
 network_dupes = None
-last_failure = {'datetime': datetime.datetime.fromordinal(1), 'count': 0}
+last_failure = {'datetime': datetime.datetime.fromordinal(1), 'count': 0}  # type: dict
 max_retry_time = 900
 max_retry_count = 3
+is_win = 'win32' == sys.platform
 
 country_timezones = {
     'AU': 'Australia/Sydney', 'AR': 'America/Buenos_Aires', 'AUSTRALIA': 'Australia/Sydney', 'BR': 'America/Sao_Paulo',
@@ -56,18 +58,25 @@ country_timezones = {
     'PT': 'Europe/Lisbon', 'RU': 'Europe/Kaliningrad', 'SE': 'Europe/Stockholm', 'SG': 'Asia/Singapore',
     'TW': 'Asia/Taipei', 'UK': 'Europe/London', 'US': 'US/Eastern', 'ZA': 'Africa/Johannesburg'}
 
+EPOCH_START = None  # type: Optional[datetime.datetime]
+EPOCH_START_WIN = None  # type: Optional[datetime.datetime]
+SG_TIMEZONE = None  # type: Optional[datetime.tzinfo]
+
 
 def reset_last_retry():
+    # type: (...) -> None
     global last_failure
     last_failure = {'datetime': datetime.datetime.fromordinal(1), 'count': 0}
 
 
 def update_last_retry():
+    # type: (...) -> None
     global last_failure
     last_failure = {'datetime': datetime.datetime.now(), 'count': last_failure.get('count', 0) + 1}
 
 
 def should_try_loading():
+    # type: (...) -> bool
     global last_failure
     if last_failure.get('count', 0) >= max_retry_count and \
             (datetime.datetime.now() - last_failure.get('datetime',
@@ -77,10 +86,16 @@ def should_try_loading():
 
 
 def tz_fallback(t):
-    return t if isinstance(t, datetime.tzinfo) else tz.tzlocal()
+    # type: (...) -> datetime.tzinfo
+    if isinstance(t, datetime.tzinfo):
+        return t
+    if is_win:
+        return tz.tzwinlocal()
+    return tz.tzlocal()
 
 
 def get_tz():
+    # type: (...) -> datetime.tzinfo
     t = None
     try:
         t = get_localzone()
@@ -97,7 +112,36 @@ def get_tz():
     return t
 
 
-sb_timezone = get_tz()
+def get_utc():
+    # type: (...) -> Optional[datetime.tzinfo]
+    if hasattr(sickbeard, 'ZONEINFO_DIR'):
+        utc = None
+        try:
+            utc = tz.gettz('GMT', zoneinfo_priority=True)
+        except (BaseException, Exception):
+            pass
+        if isinstance(utc, datetime.tzinfo):
+            return utc
+    tz_utc_file = ek.ek(os.path.join, ek.ek(os.path.dirname, zoneinfo.__file__), 'Greenwich')
+    if ek.ek(os.path.isfile, tz_utc_file):
+        return tz.tzfile(tz_utc_file)
+    return None
+
+
+def set_vars():
+    # type: (...) -> None
+    global EPOCH_START, EPOCH_START_WIN, SG_TIMEZONE
+    SG_TIMEZONE = get_tz()
+    params = dict(year=1970, month=1, day=1)
+    EPOCH_START_WIN = EPOCH_START = datetime.datetime(tzinfo=get_utc(), **params)
+    if is_win:
+        try:
+            EPOCH_START_WIN = datetime.datetime(tzinfo=tz.win.tzwin('UTC'), **params)
+        except (BaseException, Exception):
+            pass
+
+
+set_vars()
 
 
 def _remove_zoneinfo_failed(filename):
@@ -125,7 +169,7 @@ def _remove_old_zoneinfo():
 
     for (path, dirs, files) in chain.from_iterable(
             [ek.ek(os.walk, helpers.real_path(_dir))
-             for _dir in (sickbeard.ZONEINFO_DIR, ek.ek(os.path.dirname, zoneinfo.__file__))]):
+             for _dir in (sickbeard.ZONEINFO_DIR, )]):
         for filename in files:
             if filename.endswith('.tar.gz'):
                 file_w_path = ek.ek(join, path, filename)
@@ -145,8 +189,7 @@ def _update_zoneinfo():
     if not should_try_loading():
         return
 
-    global sb_timezone
+    set_vars()
-    sb_timezone = get_tz()
 
     # now check if the zoneinfo needs update
     url_zv = 'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/zoneinfo.txt'
@@ -220,7 +263,7 @@ def _update_zoneinfo():
             except AttributeError:
                 pass
 
-            sb_timezone = get_tz()
+            set_vars()
         except (BaseException, Exception):
             _remove_zoneinfo_failed(zonefile_tmp)
             return
@@ -344,7 +387,7 @@ def get_network_timezone(network, return_name=False):
     :return: timezone info or tuple of timezone info, timezone name
     """
     if None is network:
-        return sb_timezone
+        return SG_TIMEZONE
 
     timezone = None
     timezone_name = None
@@ -369,9 +412,11 @@ def get_network_timezone(network, return_name=False):
     except (BaseException, Exception):
         pass
 
+    if isinstance(timezone, datetime.tzinfo):
+        return timezone
     if return_name:
-        return timezone if isinstance(timezone, datetime.tzinfo) else sb_timezone, timezone_name
+        return SG_TIMEZONE, timezone_name
-    return timezone if isinstance(timezone, datetime.tzinfo) else sb_timezone
+    return SG_TIMEZONE
 
 
 def parse_time(t):
@@ -437,7 +482,7 @@ def parse_date_time(d, t, network):
         foreign_naive = datetime.datetime(te.year, te.month, te.day, hr, m, tzinfo=foreign_timezone)
         return foreign_naive
     except (BaseException, Exception):
-        return datetime.datetime(te.year, te.month, te.day, hr, m, tzinfo=sb_timezone)
+        return datetime.datetime(te.year, te.month, te.day, hr, m, tzinfo=SG_TIMEZONE)
 
 
 def test_timeformat(t):

sickbeard/providers/newznab.py

@@ -32,7 +32,7 @@ from ..classes import SearchResult
 from ..common import NeededQualities, Quality
 from ..helpers import remove_non_release_groups, try_int
 from ..indexers.indexer_config import *
-from ..network_timezones import sb_timezone
+from ..network_timezones import SG_TIMEZONE
 from ..sgdatetime import SGDatetime
 from ..search import get_aired_in_season, get_wanted_qualities
 from ..show_name_helpers import get_show_names
@@ -781,7 +781,7 @@ class NewznabProvider(generic.NZBProvider):
             if p:
                 p = parser.parse(p, fuzzy=True)
                 try:
-                    p = p.astimezone(sb_timezone)
+                    p = p.astimezone(SG_TIMEZONE)
                 except (BaseException, Exception):
                     pass
                 if isinstance(p, datetime.datetime):

sickbeard/search_queue.py

@@ -360,7 +360,7 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
         else:
             cur_date = (datetime.date.today() - datetime.timedelta(days=2)).toordinal()
 
-        cur_time = datetime.datetime.now(network_timezones.sb_timezone)
+        cur_time = datetime.datetime.now(network_timezones.SG_TIMEZONE)
 
         my_db = db.DBConnection()
         sql_result = my_db.select(

sickbeard/sgdatetime.py

@@ -20,12 +20,12 @@ import datetime
 import functools
 import locale
 import re
-import time
+import sys
 
 import sickbeard
-from .network_timezones import sb_timezone
+from dateutil import tz
 
-from six import string_types
+from six import integer_types, string_types
 
 # noinspection PyUnreachableCode
 if False:
@@ -96,6 +96,8 @@ time_presets = ('%I:%M:%S %p',
                 '%I:%M:%S %P',
                 '%H:%M:%S')
 
+is_win = 'win32' == sys.platform
+
 
 # helper decorator class
 # noinspection PyPep8Naming
@@ -122,10 +124,11 @@ class SGDatetime(datetime.datetime):
     @static_or_instance
     def convert_to_setting(self, dt=None, force_local=False):
         # type: (Optional[datetime.datetime, SGDatetime], bool) -> Union[SGDatetime, datetime.datetime]
-        obj = (dt, self)[self is not None]
+        obj = (dt, self)[self is not None]  # type: datetime.datetime
         try:
             if force_local or 'local' == sickbeard.TIMEZONE_DISPLAY:
-                return obj.astimezone(sb_timezone)
+                from sickbeard.network_timezones import SG_TIMEZONE
+                return obj.astimezone(SG_TIMEZONE)
         except (BaseException, Exception):
             pass
 
@@ -150,7 +153,7 @@ class SGDatetime(datetime.datetime):
 
         strt = ''
 
-        obj = (dt, self)[self is not None]
+        obj = (dt, self)[self is not None]  # type: datetime.datetime
         if None is not obj:
             tmpl = (((sickbeard.TIME_PRESET, sickbeard.TIME_PRESET_W_SECONDS)[show_seconds]),
                     t_preset)[None is not t_preset]
@@ -192,7 +195,7 @@ class SGDatetime(datetime.datetime):
 
         strd = ''
         try:
-            obj = (dt, self)[self is not None]
+            obj = (dt, self)[self is not None]  # type: datetime.datetime
             if None is not obj:
                 strd = SGDatetime.sbstrftime(obj, (sickbeard.DATE_PRESET, d_preset)[None is not d_preset])
 
@@ -207,7 +210,7 @@ class SGDatetime(datetime.datetime):
         SGDatetime.setlocale()
 
         strd = ''
-        obj = (dt, self)[self is not None]
+        obj = (dt, self)[self is not None]  # type: datetime.datetime
         try:
             if None is not obj:
                 strd = u'%s, %s' % (
@@ -229,10 +232,43 @@ class SGDatetime(datetime.datetime):
 
     @static_or_instance
     def totimestamp(self, dt=None, default=None):
-        # type: (Optional[SGDatetime, datetime.datetime], Optional[float]) -> float
+        # type: (Optional[Union[SGDatetime, datetime.datetime]], Optional[float]) -> Union[float, integer_types]
-        obj = (dt, self)[self is not None]
+        """ Calculate timestamp from datetime.datetime obj
+        """
+        obj = (dt, self)[self is not None]  # type: datetime.datetime
+        if not isinstance(getattr(obj, 'tzinfo', None), datetime.tzinfo):
+            from sickbeard.network_timezones import SG_TIMEZONE
+            obj = obj.replace(tzinfo=SG_TIMEZONE)
+        from .network_timezones import EPOCH_START
         timestamp = default
         try:
-            timestamp = time.mktime(obj.timetuple())
+            timestamp = (obj - EPOCH_START).total_seconds()
         finally:
-            return (default, timestamp)[isinstance(timestamp, float)]
+            return (default, timestamp)[isinstance(timestamp, (float, integer_types))]
+
+    @staticmethod
+    def from_timestamp(ts, local_time=True):
+        # type: (Union[integer_types, float], bool) -> datetime.datetime
+        """
+        convert timestamp to datetime.datetime obj
+        :param ts: timestamp integer, float
+        :param local_time: return as local timezone (SG_TIMEZONE)
+        """
+        from .network_timezones import EPOCH_START, SG_TIMEZONE
+        if local_time and SG_TIMEZONE:
+            return (EPOCH_START + datetime.timedelta(seconds=ts)).astimezone(SG_TIMEZONE)
+        return EPOCH_START + datetime.timedelta(seconds=ts)
+
+    @static_or_instance
+    def to_file_timestamp(self, dt=None):
+        # type: (Optional[SGDatetime, datetime.datetime]) -> Union[float, integer_types]
+        """
+        convert datetime to filetime
+        special handling for windows filetime issues
+        for pre Windows 7 this can result in an exception for pre 1970 dates
+        """
+        obj = (dt, self)[self is not None]  # type: datetime.datetime
+        if is_win:
+            from .network_timezones import EPOCH_START_WIN
+            return (obj.replace(tzinfo=tz.tzwinlocal()) - EPOCH_START_WIN).total_seconds()
+        return self.totimestamp(obj)

sickbeard/tv.py

@@ -2451,7 +2451,7 @@ class TVEpisode(TVEpisodeBase):
                                                       self.show_obj.airs,
                                                       self.show_obj.network)
         show_length = datetime.timedelta(minutes=helpers.try_int(self.show_obj.runtime, 60))
-        tz_now = datetime.datetime.now(network_timezones.sb_timezone)
+        tz_now = datetime.datetime.now(network_timezones.SG_TIMEZONE)
         future_airtime = (self.airdate > (today + delta) or
                           (not self.airdate < (today - delta) and ((show_time + show_length) > tz_now)))
 
@@ -3313,7 +3313,8 @@ class TVEpisode(TVEpisodeBase):
 
         aired_dt = datetime.datetime.combine(self.airdate, airtime)
         try:
-            aired_epoch = helpers.datetime_to_epoch(aired_dt)
+            # aired_epoch = helpers.datetime_to_epoch(aired_dt)
+            aired_epoch = SGDatetime.to_file_timestamp(aired_dt)
             filemtime = int(ek.ek(os.path.getmtime, self.location))
         except (BaseException, Exception):
             return

sickbeard/webserve.py

@@ -1087,8 +1087,8 @@ class MainHandler(WebHandler):
 
         sql_result.sort(key=sorts[sickbeard.EPISODE_VIEW_SORT])
 
-        t.next_week = datetime.datetime.combine(next_week_dt, datetime.time(tzinfo=network_timezones.sb_timezone))
+        t.next_week = datetime.datetime.combine(next_week_dt, datetime.time(tzinfo=network_timezones.SG_TIMEZONE))
-        t.today = datetime.datetime.now(network_timezones.sb_timezone)
+        t.today = datetime.datetime.now(network_timezones.SG_TIMEZONE)
         t.sql_results = sql_result
 
         return t.respond()

tests/network_timezone_tests.py

@@ -43,7 +43,7 @@ class NetworkTimezoneTests(test.SickbeardTestDBCase):
 
     def test_timezone(self):
         network_timezones.update_network_dict()
-        network_timezones.sb_timezone = tz.gettz('CET', zoneinfo_priority=True)
+        network_timezones.SG_TIMEZONE = tz.gettz('CET', zoneinfo_priority=True)
         d = datetime.date(2018, 9, 2).toordinal()
         t = 'Monday 9:00 PM'
         network = 'NBC'

tests/newznab_tests.py

@@ -18,7 +18,7 @@ sys.path.insert(1, os.path.abspath('../lib'))
 from lxml_etree import etree
 from lib.dateutil import parser
 from sickbeard.indexers.indexer_config import *
-from sickbeard.network_timezones import sb_timezone
+from sickbeard.network_timezones import SG_TIMEZONE
 from sickbeard.providers import newznab
 from six import iteritems, iterkeys
 
@@ -298,7 +298,7 @@ class BasicTests(test.SickbeardTestDBCase):
             if date_str:
                 p = parser.parse(date_str, fuzzy=True)
                 try:
-                    p = p.astimezone(sb_timezone)
+                    p = p.astimezone(SG_TIMEZONE)
                 except (BaseException, Exception):
                     pass
                 if isinstance(p, datetime.datetime):