@ -19,25 +19,33 @@
from collections import defaultdict from collections import defaultdict
import datetime import datetime
import io
import os
import re import re
import sys
import threading import threading
import time
import traceback import traceback
import sickbeard import sickbeard
# noinspection PyPep8Naming
import encodingKludge as ek
from exceptions_helper import ex
from . import db , helpers , logger , name_cache from . import db , helpers , logger , name_cache
from . anime import create_anidb_obj from . anime import create_anidb_obj
from . classes import OrderedDefaultdict from . classes import OrderedDefaultdict
from . helpers import json
from . indexers . indexer_config import TVINFO_TVDB from . indexers . indexer_config import TVINFO_TVDB
from . sgdatetime import timestamp_near from . sgdatetime import timestamp_near
from _23 import filter_iter , map_iter import lib . rarfile . rarfile as rarfile
from _23 import filter_iter , list_range , map_iter
from six import iteritems , PY2 , text_type from six import iteritems , PY2 , text_type
# noinspection PyUnreachableCode # noinspection PyUnreachableCode
if False : if False :
# noinspection PyUnresolvedReferences # noinspection PyUnresolvedReferences
from typing import AnyStr , List , Tuple from typing import AnyStr , List , Tuple , Union
exception_dict = { } exception_dict = { }
anidb_exception_dict = { } anidb_exception_dict = { }
@ -50,20 +58,22 @@ exceptionsSeasonCache = {}
exceptionLock = threading . Lock ( ) exceptionLock = threading . Lock ( )
def should_refresh ( name ) : def should_refresh ( name , max_refresh_age_secs = 86400 , remaining = False ) :
# type: (AnyStr, int, bool) -> Union[bool, int]
""" """
: param name : name : param name : name
: type name : AnyStr : param max_refresh_age_secs :
: param remaining : True to return remaining seconds
: return : : return :
: rtype : bool
""" """
max_refresh_age_secs = 86400 # 1 day
my_db = db . DBConnection ( ) my_db = db . DBConnection ( )
rows = my_db . select ( ' SELECT last_refreshed FROM scene_exceptions_refresh WHERE list = ? ' , [ name ] ) rows = my_db . select ( ' SELECT last_refreshed FROM scene_exceptions_refresh WHERE list = ? ' , [ name ] )
if rows : if rows :
last_refresh = int ( rows [ 0 ] [ ' last_refreshed ' ] ) last_refresh = int ( rows [ 0 ] [ ' last_refreshed ' ] )
if remaining :
time_left = ( last_refresh + max_refresh_age_secs - int ( timestamp_near ( datetime . datetime . now ( ) ) ) )
return ( 0 , time_left ) [ time_left > 0 ]
return int ( timestamp_near ( datetime . datetime . now ( ) ) ) > last_refresh + max_refresh_age_secs return int ( timestamp_near ( datetime . datetime . now ( ) ) ) > last_refresh + max_refresh_age_secs
return True return True
@ -80,6 +90,13 @@ def set_last_refresh(name):
{ ' list ' : name } ) { ' list ' : name } )
def has_season_exceptions ( tvid , prodid , season ) :
get_scene_exceptions ( tvid , prodid , season = season )
if ( tvid , prodid ) in exceptionsCache and - 1 < season and season in exceptionsCache [ ( tvid , prodid ) ] :
return True
return False
def get_scene_exceptions ( tvid , prodid , season = - 1 ) : def get_scene_exceptions ( tvid , prodid , season = - 1 ) :
""" """
Given a indexer_id , return a list of all the scene exceptions . Given a indexer_id , return a list of all the scene exceptions .
@ -133,11 +150,19 @@ def get_all_scene_exceptions(tvid_prodid):
exceptions = my_db . select ( ' SELECT show_name,season ' exceptions = my_db . select ( ' SELECT show_name,season '
' FROM scene_exceptions ' ' FROM scene_exceptions '
' WHERE indexer = ? AND indexer_id = ? ' ' WHERE indexer = ? AND indexer_id = ? '
' ORDER BY season ' , ' ORDER BY season DESC, show_name DESC ' ,
TVidProdid ( tvid_prodid ) . list ) TVidProdid ( tvid_prodid ) . list )
exceptions_seasons = [ ]
if exceptions : if exceptions :
for cur_exception in exceptions : for cur_exception in exceptions :
# order as, s*, and then season desc, show_name also desc (so years in names may fall newest on top)
if - 1 == cur_exception [ ' season ' ] :
exceptions_dict [ cur_exception [ ' season ' ] ] . append ( cur_exception [ ' show_name ' ] )
else :
exceptions_seasons + = [ cur_exception ]
for cur_exception in exceptions_seasons :
exceptions_dict [ cur_exception [ ' season ' ] ] . append ( cur_exception [ ' show_name ' ] ) exceptions_dict [ cur_exception [ ' season ' ] ] . append ( cur_exception [ ' show_name ' ] )
return exceptions_dict return exceptions_dict
@ -257,6 +282,14 @@ def retrieve_exceptions():
else : else :
exception_dict [ anidb_ex ] = anidb_exception_dict [ anidb_ex ] exception_dict [ anidb_ex ] = anidb_exception_dict [ anidb_ex ]
# Custom exceptions
custom_exception_dict , cnt_updated_numbers , min_remain_iv = _custom_exceptions_fetcher ( )
for custom_ex in custom_exception_dict :
if custom_ex in exception_dict :
exception_dict [ custom_ex ] = exception_dict [ custom_ex ] + custom_exception_dict [ custom_ex ]
else :
exception_dict [ custom_ex ] = custom_exception_dict [ custom_ex ]
changed_exceptions = False changed_exceptions = False
# write all the exceptions we got off the net into the database # write all the exceptions we got off the net into the database
@ -265,16 +298,14 @@ def retrieve_exceptions():
for cur_tvid_prodid in exception_dict : for cur_tvid_prodid in exception_dict :
# get a list of the existing exceptions for this ID # get a list of the existing exceptions for this ID
existing_exceptions = [ x [ ' show_name ' ] for x in existing_exceptions = [ { x [ ' show_name ' ] : x [ ' season ' ] } for x in
my_db . select ( ' SELECT show_name ' my_db . select ( ' SELECT show_name, season '
' FROM scene_exceptions ' ' FROM scene_exceptions '
' WHERE indexer = ? AND indexer_id = ? ' , ' WHERE indexer = ? AND indexer_id = ? ' ,
list ( cur_tvid_prodid ) ) ] list ( cur_tvid_prodid ) ) ]
if cur_tvid_prodid not in exception_dict : # if this exception isn't already in the DB then add it
continue for cur_exception_dict in filter_iter ( lambda e : e not in existing_exceptions , exception_dict [ cur_tvid_prodid ] ) :
for cur_exception_dict in exception_dict [ cur_tvid_prodid ] :
try : try :
cur_exception , cur_season = next ( iteritems ( cur_exception_dict ) ) cur_exception , cur_season = next ( iteritems ( cur_exception_dict ) )
except ( BaseException , Exception ) : except ( BaseException , Exception ) :
@ -282,16 +313,13 @@ def retrieve_exceptions():
logger . log ( traceback . format_exc ( ) , logger . ERROR ) logger . log ( traceback . format_exc ( ) , logger . ERROR )
continue continue
# if this exception isn't already in the DB then add it if PY2 and not isinstance ( cur_exception , text_type ) :
if cur_exception not in existing_exceptions : cur_exception = text_type ( cur_exception , ' utf-8 ' , ' replace ' )
if PY2 and not isinstance ( cur_exception , text_type ) :
cur_exception = text_type ( cur_exception , ' utf-8 ' , ' replace ' )
cl . append ( [ ' INSERT INTO scene_exceptions ' cl . append ( [ ' INSERT INTO scene_exceptions '
' (indexer, indexer_id, show_name, season) VALUES (?,?,?,?) ' , ' (indexer, indexer_id, show_name, season) VALUES (?,?,?,?) ' ,
list ( cur_tvid_prodid ) + [ cur_exception , cur_season ] ] ) list ( cur_tvid_prodid ) + [ cur_exception , cur_season ] ] )
changed_exceptions = True changed_exceptions = True
if cl : if cl :
my_db . mass_action ( cl ) my_db . mass_action ( cl )
@ -308,6 +336,8 @@ def retrieve_exceptions():
anidb_exception_dict . clear ( ) anidb_exception_dict . clear ( )
xem_exception_dict . clear ( ) xem_exception_dict . clear ( )
return changed_exceptions , cnt_updated_numbers , min_remain_iv
def update_scene_exceptions ( tvid , prodid , scene_exceptions ) : def update_scene_exceptions ( tvid , prodid , scene_exceptions ) :
""" """
@ -331,6 +361,12 @@ def update_scene_exceptions(tvid, prodid, scene_exceptions):
logger . log ( u ' Updating scene exceptions ' , logger . MESSAGE ) logger . log ( u ' Updating scene exceptions ' , logger . MESSAGE )
for exception in scene_exceptions : for exception in scene_exceptions :
cur_season , cur_exception = exception . split ( ' | ' , 1 ) cur_season , cur_exception = exception . split ( ' | ' , 1 )
try :
cur_season = int ( cur_season )
except ( BaseException , Exception ) :
logger . log ( ' invalid scene exception: %s - %s : %s ' % ( ' %s : %s ' % ( tvid , prodid ) , cur_season , cur_exception ) ,
logger . ERROR )
continue
exceptionsCache [ ( tvid , prodid ) ] [ cur_season ] . append ( cur_exception ) exceptionsCache [ ( tvid , prodid ) ] [ cur_season ] . append ( cur_exception )
@ -344,6 +380,107 @@ def update_scene_exceptions(tvid, prodid, scene_exceptions):
sickbeard . name_cache . buildNameCache ( update_only_scene = True ) sickbeard . name_cache . buildNameCache ( update_only_scene = True )
def _custom_exceptions_fetcher ( ) :
custom_exception_dict = { }
cnt_updated_numbers = 0
src_id = ' GHSG '
logger . log ( u ' Checking to update custom alternatives from %s ' % src_id )
dirpath = ek . ek ( os . path . join , sickbeard . CACHE_DIR , ' alts ' )
tmppath = ek . ek ( os . path . join , dirpath , ' tmp ' )
file_rar = ek . ek ( os . path . join , tmppath , ' alt.rar ' )
file_cache = ek . ek ( os . path . join , dirpath , ' alt.json ' )
iv = 30 * 60 # min interval to fetch updates
refresh = should_refresh ( src_id , iv )
fetch_data = not ek . ek ( os . path . isfile , file_cache ) or ( not int ( os . environ . get ( ' NO_ALT_GET ' , 0 ) ) and refresh )
if fetch_data :
if ek . ek ( os . path . exists , tmppath ) :
helpers . remove_file ( tmppath , tree = True )
helpers . make_dirs ( tmppath )
helpers . download_file ( r ' https://github.com/SickGear/sickgear.altdata/raw/master/alt.rar ' , file_rar )
rar_handle = None
if ' win32 ' == sys . platform :
rarfile . UNRAR_TOOL = ek . ek ( os . path . join , sickbeard . PROG_DIR , ' lib ' , ' rarfile ' , ' UnRAR.exe ' )
try :
rar_handle = rarfile . RarFile ( file_rar )
rar_handle . extractall ( path = dirpath , pwd = ' sickgear_alt ' )
except ( BaseException , Exception ) as e :
logger . log ( u ' Failed to unpack archive: %s with error: %s ' % ( file_rar , ex ( e ) ) , logger . ERROR )
if rar_handle :
rar_handle . close ( )
del rar_handle
helpers . remove_file ( tmppath , tree = True )
if refresh :
set_last_refresh ( src_id )
data = { }
try :
with io . open ( file_cache ) as fh :
data = json . load ( fh )
except ( BaseException , Exception ) as e :
logger . log ( u ' Failed to unpack json data: %s with error: %s ' % ( file_rar , ex ( e ) ) , logger . ERROR )
# handle data
from . scene_numbering import find_scene_numbering , set_scene_numbering_helper
from . tv import TVidProdid
for tvid_prodid , season_data in iteritems ( data ) :
show_obj = sickbeard . helpers . find_show_by_id ( tvid_prodid , no_mapped_ids = True )
if not show_obj :
continue
used = set ( )
for for_season , data in iteritems ( season_data ) :
for_season = helpers . try_int ( for_season , None )
tvid , prodid = TVidProdid ( tvid_prodid ) . tuple
if data . get ( ' n ' ) : # alt names
custom_exception_dict . setdefault ( ( tvid , prodid ) , [ ] )
custom_exception_dict [ ( tvid , prodid ) ] + = [ { name : for_season } for name in data . get ( ' n ' ) ]
for update in data . get ( ' se ' ) or [ ] :
for for_episode , se_range in iteritems ( update ) : # scene episode alt numbers
for_episode = helpers . try_int ( for_episode , None )
target_season , episode_range = se_range . split ( ' x ' )
scene_episodes = [ int ( x ) for x in episode_range . split ( ' - ' ) if None is not helpers . try_int ( x , None ) ]
if 2 == len ( scene_episodes ) :
desc = scene_episodes [ 0 ] > scene_episodes [ 1 ]
if desc : # handle a descending range case
scene_episodes . reverse ( )
scene_episodes = list_range ( * [ scene_episodes [ 0 ] , scene_episodes [ 1 ] + 1 ] )
if desc :
scene_episodes . reverse ( )
target_season = helpers . try_int ( target_season , None )
for target_episode in scene_episodes :
sn = find_scene_numbering ( tvid , prodid , for_season , for_episode )
used . add ( ( for_season , for_episode , target_season , target_episode ) )
if sn and ( ( for_season , for_episode ) + sn ) not in used \
and ( for_season , for_episode ) not in used :
logger . log (
u ' Skipped setting " %s " episode %s x %s to target a release %s x %s because set to %s x %s '
% ( show_obj . name , for_season , for_episode , target_season , target_episode , sn [ 0 ] , sn [ 1 ] ) ,
logger . DEBUG )
else :
used . add ( ( for_season , for_episode ) )
if not sn or sn != ( target_season , target_episode ) : # not already set
result = set_scene_numbering_helper (
tvid , prodid , for_season = for_season , for_episode = for_episode ,
scene_season = target_season , scene_episode = target_episode )
if result . get ( ' success ' ) :
cnt_updated_numbers + = 1
for_episode = for_episode + 1
return custom_exception_dict , cnt_updated_numbers , should_refresh ( src_id , iv , remaining = True )
def _anidb_exceptions_fetcher ( ) : def _anidb_exceptions_fetcher ( ) :
global anidb_exception_dict global anidb_exception_dict
JackDandy
5 years ago