diff --git a/couchpotato/__init__.py b/couchpotato/__init__.py index fb6b4dc..daa93bc 100644 --- a/couchpotato/__init__.py +++ b/couchpotato/__init__.py @@ -1,3 +1,7 @@ +import os +import time +import traceback + from couchpotato.api import api_docs, api_docs_missing, api from couchpotato.core.event import fireEvent from couchpotato.core.helpers.variable import md5, tryInt @@ -5,9 +9,6 @@ from couchpotato.core.logger import CPLog from couchpotato.environment import Env from tornado import template from tornado.web import RequestHandler, authenticated -import os -import time -import traceback log = CPLog(__name__) diff --git a/couchpotato/api.py b/couchpotato/api.py index 99a2c6a..b21cfeb 100644 --- a/couchpotato/api.py +++ b/couchpotato/api.py @@ -89,6 +89,7 @@ class ApiHandler(RequestHandler): route = route.strip('/') if not api.get(route): self.write('API call doesn\'t seem to exist') + self.finish() return # Create lock if it doesn't exist diff --git a/couchpotato/core/_base/downloader/main.py b/couchpotato/core/_base/downloader/main.py index 70e5cc9..7ef98af 100644 --- a/couchpotato/core/_base/downloader/main.py +++ b/couchpotato/core/_base/downloader/main.py @@ -25,6 +25,7 @@ class DownloaderBase(Provider): status_support = True torrent_sources = [ + 'https://zoink.it/torrent/%s.torrent', 'http://torrage.com/torrent/%s.torrent', 'https://torcache.net/torrent/%s.torrent', ] diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index d753d87..c7051f5 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -3,10 +3,11 @@ import os import time import traceback +from CodernityDB.index import IndexException, IndexNotFoundException, IndexConflict from couchpotato import CPLog from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, sp from couchpotato.core.helpers.variable import getImdb, tryInt @@ -15,11 +16,13 @@ log = CPLog(__name__) class Database(object): - indexes = [] + indexes = None db = None def __init__(self): + self.indexes = {} + addApiView('database.list_documents', self.listDocuments) addApiView('database.reindex', self.reindex) addApiView('database.compact', self.compact) @@ -45,26 +48,45 @@ class Database(object): def setupIndex(self, index_name, klass): - self.indexes.append(index_name) + self.indexes[index_name] = klass db = self.getDB() # Category index index_instance = klass(db.path, index_name) try: - db.add_index(index_instance) - db.reindex_index(index_name) - except: - previous = db.indexes_names[index_name] - previous_version = previous._version - current_version = klass._version - - # Only edit index if versions are different - if previous_version < current_version: - log.debug('Index "%s" already exists, updating and reindexing', index_name) - db.destroy_index(previous) + + # Make sure store and bucket don't exist + exists = [] + for x in ['buck', 'stor']: + full_path = os.path.join(db.path, '%s_%s' % (index_name, x)) + if os.path.exists(full_path): + exists.append(full_path) + + if index_name not in db.indexes_names: + + # Remove existing buckets if index isn't there + for x in exists: + os.unlink(x) + + # Add index (will restore buckets) db.add_index(index_instance) db.reindex_index(index_name) + else: + # Previous info + previous = db.indexes_names[index_name] + previous_version = previous._version + current_version = klass._version + + # Only edit index if versions are different + if previous_version < current_version: + log.debug('Index "%s" already exists, updating and reindexing', index_name) + db.destroy_index(previous) + db.add_index(index_instance) + db.reindex_index(index_name) + + except: + log.error('Failed adding index %s: %s', (index_name, traceback.format_exc())) def deleteDocument(self, **kwargs): @@ -138,21 +160,62 @@ class Database(object): 'success': success } - def compact(self, **kwargs): + def compact(self, try_repair = True, **kwargs): + + success = False + db = self.getDB() + + # Removing left over compact files + db_path = sp(db.path) + for f in os.listdir(sp(db.path)): + for x in ['_compact_buck', '_compact_stor']: + if f[-len(x):] == x: + os.unlink(os.path.join(db_path, f)) - success = True try: start = time.time() - db = self.getDB() size = float(db.get_db_details().get('size', 0)) log.debug('Compacting database, current size: %sMB', round(size/1048576, 2)) db.compact() new_size = float(db.get_db_details().get('size', 0)) log.debug('Done compacting database in %ss, new size: %sMB, saved: %sMB', (round(time.time()-start, 2), round(new_size/1048576, 2), round((size-new_size)/1048576, 2))) + success = True + except (IndexException, AttributeError): + if try_repair: + log.error('Something wrong with indexes, trying repair') + + # Remove all indexes + old_indexes = self.indexes.keys() + for index_name in old_indexes: + try: + db.destroy_index(index_name) + except IndexNotFoundException: + pass + except: + log.error('Failed removing old index %s', index_name) + + # Add them again + for index_name in self.indexes: + klass = self.indexes[index_name] + + # Category index + index_instance = klass(db.path, index_name) + try: + db.add_index(index_instance) + db.reindex_index(index_name) + except IndexConflict: + pass + except: + log.error('Failed adding index %s', index_name) + raise + + self.compact(try_repair = False) + else: + log.error('Failed compact: %s', traceback.format_exc()) + except: log.error('Failed compact: %s', traceback.format_exc()) - success = False return { 'success': success @@ -166,6 +229,7 @@ class Database(object): size = db.get_db_details().get('size') prop_name = 'last_db_compact' last_check = int(Env.prop(prop_name, default = 0)) + if size > 26214400 and last_check < time.time()-604800: # 25MB / 7 days self.compact() Env.prop(prop_name, value = int(time.time())) diff --git a/couchpotato/core/downloaders/rtorrent_.py b/couchpotato/core/downloaders/rtorrent_.py index 822501a..7474697 100644 --- a/couchpotato/core/downloaders/rtorrent_.py +++ b/couchpotato/core/downloaders/rtorrent_.py @@ -5,7 +5,6 @@ from urlparse import urlparse import os from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList - from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import sp from couchpotato.core.helpers.variable import cleanHost, splitString diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 66e01f5..fc844aa 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -1,4 +1,5 @@ import collections +import ctypes import hashlib import os import platform @@ -291,9 +292,14 @@ def dictIsSubset(a, b): return all([k in b and b[k] == v for k, v in a.items()]) +# Returns True if sub_folder is the same as or inside base_folder def isSubFolder(sub_folder, base_folder): - # Returns True if sub_folder is the same as or inside base_folder - return base_folder and sub_folder and ss(os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep) in ss(os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep) + if base_folder and sub_folder: + base = sp(os.path.realpath(base_folder)) + os.path.sep + subfolder = sp(os.path.realpath(sub_folder)) + os.path.sep + return os.path.commonprefix([subfolder, base]) == base + + return False # From SABNZBD @@ -341,3 +347,36 @@ def removePyc(folder, only_excess = True, show_logs = True): os.rmdir(full_path) except: log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc())) + + +def getFreeSpace(directories): + + single = not isinstance(directories, (tuple, list)) + if single: + directories = [directories] + + free_space = {} + for folder in directories: + + size = None + if os.path.isdir(folder): + if os.name == 'nt': + _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), \ + ctypes.c_ulonglong() + if sys.version_info >= (3,) or isinstance(folder, unicode): + fun = ctypes.windll.kernel32.GetDiskFreeSpaceExW #@UndefinedVariable + else: + fun = ctypes.windll.kernel32.GetDiskFreeSpaceExA #@UndefinedVariable + ret = fun(folder, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free)) + if ret == 0: + raise ctypes.WinError() + return [total.value, free.value] + else: + s = os.statvfs(folder) + size = [s.f_blocks * s.f_frsize / (1024 * 1024), (s.f_bavail * s.f_frsize) / (1024 * 1024)] + + if single: return size + + free_space[folder] = size + + return free_space diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 4cfe597..1d3e153 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -1,3 +1,6 @@ +from datetime import timedelta +from operator import itemgetter +import time import traceback from string import ascii_lowercase @@ -164,8 +167,15 @@ class MediaPlugin(MediaBase): status = list(status if isinstance(status, (list, tuple)) else [status]) for s in status: - for ms in db.get_many('media_status', s, with_doc = with_doc): - yield ms['doc'] if with_doc else ms + for ms in db.get_many('media_status', s): + if with_doc: + try: + doc = db.get('id', ms['_id']) + yield doc + except RecordNotFound: + log.debug('Record not found, skipping: %s', ms['_id']) + else: + yield ms def withIdentifiers(self, identifiers, with_doc = False): @@ -282,8 +292,8 @@ class MediaPlugin(MediaBase): release_status = splitString(kwargs.get('release_status')), status_or = kwargs.get('status_or') is not None, limit_offset = kwargs.get('limit_offset'), - with_tags = kwargs.get('with_tags'), - starts_with = splitString(kwargs.get('starts_with')), + with_tags = splitString(kwargs.get('with_tags')), + starts_with = kwargs.get('starts_with'), search = kwargs.get('search') ) @@ -401,11 +411,11 @@ class MediaPlugin(MediaBase): total_deleted += 1 new_media_status = 'done' elif delete_from == 'manage': - if release.get('status') == 'done': + if release.get('status') == 'done' or media.get('status') == 'done': db.delete(release) total_deleted += 1 - if (total_releases == total_deleted and media['status'] != 'active') or (not new_media_status and delete_from == 'late'): + if (total_releases == total_deleted and media['status'] != 'active') or (total_releases == 0 and not new_media_status) or (not new_media_status and delete_from == 'late'): db.delete(media) deleted = True elif new_media_status: @@ -452,28 +462,35 @@ class MediaPlugin(MediaBase): if not m['profile_id']: m['status'] = 'done' else: - move_to_wanted = True + m['status'] = 'active' try: profile = db.get('id', m['profile_id']) media_releases = fireEvent('release.for_media', m['_id'], single = True) + done_releases = [release for release in media_releases if release.get('status') == 'done'] - for q_identifier in profile['qualities']: - index = profile['qualities'].index(q_identifier) + if done_releases: + # Only look at latest added release + release = sorted(done_releases, key = itemgetter('last_edit'), reverse = True)[0] - for release in media_releases: - if q_identifier == release['quality'] and (release.get('status') == 'done' and profile['finish'][index]): - move_to_wanted = False + # Check if we are finished with the media + if fireEvent('quality.isfinish', {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, timedelta(seconds = time.time() - release['last_edit']).days, single = True): + m['status'] = 'done' + elif previous_status == 'done': + m['status'] = 'done' - m['status'] = 'active' if move_to_wanted else 'done' except RecordNotFound: - log.debug('Failed restatus: %s', traceback.format_exc()) + log.debug('Failed restatus, keeping previous: %s', traceback.format_exc()) + m['status'] = previous_status # Only update when status has changed if previous_status != m['status']: db.update(m) - return True + # Tag media as recent + self.tag(media_id, 'recent') + + return m['status'] except: log.error('Failed restatus: %s', traceback.format_exc()) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentshack.py b/couchpotato/core/media/_base/providers/torrent/torrentshack.py index 0cfa04d..1af7e55 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentshack.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentshack.py @@ -48,9 +48,9 @@ class Base(TorrentProvider): 'name': six.text_type(link.span.string).translate({ord(six.u('\xad')): None}), 'url': self.urls['download'] % url['href'], 'detail_url': self.urls['download'] % link['href'], - 'size': self.parseSize(result.find_all('td')[4].string), - 'seeders': tryInt(result.find_all('td')[6].string), - 'leechers': tryInt(result.find_all('td')[7].string), + 'size': self.parseSize(result.find_all('td')[5].string), + 'seeders': tryInt(result.find_all('td')[7].string), + 'leechers': tryInt(result.find_all('td')[8].string), }) except: diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 07c4751..336d803 100644 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -236,7 +236,7 @@ class MovieBase(MovieTypeBase): db.update(m) - fireEvent('media.restatus', m['_id']) + fireEvent('media.restatus', m['_id'], single = True) m = db.get('id', media_id) diff --git a/couchpotato/core/media/movie/providers/info/themoviedb.py b/couchpotato/core/media/movie/providers/info/themoviedb.py index 4a397ed..ac1daec 100644 --- a/couchpotato/core/media/movie/providers/info/themoviedb.py +++ b/couchpotato/core/media/movie/providers/info/themoviedb.py @@ -154,7 +154,7 @@ class TheMovieDb(MovieProvider): # Add alternative names if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']: - movie_data['titles'].insert(0, movie_data['original_title']) + movie_data['titles'].append(movie_data['original_title']) if extended: for alt in movie.alternate_titles: diff --git a/couchpotato/core/media/movie/providers/trailer/hdtrailers.py b/couchpotato/core/media/movie/providers/trailer/hdtrailers.py index 7942533..828f017 100644 --- a/couchpotato/core/media/movie/providers/trailer/hdtrailers.py +++ b/couchpotato/core/media/movie/providers/trailer/hdtrailers.py @@ -21,6 +21,7 @@ class HDTrailers(TrailerProvider): 'backup': 'http://www.hd-trailers.net/blog/', } providers = ['apple.ico', 'yahoo.ico', 'moviefone.ico', 'myspace.ico', 'favicon.ico'] + only_tables_tags = SoupStrainer('table') def search(self, group): @@ -67,8 +68,7 @@ class HDTrailers(TrailerProvider): return results try: - tables = SoupStrainer('div') - html = BeautifulSoup(data, parse_only = tables) + html = BeautifulSoup(data, 'html.parser', parse_only = self.only_tables_tags) result_table = html.find_all('h2', text = re.compile(movie_name)) for h2 in result_table: @@ -90,8 +90,7 @@ class HDTrailers(TrailerProvider): results = {'480p':[], '720p':[], '1080p':[]} try: - tables = SoupStrainer('table') - html = BeautifulSoup(data, parse_only = tables) + html = BeautifulSoup(data, 'html.parser', parse_only = self.only_tables_tags) result_table = html.find('table', attrs = {'class':'bottomTable'}) for tr in result_table.find_all('tr'): diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 1053ec3..7d92c57 100644 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -120,8 +120,19 @@ class MovieSearcher(SearcherBase, MovieTypeBase): if not movie['profile_id'] or (movie['status'] == 'done' and not manual): log.debug('Movie doesn\'t have a profile or already done, assuming in manage tab.') + fireEvent('media.restatus', movie['_id'], single = True) return + default_title = getTitle(movie) + if not default_title: + log.error('No proper info found for movie, removing it from library to stop it from causing more issues.') + fireEvent('media.delete', movie['_id'], single = True) + return + + # Update media status and check if it is still not done (due to the stop searching after feature + if fireEvent('media.restatus', movie['_id'], single = True) == 'done': + log.debug('No better quality found, marking movie %s as done.', default_title) + pre_releases = fireEvent('quality.pre_releases', single = True) release_dates = fireEvent('movie.update_release_dates', movie['_id'], merge = True) @@ -133,12 +144,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase): ignore_eta = manual total_result_count = 0 - default_title = getTitle(movie) - if not default_title: - log.error('No proper info found for movie, removing it from library to cause it from having more issues.') - fireEvent('media.delete', movie['_id'], single = True) - return - fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'_id': movie['_id']}, message = 'Searching for "%s"' % default_title) # Ignore eta once every 7 days @@ -154,8 +159,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): profile = db.get('id', movie['profile_id']) ret = False - index = 0 - for q_identifier in profile.get('qualities'): + for index, q_identifier in enumerate(profile.get('qualities', [])): quality_custom = { 'index': index, 'quality': q_identifier, @@ -164,8 +168,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase): '3d': profile['3d'][index] if profile.get('3d') else False } - index += 1 - could_not_be_released = not self.couldBeReleased(q_identifier in pre_releases, release_dates, movie['info']['year']) if not alway_search and could_not_be_released: too_early_to_search.append(q_identifier) @@ -189,7 +191,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): # Don't search for quality lower then already available. if has_better_quality > 0: log.info('Better quality (%s) already available or snatched for %s', (q_identifier, default_title)) - fireEvent('media.restatus', movie['_id']) + fireEvent('media.restatus', movie['_id'], single = True) break quality = fireEvent('quality.single', identifier = q_identifier, single = True) diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py index c2cc907..146a6a0 100644 --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -1,4 +1,3 @@ -from couchpotato import get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent from couchpotato.core.helpers.variable import splitString, removeDuplicate, getIdentifier diff --git a/couchpotato/core/notifications/xbmc.py b/couchpotato/core/notifications/xbmc.py index 8dbf936..eb0b699 100644 --- a/couchpotato/core/notifications/xbmc.py +++ b/couchpotato/core/notifications/xbmc.py @@ -8,7 +8,7 @@ from couchpotato.core.helpers.variable import splitString, getTitle from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification import requests -from requests.packages.urllib3.exceptions import MaxRetryError +from requests.packages.urllib3.exceptions import MaxRetryError, ConnectionError log = CPLog(__name__) @@ -172,7 +172,7 @@ class XBMC(Notification): # manually fake expected response array return [{'result': 'Error'}] - except (MaxRetryError, requests.exceptions.Timeout): + except (MaxRetryError, requests.exceptions.Timeout, ConnectionError): log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off') return [{'result': 'Error'}] except: diff --git a/couchpotato/core/plugins/file.py b/couchpotato/core/plugins/file.py index 51adf8c..80c073f 100644 --- a/couchpotato/core/plugins/file.py +++ b/couchpotato/core/plugins/file.py @@ -5,7 +5,7 @@ from couchpotato import get_db from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.variable import md5, getExt +from couchpotato.core.helpers.variable import md5, getExt, isSubFolder from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -32,6 +32,8 @@ class FileManager(Plugin): fireEvent('schedule.interval', 'file.cleanup', self.cleanup, hours = 24) + addEvent('app.test', self.doSubfolderTest) + def cleanup(self): # Wait a bit after starting before cleanup @@ -76,3 +78,33 @@ class FileManager(Plugin): self.createFile(dest, filedata, binary = True) return dest + + def doSubfolderTest(self): + + tests = { + ('/test/subfolder', '/test/sub'): False, + ('/test/sub/folder', '/test/sub'): True, + ('/test/sub/folder', '/test/sub2'): False, + ('/sub/fold', '/test/sub/fold'): False, + ('/sub/fold', '/test/sub/folder'): False, + ('/opt/couchpotato', '/var/opt/couchpotato'): False, + ('/var/opt', '/var/opt/couchpotato'): False, + ('/CapItaLs/Are/OK', '/CapItaLs/Are/OK'): True, + ('/CapItaLs/Are/OK', '/CapItaLs/Are/OK2'): False, + ('/capitals/are/not/OK', '/capitals/are/NOT'): False, + ('\\\\Mounted\\Volume\\Test', '\\\\Mounted\\Volume'): True, + ('C:\\\\test\\path', 'C:\\\\test2'): False + } + + failed = 0 + for x in tests: + if isSubFolder(x[0], x[1]) is not tests[x]: + log.error('Failed subfolder test %s %s', x) + failed += 1 + + if failed > 0: + log.error('Subfolder test failed %s tests', failed) + else: + log.info('Subfolder test succeeded') + + return failed == 0 \ No newline at end of file diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index bec204b..c8d53ea 100644 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -1,13 +1,12 @@ -import ctypes import os -import sys import time import traceback +from couchpotato import get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent, fireEventAsync from couchpotato.core.helpers.encoding import sp -from couchpotato.core.helpers.variable import splitString, getTitle, tryInt, getIdentifier +from couchpotato.core.helpers.variable import splitString, getTitle, tryInt, getIdentifier, getFreeSpace from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -179,6 +178,10 @@ class Manage(Plugin): if self.shuttingDown(): break + if not self.shuttingDown(): + db = get_db() + db.reindex() + Env.prop(last_update_key, time.time()) except: log.error('Failed updating library: %s', (traceback.format_exc())) @@ -268,31 +271,7 @@ class Manage(Plugin): fireEvent('release.add', group = group) def getDiskSpace(self): - - free_space = {} - for folder in self.directories(): - - size = None - if os.path.isdir(folder): - if os.name == 'nt': - _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), \ - ctypes.c_ulonglong() - if sys.version_info >= (3,) or isinstance(folder, unicode): - fun = ctypes.windll.kernel32.GetDiskFreeSpaceExW #@UndefinedVariable - else: - fun = ctypes.windll.kernel32.GetDiskFreeSpaceExA #@UndefinedVariable - ret = fun(folder, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free)) - if ret == 0: - raise ctypes.WinError() - used = total.value - free.value - return [total.value, used, free.value] - else: - s = os.statvfs(folder) - size = [s.f_blocks * s.f_frsize / (1024 * 1024), (s.f_bavail * s.f_frsize) / (1024 * 1024)] - - free_space[folder] = size - - return free_space + return getFreeSpace(self.directories()) config = [{ diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index 1098719..489c34d 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -88,6 +88,7 @@ class ProfilePlugin(Plugin): 'core': kwargs.get('core', False), 'qualities': [], 'wait_for': [], + 'stop_after': [], 'finish': [], '3d': [] } @@ -97,6 +98,7 @@ class ProfilePlugin(Plugin): for type in kwargs.get('types', []): profile['qualities'].append(type.get('quality')) profile['wait_for'].append(tryInt(kwargs.get('wait_for', 0))) + profile['stop_after'].append(tryInt(kwargs.get('stop_after', 0))) profile['finish'].append((tryInt(type.get('finish')) == 1) if order > 0 else True) profile['3d'].append(tryInt(type.get('3d'))) order += 1 @@ -217,6 +219,7 @@ class ProfilePlugin(Plugin): 'qualities': profile.get('qualities'), 'finish': [], 'wait_for': [], + 'stop_after': [], '3d': [] } @@ -224,6 +227,7 @@ class ProfilePlugin(Plugin): for q in profile.get('qualities'): pro['finish'].append(True) pro['wait_for'].append(0) + pro['stop_after'].append(0) pro['3d'].append(threed.pop() if threed else False) db.insert(pro) diff --git a/couchpotato/core/plugins/profile/static/profile.css b/couchpotato/core/plugins/profile/static/profile.css index f8a1b42..edab831 100644 --- a/couchpotato/core/plugins/profile/static/profile.css +++ b/couchpotato/core/plugins/profile/static/profile.css @@ -43,9 +43,8 @@ } .profile .wait_for { - position: absolute; - right: 60px; - top: 0; + padding-top: 0; + padding-bottom: 20px; } .profile .wait_for input { diff --git a/couchpotato/core/plugins/profile/static/profile.js b/couchpotato/core/plugins/profile/static/profile.js index c62b137..89f1a69 100644 --- a/couchpotato/core/plugins/profile/static/profile.js +++ b/couchpotato/core/plugins/profile/static/profile.js @@ -37,20 +37,28 @@ var Profile = new Class({ 'placeholder': 'Profile name' }) ), - new Element('div.wait_for.ctrlHolder').adopt( - new Element('span', {'text':'Wait'}), - new Element('input.inlay.xsmall', { - 'type':'text', - 'value': data.wait_for && data.wait_for.length > 0 ? data.wait_for[0] : 0 - }), - new Element('span', {'text':'day(s) for a better quality.'}) - ), new Element('div.qualities.ctrlHolder').adopt( new Element('label', {'text': 'Search for'}), self.type_container = new Element('ol.types'), new Element('div.formHint', { 'html': "Search these qualities (2 minimum), from top to bottom. Use the checkbox, to stop searching after it found this quality." }) + ), + new Element('div.wait_for.ctrlHolder').adopt( + // "Wait the entered number of days for a checked quality, before downloading a lower quality release." + new Element('span', {'text':'Wait'}), + new Element('input.inlay.wait_for_input.xsmall', { + 'type':'text', + 'value': data.wait_for && data.wait_for.length > 0 ? data.wait_for[0] : 0 + }), + new Element('span', {'text':'day(s) for a better quality '}), + new Element('span.advanced', {'text':'and keep searching'}), + // "After a checked quality is found and downloaded, continue searching for even better quality releases for the entered number of days." + new Element('input.inlay.xsmall.stop_after_input.advanced', { + 'type':'text', + 'value': data.stop_after && data.stop_after.length > 0 ? data.stop_after[0] : 0 + }), + new Element('span.advanced', {'text':'day(s) for a better (checked) quality.'}) ) ); @@ -116,7 +124,8 @@ var Profile = new Class({ var data = { 'id' : self.data._id, 'label' : self.el.getElement('.quality_label input').get('value'), - 'wait_for' : self.el.getElement('.wait_for input').get('value'), + 'wait_for' : self.el.getElement('.wait_for_input').get('value'), + 'stop_after' : self.el.getElement('.stop_after_input').get('value'), 'types': [] }; diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index dd820cf..bc61afa 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -25,14 +25,14 @@ class QualityPlugin(Plugin): {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']}, {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']}, {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, - {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, + {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, - {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':[], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, + {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]}, {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':[]}, {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':[]}, - {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':[]} + {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]} ] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] threed_tags = { @@ -379,26 +379,31 @@ class QualityPlugin(Plugin): if score.get(q.get('identifier')): score[q.get('identifier')]['score'] -= 1 - def isFinish(self, quality, profile): + def isFinish(self, quality, profile, release_age = 0): if not isinstance(profile, dict) or not profile.get('qualities'): - return False + # No profile so anything (scanned) is good enough + return True try: - quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0] - return profile['finish'][quality_order] + index = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else False) == bool(quality.get('is_3d', False))][0] + + if index == 0 or (profile['finish'][index] and int(release_age) >= int(profile.get('stop_after', [0])[0])): + return True + + return False except: return False def isHigher(self, quality, compare_with, profile = None): if not isinstance(profile, dict) or not profile.get('qualities'): - profile = {'qualities': self.order} + profile = fireEvent('profile.default', single = True) # Try to find quality in profile, if not found: a quality we do not want is lower than anything else try: quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0] except: log.debug('Quality %s not found in profile identifiers %s', (quality['identifier'] + (' 3D' if quality.get('is_3d', 0) else ''), \ - [identifier + ('3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])])) + [identifier + (' 3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])])) return 'lower' # Try to find compare quality in profile, if not found: anything is higher than a not wanted quality @@ -446,6 +451,9 @@ class QualityPlugin(Plugin): '/movies/BluRay HDDVD H.264 MKV 720p EngSub/QuiQui le fou (criterion collection #123, 1915)/QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'}, 'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'}, 'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) half-sbs 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p', 'is_3d': True}, + 'Moviename 2014 720p HDCAM XviD DualAudio': {'size': 4000, 'quality': 'cam'}, + 'Moviename (2014) - 720p CAM x264': {'size': 2250, 'quality': 'cam'}, + 'Movie Name (2014).mp4': {'size': 750, 'quality': 'brrip'}, } correct = 0 diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index cb16c5a..196892c 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -3,7 +3,7 @@ import os import time import traceback -from CodernityDB.database import RecordDeleted +from CodernityDB.database import RecordDeleted, RecordNotFound from couchpotato import md5, get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent @@ -79,6 +79,13 @@ class Release(Plugin): try: db.get('id', release.get('key')) media_exist.append(release.get('key')) + + try: + if release['doc'].get('status') == 'ignore': + release['doc']['status'] = 'ignored' + db.update(release['doc']) + except: + log.error('Failed fixing mis-status tag: %s', traceback.format_exc()) except RecordDeleted: db.delete(release['doc']) log.debug('Deleted orphaned release: %s', release['doc']) @@ -100,9 +107,9 @@ class Release(Plugin): if rel['status'] in ['available']: self.delete(rel['_id']) - # Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the move + # Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the media elif rel['status'] in ['snatched', 'downloaded']: - self.updateStatus(rel['_id'], status = 'ignore') + self.updateStatus(rel['_id'], status = 'ignored') fireEvent('media.untag', media.get('_id'), 'recent', single = True) @@ -164,7 +171,7 @@ class Release(Plugin): release['files'] = dict((k, [toUnicode(x) for x in v]) for k, v in group['files'].items() if v) db.update(release) - fireEvent('media.restatus', media['_id']) + fireEvent('media.restatus', media['_id'], single = True) return True except: @@ -331,24 +338,14 @@ class Release(Plugin): if media['status'] == 'active': profile = db.get('id', media['profile_id']) - finished = False - if rls['quality'] in profile['qualities']: - nr = profile['qualities'].index(rls['quality']) - finished = profile['finish'][nr] - - if finished: + if fireEvent('quality.isfinish', {'identifier': rls['quality'], 'is_3d': rls.get('is_3d', False)}, profile, single = True): log.info('Renamer disabled, marking media as finished: %s', log_movie) # Mark release done self.updateStatus(rls['_id'], status = 'done') # Mark media done - mdia = db.get('id', media['_id']) - mdia['status'] = 'done' - mdia['last_edit'] = int(time.time()) - db.update(mdia) - - fireEvent('media.tag', media['_id'], 'recent', single = True) + fireEvent('media.restatus', media['_id'], single = True) return True @@ -511,8 +508,15 @@ class Release(Plugin): status = list(status if isinstance(status, (list, tuple)) else [status]) for s in status: - for ms in db.get_many('release_status', s, with_doc = with_doc): - yield ms['doc'] if with_doc else ms + for ms in db.get_many('release_status', s): + if with_doc: + try: + doc = db.get('id', ms['_id']) + yield doc + except RecordNotFound: + log.debug('Record not found, skipping: %s', ms['_id']) + else: + yield ms def forMedia(self, media_id): diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 8b57103..481c9dd 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -136,7 +136,7 @@ class Renamer(Plugin): else: for item in no_process: if isSubFolder(item, base_folder): - log.error('To protect your data, the media libraries can\'t be inside of or the same as the "from" folder.') + log.error('To protect your data, the media libraries can\'t be inside of or the same as the "from" folder. "%s" in "%s"', (item, base_folder)) return # Check to see if the no_process folders are inside the provided media_folder @@ -168,7 +168,7 @@ class Renamer(Plugin): if media_folder: for item in no_process: if isSubFolder(item, media_folder): - log.error('To protect your data, the media libraries can\'t be inside of or the same as the provided media folder.') + log.error('To protect your data, the media libraries can\'t be inside of or the same as the provided media folder. "%s" in "%s"', (item, media_folder)) return # Make sure a checkSnatched marked all downloads/seeds as such @@ -446,22 +446,19 @@ class Renamer(Plugin): # Before renaming, remove the lower quality files remove_leftovers = True - # Mark movie "done" once it's found the quality with the finish check + # Get media quality profile profile = None - try: - if media.get('status') == 'active' and media.get('profile_id'): + if media.get('profile_id'): + try: profile = db.get('id', media['profile_id']) - if fireEvent('quality.isfinish', group['meta_data']['quality'], profile, single = True): - mdia = db.get('id', media['_id']) - mdia['status'] = 'done' - mdia['last_edit'] = int(time.time()) - db.update(mdia) - - # List movie on dashboard - fireEvent('media.tag', media['_id'], 'recent', single = True) - - except: - log.error('Failed marking movie finished: %s', (traceback.format_exc())) + except: + # Set profile to None as it does not exist anymore + mdia = db.get('id', media['_id']) + mdia['profile_id'] = None + db.update(mdia) + log.error('Error getting quality profile for %s: %s', (media_title, traceback.format_exc())) + else: + log.debug('Media has no quality profile: %s', media_title) # Mark media for dashboard mark_as_recent = False @@ -474,7 +471,7 @@ class Renamer(Plugin): # This is where CP removes older, lesser quality releases or releases that are not wanted anymore is_higher = fireEvent('quality.ishigher', \ - group['meta_data']['quality'], {'identifier': release['quality'], 'is_3d': release.get('is_3d', 0)}, profile, single = True) + group['meta_data']['quality'], {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, single = True) if is_higher == 'higher': log.info('Removing lesser or not wanted quality %s for %s.', (media_title, release.get('quality'))) @@ -499,7 +496,7 @@ class Renamer(Plugin): self.tagRelease(group = group, tag = 'exists') # Notify on rename fail - download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (media_title, group['meta_data']['quality']['label'], release.get('identifier')) + download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (media_title, group['meta_data']['quality']['label'], release.get('quality')) fireEvent('movie.renaming.canceled', message = download_message, data = group) remove_leftovers = False @@ -518,7 +515,7 @@ class Renamer(Plugin): fireEvent('release.update_status', release['_id'], status = 'seeding', single = True) mark_as_recent = True - elif release.get('identifier') == group['meta_data']['quality']['identifier']: + elif release.get('quality') == group['meta_data']['quality']['identifier']: # Set the release to downloaded fireEvent('release.update_status', release['_id'], status = 'downloaded', single = True) group['release_download'] = release_download diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index 01a88fb..6a4d537 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -639,9 +639,9 @@ class Scanner(Plugin): # Try with other if len(movie) == 0 and name_year.get('other') and name_year['other'].get('name') and name_year['other'].get('year'): - search_q2 = '%(name)s %(year)s' % name_year + search_q2 = '%(name)s %(year)s' % name_year.get('other') if search_q2 != search_q: - movie = fireEvent('movie.search', q = '%(name)s %(year)s' % name_year.get('other'), merge = True, limit = 1) + movie = fireEvent('movie.search', q = search_q2, merge = True, limit = 1) if len(movie) > 0: imdb_id = movie[0].get('imdb') diff --git a/couchpotato/core/plugins/trailer.py b/couchpotato/core/plugins/trailer.py index ae52586..82216b8 100644 --- a/couchpotato/core/plugins/trailer.py +++ b/couchpotato/core/plugins/trailer.py @@ -32,7 +32,7 @@ class Trailer(Plugin): destination = os.path.join(group['destination_dir'], filename) if not os.path.isfile(destination): trailer_file = fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True) - if os.path.getsize(trailer_file) < (1024 * 1024): # Don't trust small trailers (1MB), try next one + if trailer_file and os.path.getsize(trailer_file) < (1024 * 1024): # Don't trust small trailers (1MB), try next one os.unlink(trailer_file) continue else: diff --git a/couchpotato/core/settings.py b/couchpotato/core/settings.py index c6de952..4315ec1 100644 --- a/couchpotato/core/settings.py +++ b/couchpotato/core/settings.py @@ -71,15 +71,7 @@ class Settings(object): self.connectEvents() def databaseSetup(self): - from couchpotato import get_db - - db = get_db() - - try: - db.add_index(PropertyIndex(db.path, 'property')) - except: - self.log.debug('Index for properties already exists') - db.edit_index(PropertyIndex(db.path, 'property')) + fireEvent('database.setup_index', 'property', PropertyIndex) def parser(self): return self.p diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 8a44605..e5f9bca 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -17,7 +17,7 @@ from couchpotato import KeyHandler, LoginHandler, LogoutHandler from couchpotato.api import NonBlockHandler, ApiHandler from couchpotato.core.event import fireEventAsync, fireEvent from couchpotato.core.helpers.encoding import sp -from couchpotato.core.helpers.variable import getDataDir, tryInt +from couchpotato.core.helpers.variable import getDataDir, tryInt, getFreeSpace import requests from tornado.httpserver import HTTPServer from tornado.web import Application, StaticFileHandler, RedirectHandler @@ -195,6 +195,15 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En log = CPLog(__name__) log.debug('Started with options %s', options) + # Check available space + try: + total_space, available_space = getFreeSpace(data_dir) + if available_space < 100: + log.error('Shutting down as CP needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left', available_space) + return + except: + log.error('Failed getting diskspace: %s', traceback.format_exc()) + def customwarn(message, category, filename, lineno, file = None, line = None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn diff --git a/couchpotato/static/style/settings.css b/couchpotato/static/style/settings.css index 50b305e..7fb1df2 100644 --- a/couchpotato/static/style/settings.css +++ b/couchpotato/static/style/settings.css @@ -75,6 +75,8 @@ color: #edc07f; } .page.show_advanced .advanced { display: block; } + .page.show_advanced span.advanced, + .page.show_advanced input.advanced { display: inline; } .page.settings .tab_content { display: none; @@ -176,7 +178,7 @@ padding: 6px 0 0; } - .page .xsmall { width: 20px !important; text-align: center; } + .page .xsmall { width: 25px !important; text-align: center; } .page .enabler { display: block;