Browse Source

nosql

pull/3111/head
Ruud 11 years ago
parent
commit
ecf91d616b
  1. 4
      couchpotato/core/media/_base/media/main.py
  2. 2
      couchpotato/core/media/movie/searcher/main.py
  3. 2
      couchpotato/core/plugins/manage/main.py
  4. 14
      couchpotato/core/plugins/release/index.py
  5. 117
      couchpotato/core/plugins/release/main.py
  6. 218
      couchpotato/core/plugins/renamer/main.py
  7. 9
      couchpotato/core/plugins/scanner/main.py

4
couchpotato/core/media/_base/media/main.py

@ -251,9 +251,7 @@ class MediaPlugin(MediaBase):
media = db.run('media', 'to_dict', media_id)
media['releases'] = []
for r in db.get_many('release', media_id, with_doc = True):
media['releases'].append(r['doc'])
media['releases'] = list(db.run('release', 'for_media', media_id))
# Merge releases with movie dict
medias.append(media)

2
couchpotato/core/media/movie/searcher/main.py

@ -184,7 +184,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
ret = True
# Remove releases that aren't found anymore
for release in movie.get('releases', []):
for release in db.run('release', 'for_media', movie['_id']):
if release.get('status') == 'available' and release.get('identifier') not in found_releases:
fireEvent('release.delete', release.get('_id'), single = True)

2
couchpotato/core/plugins/manage/main.py

@ -129,7 +129,7 @@ class Manage(Plugin):
releases = list(db.run('release', 'for_media', done_movie.get('_id')))
for release in releases:
if len(release.get('files', [])) > 0:
if release.get('files'):
for file_type in release.get('files', {}):
for release_file in release['files'][file_type]:
# Remove release not available anymore

14
couchpotato/core/plugins/release/index.py

@ -55,3 +55,17 @@ class ReleaseIDIndex(HashIndex):
def make_key_value(self, data):
if data.get('_t') == 'release' and data.get('identifier'):
return md5(data.get('identifier')).hexdigest(), {'media_id': data.get('media_id')}
class ReleaseDownloadIndex(HashIndex):
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
super(ReleaseDownloadIndex, self).__init__(*args, **kwargs)
def make_key(self, key):
return md5(key).hexdigest()
def make_key_value(self, data):
if data.get('_t') == 'release' and data.get('download_info') and data['download_info']['id'] and data['download_info']['downloader']:
return md5('%s-%s' % (data['download_info']['downloader'], data['download_info']['id'])).hexdigest(), None

117
couchpotato/core/plugins/release/main.py

@ -1,18 +1,14 @@
from couchpotato import get_session, md5, get_db
from couchpotato import md5, get_db
from couchpotato.api import addApiView
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import ss, toUnicode
from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from .index import ReleaseIndex, ReleaseStatusIndex, ReleaseIDIndex
from couchpotato.core.plugins.scanner.main import Scanner
from couchpotato.core.settings.model import Release as Relea, Media, \
ReleaseInfo
from .index import ReleaseIndex, ReleaseStatusIndex, ReleaseIDIndex, ReleaseDownloadIndex
from couchpotato.environment import Env
from inspect import ismethod, isfunction
from sqlalchemy.exc import InterfaceError
from sqlalchemy.sql.expression import and_, or_
import os
import time
import traceback
@ -81,6 +77,13 @@ class Release(Plugin):
log.debug('Index already exists')
db.edit_index(ReleaseIDIndex(db.path, 'release_identifier'))
# Release identifier index
try:
db.add_index(ReleaseDownloadIndex(db.path, 'release_download'))
except:
log.debug('Index already exists')
db.edit_index(ReleaseDownloadIndex(db.path, 'release_download'))
def cleanDone(self):
log.debug('Removing releases from dashboard')
@ -217,40 +220,35 @@ class Release(Plugin):
def manualDownload(self, id = None, **kwargs):
db = get_session()
db = get_db()
rel = db.query(Relea).filter_by(id = id).first()
if not rel:
log.error('Couldn\'t find release with id: %s', id)
return {
'success': False
}
try:
release = db.get('id', id)
item = release['info']
movie = db.get('id', release['media_id'])
item = {}
for info in rel.info:
item[info.identifier] = info.value
fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Snatching "%s"' % item['name'])
fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Snatching "%s"' % item['name'])
# Get matching provider
provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True)
# Get matching provider
provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True)
if item.get('protocol') != 'torrent_magnet':
item['download'] = provider.loginDownload if provider.urls.get('login') else provider.download
if item.get('protocol') != 'torrent_magnet':
item['download'] = provider.loginDownload if provider.urls.get('login') else provider.download
success = self.download(data = item, media = movie, manual = True)
success = self.download(data = item, media = rel.movie.to_dict({
'profile': {'types': {'quality': {}}},
'releases': {'status': {}, 'quality': {}},
'library': {'titles': {}, 'files': {}},
'files': {}
}), manual = True)
if success:
fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name'])
if success:
fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name'])
return {
'success': success == True
}
return {
'success': success == True
}
except:
log.error('Couldn\'t find release with id: %s: %s', (id, traceback.format_exc()))
return {
'success': False
}
def download(self, data, media, manual = False):
@ -282,9 +280,11 @@ class Release(Plugin):
log.debug('Downloader result: %s', download_result)
try:
db = get_session()
rls = db.query(Relea).filter_by(identifier = md5(data['url'])).first()
if not rls:
db = get_db()
try:
rls = db.get('release_identifier', md5(data['url']), with_doc = True)['doc']
except:
log.error('No release found to store download information in')
return False
@ -292,44 +292,44 @@ class Release(Plugin):
# Save download-id info if returned
if isinstance(download_result, dict):
for key in download_result:
rls_info = ReleaseInfo(
identifier = 'download_%s' % key,
value = toUnicode(download_result.get(key))
)
rls.info.append(rls_info)
db.commit()
log_movie = '%s (%s) in %s' % (getTitle(media), media['info']['year'], rls.quality.label)
rls['download_info'] = download_result
db.update(rls)
log_movie = '%s (%s) in %s' % (getTitle(media), media['info']['year'], rls['quality'])
snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie)
log.info(snatch_message)
fireEvent('%s.snatched' % data['type'], message = snatch_message, data = rls.to_dict())
fireEvent('%s.snatched' % data['type'], message = snatch_message, data = rls)
# Mark release as snatched
if renamer_enabled:
self.updateStatus(rls.id, status = 'snatched')
self.updateStatus(rls['_id'], status = 'snatched')
# If renamer isn't used, mark media done if finished or release downloaded
else:
if media['status'] == 'active':
finished = next((True for profile_type in media['profile']['types']
if profile_type['quality_id'] == rls.quality.id and profile_type['finish']), False)
profile = db.get('id', media['profile_id'])
finished = False
if rls['quality'] in profile['qualities']:
nr = profile['qualities'].index(rls['quality'])
finished = profile['finish'][nr]
if finished:
log.info('Renamer disabled, marking media as finished: %s', log_movie)
# Mark release done
self.updateStatus(rls.id, status = 'done')
self.updateStatus(rls['_id'], status = 'done')
# Mark media done
mdia = db.query(Media).filter_by(id = media['id']).first()
mdia.status = 'done'
mdia.last_edit = int(time.time())
db.commit()
mdia = db.get('id', media['_id'])
mdia['status'] = 'done'
mdia['last_edit'] = int(time.time())
db.update(mdia)
return True
# Assume release downloaded
self.updateStatus(rls.id, status = 'downloaded')
self.updateStatus(rls['_id'], status = 'downloaded')
except:
log.error('Failed storing download status: %s', traceback.format_exc())
@ -420,10 +420,11 @@ class Release(Plugin):
release_name = rel.get('name')
if rel.get('files'):
for file_item in rel.get('files', []):
if file_item.get('type') == 'movie':
release_name = os.path.basename(file_item.get('path'))
break
for file_type in rel.get('files', {}):
if file_type == 'movie':
for release_file in rel['files'][file_type]:
release_name = os.path.basename(release_file)
break
#update status in Db
log.debug('Marking release %s as %s', (release_name, status))

218
couchpotato/core/plugins/renamer/main.py

@ -1,4 +1,4 @@
from couchpotato import get_session, get_db
from couchpotato import get_db
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import toUnicode, ss, sp
@ -6,8 +6,6 @@ from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Library, File, Profile, \
ReleaseInfo
from couchpotato.environment import Env
from scandir import scandir
from unrar2 import RarFile
@ -113,6 +111,9 @@ class Renamer(Plugin):
# Get media folder to process
media_folder = release_download.get('folder')
# Quality order for calculation quality priority
quality_order = fireEvent('quality.order', single = True)
# Get all folders that should not be processed
no_process = [to_folder]
cat_list = fireEvent('category.all', single = True) or []
@ -192,7 +193,7 @@ class Renamer(Plugin):
except:
log.error('Failed getting files from %s: %s', (media_folder, traceback.format_exc()))
db = get_session()
db = get_db()
# Extend the download info with info stored in the downloaded release
release_download = self.extendReleaseDownload(release_download)
@ -223,36 +224,47 @@ class Renamer(Plugin):
remove_files = []
remove_releases = []
movie_title = getTitle(group)
media_title = getTitle(group)
# Add _UNKNOWN_ if no library item is connected
if not group.get('info') or not movie_title:
if not group.get('info') or not media_title:
self.tagRelease(group = group, tag = 'unknown')
continue
# Rename the files using the library data
else:
group['media'] = fireEvent('movie.update_info', identifier = group['media']['identifier'], single = True)
if not group['media']:
# Media not in library, add it first
if not group['media'].get('_id'):
group['media'] = fireEvent('movie.add', params = {
'identifier': group['identifier'],
'profile_id': None
}, search_after = False, status = 'done')
else:
group['media'] = fireEvent('movie.update_info', identifier = group['media']['identifier'], single = True)
if not group['media'] or not group['media'].get('_id'):
log.error('Could not rename, no library item to work with: %s', group_identifier)
continue
movie_title = getTitle(group['media'])
media = group['media']
media_title = getTitle(media)
# Overwrite destination when set in category
destination = to_folder
category_label = ''
for movie in library_ent.movies:
if movie.category and movie.category.label:
category_label = movie.category.label
if movie.category and movie.category.destination and len(movie.category.destination) > 0 and movie.category.destination != 'None':
destination = movie.category.destination
log.debug('Setting category destination for "%s": %s' % (movie_title, destination))
else:
log.debug('No category destination found for "%s"' % movie_title)
if media.get('category_id'):
try:
category = db.get('id', media['category_id'])
category_label = category['label']
break
if category['destination'] and len(category['destination']) > 0 and category['destination'] != 'None':
destination = category['destination']
log.debug('Setting category destination for "%s": %s' % (media_title, destination))
else:
log.debug('No category destination found for "%s"' % media_title)
except:
log.error('Failed getting category label: %s', traceback.format_exc())
# Find subtitle for renaming
group['before_rename'] = []
@ -263,7 +275,7 @@ class Renamer(Plugin):
group['before_rename'].extend(extr_files)
# Remove weird chars from movie name
movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', movie_title)
movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', media_title)
# Put 'The' at the end
name_the = movie_name
@ -274,7 +286,7 @@ class Renamer(Plugin):
'ext': 'mkv',
'namethe': name_the.strip(),
'thename': movie_name.strip(),
'year': library['year'],
'year': media['info']['year'],
'first': name_the[0].upper(),
'quality': group['meta_data']['quality']['label'],
'quality_type': group['meta_data']['quality_type'],
@ -285,10 +297,10 @@ class Renamer(Plugin):
'resolution_width': group['meta_data'].get('resolution_width'),
'resolution_height': group['meta_data'].get('resolution_height'),
'audio_channels': group['meta_data'].get('audio_channels'),
'imdb_id': library['identifier'],
'imdb_id': group['identifier'],
'cd': '',
'cd_nr': '',
'mpaa': library['info'].get('mpaa', ''),
'mpaa': media['info'].get('mpaa', ''),
'category': category_label,
}
@ -414,73 +426,73 @@ class Renamer(Plugin):
# Before renaming, remove the lower quality files
remove_leftovers = True
# Add it to the wanted list before we continue
if len(library_ent.movies) == 0:
profile = db.query(Profile).filter_by(core = True, label = group['meta_data']['quality']['label']).first()
fireEvent('movie.add', params = {'identifier': group['identifier'], 'profile_id': profile.id}, search_after = False)
db.expire_all()
library_ent = db.query(Library).filter_by(identifier = group['identifier']).first()
for movie in library_ent.movies:
# Mark movie "done" once it's found the quality with the finish check
try:
if media.get('status') == 'active' and media.get('profile_id'):
profile = db.get('id', media['profile_id'])
if group['meta_data']['quality']['identifier'] in profile.get('qualities', []):
nr = profile['qualities'].index(group['meta_data']['quality']['identifier'])
finish = profile['finish'][nr]
if finish:
mdia = db.get('id', media['_id'])
mdia['status'] = 'done'
mdia['last_edit'] = int(time.time())
db.update(mdia)
# Mark movie "done" once it's found the quality with the finish check
try:
if movie.get('status') == 'active' and movie.get('profile_id'):
for profile_type in movie.profile.types:
if profile_type.quality_id == group['meta_data']['quality']['id'] and profile_type.finish:
movie['status'] = 'done'
movie['last_edit'] = int(time.time())
db.commit()
except Exception as e:
log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc()))
db.rollback()
# Go over current movie releases
for release in movie.releases:
# When a release already exists
if release.get('status') == 'done':
# This is where CP removes older, lesser quality releases
if release.quality.order > group['meta_data']['quality']['order']:
log.info('Removing lesser quality %s for %s.', (movie.library.titles[0].title, release.quality.label))
for current_file in release.files:
remove_files.append(current_file)
remove_releases.append(release)
# Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
elif release.quality.order is group['meta_data']['quality']['order']:
log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (movie.library.titles[0].title, release.quality.label))
for current_file in release.files:
remove_files.append(current_file)
remove_releases.append(release)
# Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan
else:
log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label))
except Exception as e:
log.error('Failed marking movie finished: %s', (traceback.format_exc()))
# Go over current movie releases
for release in db.run('release', 'for_media', media['_id']):
# When a release already exists
if release.get('status') == 'done':
release_order = quality_order.index(release['quality'])
group_quality_order = quality_order.index(group['meta_data']['quality']['identifier'])
# This is where CP removes older, lesser quality releases
if release_order > group_quality_order:
log.info('Removing lesser quality %s for %s.', (media_title, release.get('quality')))
for file_type in release.get('files', {}):
for release_file in release['files'][file_type]:
remove_files.append(release_file)
remove_releases.append(release)
# Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
elif release_order == group_quality_order:
log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (media_title, release.get('quality')))
for file_type in release.get('files', {}):
for release_file in release['files'][file_type]:
remove_files.append(release_file)
remove_releases.append(release)
# Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan
else:
log.info('Better quality release already exists for %s, with quality %s', (media_title, release.get('quality')))
# Add exists tag to the .ignore file
self.tagRelease(group = group, tag = 'exists')
# Add exists tag to the .ignore file
self.tagRelease(group = group, tag = 'exists')
# Notify on rename fail
download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (getTitle(movie), group['meta_data']['quality']['label'], release.quality.label)
fireEvent('movie.renaming.canceled', message = download_message, data = group)
remove_leftovers = False
# Notify on rename fail
download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (media_title, group['meta_data']['quality']['label'], release.get('identifier'))
fireEvent('movie.renaming.canceled', message = download_message, data = group)
remove_leftovers = False
break
break
elif release.get('status') in ['snatched', 'seeding']:
if release_download and release_download.get('rls_id'):
if release_download['rls_id'] == release.id:
if release_download['status'] == 'completed':
# Set the release to downloaded
fireEvent('release.update_status', release.id, status = 'downloaded', single = True)
elif release_download['status'] == 'seeding':
# Set the release to seeding
fireEvent('release.update_status', release.id, status = 'seeding', single = True)
elif release.quality.id is group['meta_data']['quality']['id']:
elif release.get('status') in ['snatched', 'seeding']:
if release_download and release_download.get('release_id'):
if release_download['release_id'] == release['_id']:
if release_download['status'] == 'completed':
# Set the release to downloaded
fireEvent('release.update_status', release.id, status = 'downloaded', single = True)
fireEvent('release.update_status', release['_id'], status = 'downloaded', single = True)
elif release_download['status'] == 'seeding':
# Set the release to seeding
fireEvent('release.update_status', release['_id'], status = 'seeding', single = True)
elif release.get('identifier') == group['meta_data']['quality']['identifier']:
# Set the release to downloaded
fireEvent('release.update_status', release['_id'], status = 'downloaded', single = True)
# Remove leftover files
if not remove_leftovers: # Don't remove anything
@ -496,9 +508,6 @@ class Renamer(Plugin):
delete_folders = []
for src in remove_files:
if isinstance(src, File):
src = src.path
if rename_files.get(src):
log.debug('Not removing file that will be renamed: %s', src)
continue
@ -542,7 +551,7 @@ class Renamer(Plugin):
self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(release_download) or self.fileIsAdded(src, group))
group['renamed_files'].append(dst)
except:
log.error('Failed ranaming the file "%s" : %s', (os.path.basename(src), traceback.format_exc()))
log.error('Failed renaming the file "%s" : %s', (os.path.basename(src), traceback.format_exc()))
failed_rename = True
break
@ -581,7 +590,7 @@ class Renamer(Plugin):
log.error('Failed removing %s: %s', (group_folder, traceback.format_exc()))
# Notify on download, search for trailers etc
download_message = 'Downloaded %s (%s)' % (movie_title, replacements['quality'])
download_message = 'Downloaded %s (%s)' % (media_title, replacements['quality'])
try:
fireEvent('renamer.after', message = download_message, group = group, in_order = True)
except:
@ -1040,32 +1049,21 @@ Remove it if you want it to be renamed (again, or at least let it try again)
def extendReleaseDownload(self, release_download):
rls = None
db = get_db()
if release_download and release_download.get('id') and release_download.get('downloader'):
db = get_session()
rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = release_download.get('downloader')).all()
rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = release_download.get('id')).all()
for rlsnfo_dwnld in rlsnfo_dwnlds:
for rlsnfo_id in rlsnfo_ids:
if rlsnfo_id.release == rlsnfo_dwnld.release:
rls = rlsnfo_id.release
break
if rls: break
if not rls:
if release_download and release_download.get('id'):
try:
rls = db.get('release_download', '%s_%s' % (release_download.get('downloader'), release_download.get('id')), with_doc = True)['doc']
except:
log.error('Download ID %s from downloader %s not found in releases', (release_download.get('id'), release_download.get('downloader')))
if rls:
rls_dict = rls.to_dict({'info':{}})
media = db.get('id', rls['media_id'])
release_download.update({
'imdb_id': rls.movie.library.identifier,
'quality': rls.quality.identifier,
'protocol': rls_dict.get('info', {}).get('protocol') or rls_dict.get('info', {}).get('type'),
'rls_id': rls.id,
'imdb_id': media['identifier'],
'quality': rls['quality'],
'protocol': rls.get('info', {}).get('protocol') or rls.get('info', {}).get('type'),
'release_id': rls['_id'],
})
return release_download

9
couchpotato/core/plugins/scanner/main.py

@ -419,7 +419,7 @@ class Scanner(Plugin):
if not group['media']:
log.error('Unable to determine media: %s', group['identifiers'])
else:
group['identifier'] = group['media']['imdb']
group['identifier'] = group['media'].get('identifier') or group['media']['info'].get('imdb')
processed_movies[identifier] = group
@ -613,10 +613,13 @@ class Scanner(Plugin):
if imdb_id:
try:
db = get_db()
return db.get('media', imdb_id, with_doc = True)['doc']['info']
return db.get('media', imdb_id, with_doc = True)['doc']
except:
log.debug('Movie "%s" not in library, just getting info', imdb_id)
return fireEvent('movie.info', identifier = imdb_id, merge = True, extended = False)
return {
'identifier': imdb_id,
'info': fireEvent('movie.info', identifier = imdb_id, merge = True, extended = False)
}
log.error('No imdb_id found for %s. Add a NFO file with IMDB id or add the year to the filename.', group['identifiers'])
return {}

Loading…
Cancel
Save