You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

1189 lines
56 KiB

from couchpotato import get_db
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import toUnicode, ss, sp
from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
from scandir import scandir
from unrar2 import RarFile
import errno
import fnmatch
import os
import re
import shutil
import time
import traceback
import six
from six.moves import filter
log = CPLog(__name__)
class Renamer(Plugin):
renaming_started = False
checking_snatched = False
def __init__(self):
addApiView('renamer.scan', self.scanView, docs = {
'desc': 'For the renamer to check for new files to rename in a folder',
'params': {
'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'},
'media_folder': {'desc': 'Optional: The folder of the media to scan. Keep empty for default renamer folder.'},
'files': {'desc': 'Optional: Provide the release files if more releases are in the same media_folder, delimited with a \'|\'. Note that no dedicated release folder is expected for releases with one file.'},
'base_folder': {'desc': 'Optional: The folder to find releases in. Leave empty for default folder.'},
'downloader': {'desc': 'Optional: The downloader the release has been downloaded with. \'download_id\' is required with this option.'},
'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in media_folder. \'downloader\' is required with this option.'},
'status': {'desc': 'Optional: The status of the release: \'completed\' (default) or \'seeding\''},
},
})
addEvent('renamer.scan', self.scan)
addEvent('renamer.check_snatched', self.checkSnatched)
addEvent('app.load', self.scan)
addEvent('app.load', self.setCrons)
# Enable / disable interval
addEvent('setting.save.renamer.enabled.after', self.setCrons)
addEvent('setting.save.renamer.run_every.after', self.setCrons)
addEvent('setting.save.renamer.force_every.after', self.setCrons)
def setCrons(self):
fireEvent('schedule.remove', 'renamer.check_snatched')
if self.isEnabled() and self.conf('run_every') > 0:
fireEvent('schedule.interval', 'renamer.check_snatched', self.checkSnatched, minutes = self.conf('run_every'), single = True)
fireEvent('schedule.remove', 'renamer.check_snatched_forced')
if self.isEnabled() and self.conf('force_every') > 0:
fireEvent('schedule.interval', 'renamer.check_snatched_forced', self.scan, hours = self.conf('force_every'), single = True)
return True
def scanView(self, **kwargs):
async = tryInt(kwargs.get('async', 0))
base_folder = kwargs.get('base_folder')
media_folder = sp(kwargs.get('media_folder'))
# Backwards compatibility, to be removed after a few versions :)
if not media_folder:
media_folder = sp(kwargs.get('movie_folder'))
downloader = kwargs.get('downloader')
download_id = kwargs.get('download_id')
files = '|'.join([sp(filename) for filename in splitString(kwargs.get('files'), '|')])
status = kwargs.get('status', 'completed')
release_download = None
if not base_folder and media_folder:
release_download = {'folder': media_folder}
release_download.update({'id': download_id, 'downloader': downloader, 'status': status, 'files': files} if download_id else {})
fire_handle = fireEvent if not async else fireEventAsync
fire_handle('renamer.scan', base_folder = base_folder, release_download = release_download)
return {
'success': True
}
def scan(self, base_folder = None, release_download = None):
if not release_download: release_download = {}
if self.isDisabled():
return
if self.renaming_started is True:
log.info('Renamer is already running, if you see this often, check the logs above for errors.')
return
if not base_folder:
base_folder = self.conf('from')
from_folder = sp(self.conf('from'))
to_folder = sp(self.conf('to'))
# Get media folder to process
media_folder = release_download.get('folder')
# Quality order for calculation quality priority
quality_order = fireEvent('quality.order', single = True)
# Get all folders that should not be processed
no_process = [to_folder]
cat_list = fireEvent('category.all', single = True) or []
no_process.extend([item['destination'] for item in cat_list])
try:
if Env.setting('library', section = 'manage').strip():
no_process.extend([sp(manage_folder) for manage_folder in splitString(Env.setting('library', section = 'manage'), '::')])
except:
pass
# Check to see if the no_process folders are inside the "from" folder.
if not os.path.isdir(base_folder) or not os.path.isdir(to_folder):
log.error('Both the "To" and "From" folder have to exist.')
return
else:
for item in no_process:
if isSubFolder(item, base_folder):
log.error('To protect your data, the media libraries can\'t be inside of or the same as the "from" folder.')
return
# Check to see if the no_process folders are inside the provided media_folder
if media_folder and not os.path.isdir(media_folder):
log.debug('The provided media folder %s does not exist. Trying to find it in the \'from\' folder.', media_folder)
# Update to the from folder
if len(splitString(release_download.get('files'), '|')) == 1:
new_media_folder = from_folder
else:
new_media_folder = os.path.join(from_folder, os.path.basename(media_folder))
if not os.path.isdir(new_media_folder):
log.error('The provided media folder %s does not exist and could also not be found in the \'from\' folder.', media_folder)
return
# Update the files
new_files = [os.path.join(new_media_folder, os.path.relpath(filename, media_folder)) for filename in splitString(release_download.get('files'), '|')]
if new_files and not os.path.isfile(new_files[0]):
log.error('The provided media folder %s does not exist and its files could also not be found in the \'from\' folder.', media_folder)
return
# Update release_download info to the from folder
log.debug('Release %s found in the \'from\' folder.', media_folder)
release_download['folder'] = new_media_folder
release_download['files'] = '|'.join(new_files)
media_folder = new_media_folder
if media_folder:
for item in no_process:
if isSubFolder(item, media_folder):
log.error('To protect your data, the media libraries can\'t be inside of or the same as the provided media folder.')
return
# Make sure a checkSnatched marked all downloads/seeds as such
if not release_download and self.conf('run_every') > 0:
self.checkSnatched(fire_scan = False)
self.renaming_started = True
# make sure the media folder name is included in the search
folder = None
files = []
if media_folder:
log.info('Scanning media folder %s...', media_folder)
folder = os.path.dirname(media_folder)
if release_download.get('files', ''):
files = splitString(release_download['files'], '|')
# If there is only one file in the torrent, the downloader did not create a subfolder
if len(files) == 1:
folder = media_folder
else:
# Get all files from the specified folder
try:
for root, folders, names in scandir.walk(media_folder):
files.extend([sp(os.path.join(root, name)) for name in names])
except:
log.error('Failed getting files from %s: %s', (media_folder, traceback.format_exc()))
db = get_db()
# Extend the download info with info stored in the downloaded release
release_download = self.extendReleaseDownload(release_download)
# Unpack any archives
extr_files = None
if self.conf('unrar'):
folder, media_folder, files, extr_files = self.extractFiles(folder = folder, media_folder = media_folder, files = files,
cleanup = self.conf('cleanup') and not self.downloadIsTorrent(release_download))
groups = fireEvent('scanner.scan', folder = folder if folder else base_folder,
files = files, release_download = release_download, return_ignored = False, single = True) or []
folder_name = self.conf('folder_name')
file_name = self.conf('file_name')
trailer_name = self.conf('trailer_name')
nfo_name = self.conf('nfo_name')
separator = self.conf('separator')
# Tag release folder as failed_rename in case no groups were found. This prevents check_snatched from removing the release from the downloader.
if not groups and self.statusInfoComplete(release_download):
self.tagRelease(release_download = release_download, tag = 'failed_rename')
for group_identifier in groups:
group = groups[group_identifier]
rename_files = {}
remove_files = []
remove_releases = []
media_title = getTitle(group)
# Add _UNKNOWN_ if no library item is connected
if not group.get('info') or not media_title:
self.tagRelease(group = group, tag = 'unknown')
continue
# Rename the files using the library data
else:
# Media not in library, add it first
if not group['media'].get('_id'):
group['media'] = fireEvent('movie.add', params = {
'identifier': group['identifier'],
'profile_id': None
}, search_after = False, status = 'done')
else:
group['media'] = fireEvent('movie.update_info', identifier = group['media']['identifier'], single = True)
if not group['media'] or not group['media'].get('_id'):
log.error('Could not rename, no library item to work with: %s', group_identifier)
continue
media = group['media']
media_title = getTitle(media)
# Overwrite destination when set in category
destination = to_folder
category_label = ''
if media.get('category_id'):
try:
category = db.get('id', media['category_id'])
category_label = category['label']
if category['destination'] and len(category['destination']) > 0 and category['destination'] != 'None':
destination = category['destination']
log.debug('Setting category destination for "%s": %s' % (media_title, destination))
else:
log.debug('No category destination found for "%s"' % media_title)
except:
log.error('Failed getting category label: %s', traceback.format_exc())
# Find subtitle for renaming
group['before_rename'] = []
fireEvent('renamer.before', group)
# Add extracted files to the before_rename list
if extr_files:
group['before_rename'].extend(extr_files)
# Remove weird chars from movie name
movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', media_title)
# Put 'The' at the end
name_the = movie_name
if movie_name[:4].lower() == 'the ':
name_the = movie_name[4:] + ', The'
replacements = {
'ext': 'mkv',
'namethe': name_the.strip(),
'thename': movie_name.strip(),
'year': media['info']['year'],
'first': name_the[0].upper(),
'quality': group['meta_data']['quality']['label'],
'quality_type': group['meta_data']['quality_type'],
'video': group['meta_data'].get('video'),
'audio': group['meta_data'].get('audio'),
'group': group['meta_data']['group'],
'source': group['meta_data']['source'],
'resolution_width': group['meta_data'].get('resolution_width'),
'resolution_height': group['meta_data'].get('resolution_height'),
'audio_channels': group['meta_data'].get('audio_channels'),
'imdb_id': group['identifier'],
'cd': '',
'cd_nr': '',
'mpaa': media['info'].get('mpaa', ''),
'category': category_label,
}
for file_type in group['files']:
# Move nfo depending on settings
if file_type is 'nfo' and not self.conf('rename_nfo'):
log.debug('Skipping, renaming of %s disabled', file_type)
for current_file in group['files'][file_type]:
if self.conf('cleanup') and (not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)):
remove_files.append(current_file)
continue
# Subtitle extra
if file_type is 'subtitle_extra':
continue
# Move other files
multiple = len(group['files'][file_type]) > 1 and not group['is_dvd']
cd = 1 if multiple else 0
for current_file in sorted(list(group['files'][file_type])):
current_file = sp(current_file)
# Original filename
replacements['original'] = os.path.splitext(os.path.basename(current_file))[0]
replacements['original_folder'] = fireEvent('scanner.remove_cptag', group['dirname'], single = True)
# Extension
replacements['ext'] = getExt(current_file)
# cd #
replacements['cd'] = ' cd%d' % cd if multiple else ''
replacements['cd_nr'] = cd if multiple else ''
# Naming
final_folder_name = self.doReplace(folder_name, replacements, folder = True)
final_file_name = self.doReplace(file_name, replacements)
replacements['filename'] = final_file_name[:-(len(getExt(final_file_name)) + 1)]
# Meta naming
if file_type is 'trailer':
final_file_name = self.doReplace(trailer_name, replacements, remove_multiple = True)
elif file_type is 'nfo':
final_file_name = self.doReplace(nfo_name, replacements, remove_multiple = True)
# Seperator replace
if separator:
final_file_name = final_file_name.replace(' ', separator)
# Move DVD files (no structure renaming)
if group['is_dvd'] and file_type is 'movie':
found = False
for top_dir in ['video_ts', 'audio_ts', 'bdmv', 'certificate']:
has_string = current_file.lower().find(os.path.sep + top_dir + os.path.sep)
if has_string >= 0:
structure_dir = current_file[has_string:].lstrip(os.path.sep)
rename_files[current_file] = os.path.join(destination, final_folder_name, structure_dir)
found = True
break
if not found:
log.error('Could not determine dvd structure for: %s', current_file)
# Do rename others
else:
if file_type is 'leftover':
if self.conf('move_leftover'):
rename_files[current_file] = os.path.join(destination, final_folder_name, os.path.basename(current_file))
elif file_type not in ['subtitle']:
rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name)
# Check for extra subtitle files
if file_type is 'subtitle':
remove_multiple = False
if len(group['files']['movie']) == 1:
remove_multiple = True
sub_langs = group['subtitle_language'].get(current_file, [])
# rename subtitles with or without language
sub_name = self.doReplace(file_name, replacements, remove_multiple = remove_multiple)
rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
rename_extras = self.getRenameExtras(
extra_type = 'subtitle_extra',
replacements = replacements,
folder_name = folder_name,
file_name = file_name,
destination = destination,
group = group,
current_file = current_file,
remove_multiple = remove_multiple,
)
# Don't add language if multiple languages in 1 subtitle file
if len(sub_langs) == 1:
sub_name = sub_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext']))
rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
rename_files = mergeDicts(rename_files, rename_extras)
# Filename without cd etc
elif file_type is 'movie':
rename_extras = self.getRenameExtras(
extra_type = 'movie_extra',
replacements = replacements,
folder_name = folder_name,
file_name = file_name,
destination = destination,
group = group,
current_file = current_file
)
rename_files = mergeDicts(rename_files, rename_extras)
group['filename'] = self.doReplace(file_name, replacements, remove_multiple = True)[:-(len(getExt(final_file_name)) + 1)]
group['destination_dir'] = os.path.join(destination, final_folder_name)
if multiple:
cd += 1
# Before renaming, remove the lower quality files
remove_leftovers = True
# Mark movie "done" once it's found the quality with the finish check
try:
if media.get('status') == 'active' and media.get('profile_id'):
profile = db.get('id', media['profile_id'])
if group['meta_data']['quality']['identifier'] in profile.get('qualities', []):
nr = profile['qualities'].index(group['meta_data']['quality']['identifier'])
finish = profile['finish'][nr]
if finish:
mdia = db.get('id', media['_id'])
mdia['status'] = 'done'
mdia['last_edit'] = int(time.time())
db.update(mdia)
except Exception as e:
log.error('Failed marking movie finished: %s', (traceback.format_exc()))
# Go over current movie releases
for release in db.run('release', 'for_media', media['_id']):
# When a release already exists
if release.get('status') == 'done':
release_order = quality_order.index(release['quality'])
group_quality_order = quality_order.index(group['meta_data']['quality']['identifier'])
# This is where CP removes older, lesser quality releases
if release_order > group_quality_order:
log.info('Removing lesser quality %s for %s.', (media_title, release.get('quality')))
for file_type in release.get('files', {}):
for release_file in release['files'][file_type]:
remove_files.append(release_file)
remove_releases.append(release)
# Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
elif release_order == group_quality_order:
log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (media_title, release.get('quality')))
for file_type in release.get('files', {}):
for release_file in release['files'][file_type]:
remove_files.append(release_file)
remove_releases.append(release)
# Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan
else:
log.info('Better quality release already exists for %s, with quality %s', (media_title, release.get('quality')))
# Add exists tag to the .ignore file
self.tagRelease(group = group, tag = 'exists')
# Notify on rename fail
download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (media_title, group['meta_data']['quality']['label'], release.get('identifier'))
fireEvent('movie.renaming.canceled', message = download_message, data = group)
remove_leftovers = False
break
elif release.get('status') in ['snatched', 'seeding']:
if release_download and release_download.get('release_id'):
if release_download['release_id'] == release['_id']:
if release_download['status'] == 'completed':
# Set the release to downloaded
fireEvent('release.update_status', release['_id'], status = 'downloaded', single = True)
elif release_download['status'] == 'seeding':
# Set the release to seeding
fireEvent('release.update_status', release['_id'], status = 'seeding', single = True)
elif release.get('identifier') == group['meta_data']['quality']['identifier']:
# Set the release to downloaded
fireEvent('release.update_status', release['_id'], status = 'downloaded', single = True)
# Remove leftover files
if not remove_leftovers: # Don't remove anything
break
log.debug('Removing leftover files')
for current_file in group['files']['leftover']:
if self.conf('cleanup') and not self.conf('move_leftover') and \
(not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)):
remove_files.append(current_file)
# Remove files
delete_folders = []
for src in remove_files:
if rename_files.get(src):
log.debug('Not removing file that will be renamed: %s', src)
continue
log.info('Removing "%s"', src)
try:
src = sp(src)
if os.path.isfile(src):
os.remove(src)
parent_dir = os.path.dirname(src)
if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and \
not isSubFolder(destination, parent_dir) and not isSubFolder(media_folder, parent_dir) and \
not isSubFolder(parent_dir, base_folder):
delete_folders.append(parent_dir)
except:
log.error('Failed removing %s: %s', (src, traceback.format_exc()))
self.tagRelease(group = group, tag = 'failed_remove')
# Delete leftover folder from older releases
for delete_folder in delete_folders:
try:
self.deleteEmptyFolder(delete_folder, show_error = False)
except Exception as e:
log.error('Failed to delete folder: %s %s', (e, traceback.format_exc()))
# Rename all files marked
group['renamed_files'] = []
failed_rename = False
for src in rename_files:
if rename_files[src]:
dst = rename_files[src]
log.info('Renaming "%s" to "%s"', (src, dst))
# Create dir
self.makeDir(os.path.dirname(dst))
try:
self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(release_download) or self.fileIsAdded(src, group))
group['renamed_files'].append(dst)
except:
log.error('Failed renaming the file "%s" : %s', (os.path.basename(src), traceback.format_exc()))
failed_rename = True
break
# If renaming failed tag the release folder as failed and continue with next group. Note that all old files have already been deleted.
if failed_rename:
self.tagRelease(group = group, tag = 'failed_rename')
continue
# If renaming succeeded, make sure it is not tagged as failed (scanner didn't return a group, but a download_ID was provided in an earlier attempt)
else:
self.untagRelease(group = group, tag = 'failed_rename')
# Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent
if self.movieInFromFolder(media_folder) and self.downloadIsTorrent(release_download):
self.tagRelease(group = group, tag = 'renamed_already')
# Remove matching releases
for release in remove_releases:
log.debug('Removing release %s', release.identifier)
try:
db.delete(release)
except:
log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc()))
if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(release_download):
if media_folder:
# Delete the movie folder
group_folder = media_folder
else:
# Delete the first empty subfolder in the tree relative to the 'from' folder
group_folder = sp(os.path.join(base_folder, os.path.relpath(group['parentdir'], base_folder).split(os.path.sep)[0]))
try:
log.info('Deleting folder: %s', group_folder)
self.deleteEmptyFolder(group_folder)
except:
log.error('Failed removing %s: %s', (group_folder, traceback.format_exc()))
# Notify on download, search for trailers etc
download_message = 'Downloaded %s (%s)' % (media_title, replacements['quality'])
try:
fireEvent('renamer.after', message = download_message, group = group, in_order = True)
except:
log.error('Failed firing (some) of the renamer.after events: %s', traceback.format_exc())
# Break if CP wants to shut down
if self.shuttingDown():
break
self.renaming_started = False
def getRenameExtras(self, extra_type = '', replacements = None, folder_name = '', file_name = '', destination = '', group = None, current_file = '', remove_multiple = False):
if not group: group = {}
if not replacements: replacements = {}
replacements = replacements.copy()
rename_files = {}
def test(s):
return current_file[:-len(replacements['ext'])] in sp(s)
for extra in set(filter(test, group['files'][extra_type])):
replacements['ext'] = getExt(extra)
final_folder_name = self.doReplace(folder_name, replacements, remove_multiple = remove_multiple, folder = True)
final_file_name = self.doReplace(file_name, replacements, remove_multiple = remove_multiple)
rename_files[extra] = os.path.join(destination, final_folder_name, final_file_name)
return rename_files
# This adds a file to ignore / tag a release so it is ignored later
def tagRelease(self, tag, group = None, release_download = None):
if not tag:
return
text = """This file is from CouchPotato
It has marked this release as "%s"
This file hides the release from the renamer
Remove it if you want it to be renamed (again, or at least let it try again)
""" % tag
tag_files = []
# Tag movie files if they are known
if isinstance(group, dict):
tag_files = [sorted(list(group['files']['movie']))[0]]
elif isinstance(release_download, dict):
# Tag download_files if they are known
if release_download['files']:
tag_files = splitString(release_download['files'], '|')
# Tag all files in release folder
else:
for root, folders, names in scandir.walk(release_download['folder']):
tag_files.extend([os.path.join(root, name) for name in names])
for filename in tag_files:
# Dont tag .ignore files
if os.path.splitext(filename)[1] == '.ignore':
continue
tag_filename = '%s.%s.ignore' % (os.path.splitext(filename)[0], tag)
if not os.path.isfile(tag_filename):
self.createFile(tag_filename, text)
def untagRelease(self, group = None, release_download = None, tag = ''):
if not release_download:
return
tag_files = []
folder = None
# Tag movie files if they are known
if isinstance(group, dict):
tag_files = [sorted(list(group['files']['movie']))[0]]
folder = group['parentdir']
if not group.get('dirname') or not os.path.isdir(folder):
return False
elif isinstance(release_download, dict):
# Untag download_files if they are known
if release_download['files']:
tag_files = splitString(release_download['files'], '|')
# Untag all files in release folder
else:
for root, folders, names in scandir.walk(release_download['folder']):
tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore'])
folder = release_download['folder']
if not os.path.isdir(folder):
return False
if not folder:
return False
# Find all .ignore files in folder
ignore_files = []
for root, dirnames, filenames in scandir.walk(folder):
ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag))
# Match all found ignore files with the tag_files and delete if found
for tag_file in tag_files:
ignore_file = fnmatch.filter(ignore_files, fnEscape('%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*')))
for filename in ignore_file:
try:
os.remove(filename)
except:
log.debug('Unable to remove ignore file: %s. Error: %s.' % (filename, traceback.format_exc()))
def hastagRelease(self, release_download, tag = ''):
if not release_download:
return False
folder = release_download['folder']
if not os.path.isdir(folder):
return False
tag_files = []
ignore_files = []
# Find tag on download_files if they are known
if release_download['files']:
tag_files = splitString(release_download['files'], '|')
# Find tag on all files in release folder
else:
for root, folders, names in scandir.walk(release_download['folder']):
tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore'])
# Find all .ignore files in folder
for root, dirnames, filenames in scandir.walk(folder):
ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag))
# Match all found ignore files with the tag_files and return True found
for tag_file in tag_files:
ignore_file = fnmatch.filter(ignore_files, fnEscape('%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*')))
if ignore_file:
return True
return False
def moveFile(self, old, dest, forcemove = False):
dest = ss(dest)
try:
if forcemove or self.conf('file_action') not in ['copy', 'link']:
try:
shutil.move(old, dest)
except:
if os.path.exists(dest):
log.error('Successfully moved file "%s", but something went wrong: %s', (dest, traceback.format_exc()))
os.unlink(old)
else:
raise
elif self.conf('file_action') == 'copy':
shutil.copy(old, dest)
elif self.conf('file_action') == 'link':
# First try to hardlink
try:
log.debug('Hardlinking file "%s" to "%s"...', (old, dest))
link(old, dest)
except:
# Try to simlink next
log.debug('Couldn\'t hardlink file "%s" to "%s". Simlinking instead. Error: %s.', (old, dest, traceback.format_exc()))
shutil.copy(old, dest)
try:
symlink(dest, old + '.link')
os.unlink(old)
os.rename(old + '.link', old)
except:
log.error('Couldn\'t symlink file "%s" to "%s". Copied instead. Error: %s. ', (old, dest, traceback.format_exc()))
try:
os.chmod(dest, Env.getPermission('file'))
if os.name == 'nt' and self.conf('ntfs_permission'):
os.popen('icacls "' + dest + '"* /reset /T')
except:
log.error('Failed setting permissions for file: %s, %s', (dest, traceback.format_exc(1)))
except:
log.error('Couldn\'t move file "%s" to "%s": %s', (old, dest, traceback.format_exc()))
raise
return True
def doReplace(self, string, replacements, remove_multiple = False, folder = False):
"""
replace confignames with the real thing
"""
replacements = replacements.copy()
if remove_multiple:
replacements['cd'] = ''
replacements['cd_nr'] = ''
replaced = toUnicode(string)
for x, r in replacements.items():
if x in ['thename', 'namethe']:
continue
if r is not None:
replaced = replaced.replace(six.u('<%s>') % toUnicode(x), toUnicode(r))
else:
#If information is not available, we don't want the tag in the filename
replaced = replaced.replace('<' + x + '>', '')
replaced = self.replaceDoubles(replaced.lstrip('. '))
for x, r in replacements.items():
if x in ['thename', 'namethe']:
replaced = replaced.replace(six.u('<%s>') % toUnicode(x), toUnicode(r))
replaced = re.sub(r"[\x00:\*\?\"<>\|]", '', replaced)
sep = self.conf('foldersep') if folder else self.conf('separator')
return replaced.replace(' ', ' ' if not sep else sep)
def replaceDoubles(self, string):
replaces = [
('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '),
('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'),
]
for r in replaces:
reg, replace_with = r
string = re.sub(reg, replace_with, string)
return string
def deleteEmptyFolder(self, folder, show_error = True):
folder = sp(folder)
loge = log.error if show_error else log.debug
for root, dirs, files in scandir.walk(folder):
for dir_name in dirs:
full_path = os.path.join(root, dir_name)
if len(os.listdir(full_path)) == 0:
try:
os.rmdir(full_path)
except:
loge('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
try:
os.rmdir(folder)
except:
loge('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc()))
def checkSnatched(self, fire_scan = True):
if self.checking_snatched:
log.debug('Already checking snatched')
return False
self.checking_snatched = True
try:
db = get_db()
rels = list(db.run('release', 'with_status', ['snatched', 'seeding', 'missing']))
if not rels:
#No releases found that need status checking
self.checking_snatched = False
return True
# Collect all download information with the download IDs from the releases
download_ids = []
no_status_support = []
try:
for rel in rels:
if not rel.get('download_info'): continue
if rel['download_info'].get('id') and rel['download_info'].get('downloader'):
download_ids.append(rel['download_info'])
ds = rel['download_info'].get('status_support')
if ds is False or ds == 'False':
no_status_support.append(ss(rel['download_info'].get('downloader')))
except:
log.error('Error getting download IDs from database')
self.checking_snatched = False
return False
release_downloads = fireEvent('download.status', download_ids, merge = True) if download_ids else []
if len(no_status_support) > 0:
log.debug('Download status functionality is not implemented for one of the active downloaders: %s', no_status_support)
if not release_downloads:
if fire_scan:
self.scan()
self.checking_snatched = False
return True
scan_releases = []
scan_required = False
log.debug('Checking status snatched releases...')
try:
for rel in rels:
movie_dict = db.get('id', rel.get('media_id'))
download_info = rel.get('download_info')
if not isinstance(download_info, dict):
log.error('Faulty release found without any info, ignoring.')
fireEvent('release.update_status', rel.get('_id'), status = 'ignored', single = True)
continue
# Check if download ID is available
if not download_info.get('id') or not download_info.get('downloader'):
log.debug('Download status functionality is not implemented for downloader (%s) of release %s.', (download_info.get('downloader', 'unknown'), rel['info']['name']))
scan_required = True
# Continue with next release
continue
# Find release in downloaders
nzbname = self.createNzbName(rel['info'], movie_dict)
found_release = False
for release_download in release_downloads:
found_release = False
if download_info.get('id'):
if release_download['id'] == download_info['id'] and release_download['downloader'] == download_info['downloader']:
log.debug('Found release by id: %s', release_download['id'])
found_release = True
break
else:
if release_download['name'] == nzbname or rel['info']['name'] in release_download['name'] or getImdb(release_download['name']) == movie_dict['identifier']:
log.debug('Found release by release name or imdb ID: %s', release_download['name'])
found_release = True
break
if not found_release:
log.info('%s not found in downloaders', nzbname)
#Check status if already missing and for how long, if > 1 week, set to ignored else to missing
if rel.get('status') == 'missing':
if rel.last_edit < int(time.time()) - 7 * 24 * 60 * 60:
fireEvent('release.update_status', rel.get('_id'), status = 'ignored', single = True)
else:
# Set the release to missing
fireEvent('release.update_status', rel.get('_id'), status = 'missing', single = True)
# Continue with next release
continue
# Log that we found the release
timeleft = 'N/A' if release_download['timeleft'] == -1 else release_download['timeleft']
log.debug('Found %s: %s, time to go: %s', (release_download['name'], release_download['status'].upper(), timeleft))
# Check status of release
if release_download['status'] == 'busy':
# Set the release to snatched if it was missing before
fireEvent('release.update_status', rel.get('_id'), status = 'snatched', single = True)
# Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading
if self.movieInFromFolder(release_download['folder']):
self.tagRelease(release_download = release_download, tag = 'downloading')
elif release_download['status'] == 'seeding':
#If linking setting is enabled, process release
if self.conf('file_action') != 'move' and not rel.get('status') == 'seeding' and self.statusInfoComplete(release_download):
log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (release_download['name'], release_download['seed_ratio']))
# Remove the downloading tag
self.untagRelease(release_download = release_download, tag = 'downloading')
# Scan and set the torrent to paused if required
release_download.update({'pause': True, 'scan': True, 'process_complete': False})
scan_releases.append(release_download)
else:
#let it seed
log.debug('%s is seeding with ratio: %s', (release_download['name'], release_download['seed_ratio']))
# Set the release to seeding
fireEvent('release.update_status', rel.get('_id'), status = 'seeding', single = True)
elif release_download['status'] == 'failed':
# Set the release to failed
fireEvent('release.update_status', rel.get('_id'), status = 'failed', single = True)
fireEvent('download.remove_failed', release_download, single = True)
if self.conf('next_on_failed'):
fireEvent('movie.searcher.try_next_release', media_id = rel.get('media_id'))
elif release_download['status'] == 'completed':
log.info('Download of %s completed!', release_download['name'])
#Make sure the downloader sent over a path to look in
if self.statusInfoComplete(release_download):
# If the release has been seeding, process now the seeding is done
if rel.get('status') == 'seeding':
if self.conf('file_action') != 'move':
# Set the release to done as the movie has already been renamed
fireEvent('release.update_status', rel.get('_id'), status = 'downloaded', single = True)
# Allow the downloader to clean-up
release_download.update({'pause': False, 'scan': False, 'process_complete': True})
scan_releases.append(release_download)
else:
# Scan and Allow the downloader to clean-up
release_download.update({'pause': False, 'scan': True, 'process_complete': True})
scan_releases.append(release_download)
else:
# Set the release to snatched if it was missing before
fireEvent('release.update_status', rel.get('_id'), status = 'snatched', single = True)
# Remove the downloading tag
self.untagRelease(release_download = release_download, tag = 'downloading')
# Scan and Allow the downloader to clean-up
release_download.update({'pause': False, 'scan': True, 'process_complete': True})
scan_releases.append(release_download)
else:
scan_required = True
except:
log.error('Failed checking for release in downloader: %s', traceback.format_exc())
# The following can either be done here, or inside the scanner if we pass it scan_items in one go
for release_download in scan_releases:
# Ask the renamer to scan the item
if release_download['scan']:
if release_download['pause'] and self.conf('file_action') == 'link':
fireEvent('download.pause', release_download = release_download, pause = True, single = True)
self.scan(release_download = release_download)
if release_download['pause'] and self.conf('file_action') == 'link':
fireEvent('download.pause', release_download = release_download, pause = False, single = True)
if release_download['process_complete']:
#First make sure the files were succesfully processed
if not self.hastagRelease(release_download = release_download, tag = 'failed_rename'):
# Remove the seeding tag if it exists
self.untagRelease(release_download = release_download, tag = 'renamed_already')
# Ask the downloader to process the item
fireEvent('download.process_complete', release_download = release_download, single = True)
if fire_scan and (scan_required or len(no_status_support) > 0):
self.scan()
self.checking_snatched = False
return True
except:
log.error('Failed checking snatched: %s', traceback.format_exc())
self.checking_snatched = False
return False
def extendReleaseDownload(self, release_download):
rls = None
db = get_db()
if release_download and release_download.get('id'):
try:
rls = db.get('release_download', '%s_%s' % (release_download.get('downloader'), release_download.get('id')), with_doc = True)['doc']
except:
log.error('Download ID %s from downloader %s not found in releases', (release_download.get('id'), release_download.get('downloader')))
if rls:
media = db.get('id', rls['media_id'])
release_download.update({
'imdb_id': media['identifier'],
'quality': rls['quality'],
'protocol': rls.get('info', {}).get('protocol') or rls.get('info', {}).get('type'),
'release_id': rls['_id'],
})
return release_download
def downloadIsTorrent(self, release_download):
return release_download and release_download.get('protocol') in ['torrent', 'torrent_magnet']
def fileIsAdded(self, src, group):
if not group or not group.get('before_rename'):
return False
return src in group['before_rename']
def statusInfoComplete(self, release_download):
return release_download.get('id') and release_download.get('downloader') and release_download.get('folder')
def movieInFromFolder(self, media_folder):
return media_folder and isSubFolder(media_folder, sp(self.conf('from'))) or not media_folder
def extractFiles(self, folder = None, media_folder = None, files = None, cleanup = False):
if not files: files = []
# RegEx for finding rar files
archive_regex = '(?P<file>^(?P<base>(?:(?!\.part\d+\.rar$).)*)\.(?:(?:part0*1\.)?rar)$)'
restfile_regex = '(^%s\.(?:part(?!0*1\.rar$)\d+\.rar$|[rstuvw]\d+$))'
extr_files = []
from_folder = sp(self.conf('from'))
# Check input variables
if not folder:
folder = from_folder
check_file_date = True
if media_folder:
check_file_date = False
if not files:
for root, folders, names in scandir.walk(folder):
files.extend([sp(os.path.join(root, name)) for name in names])
# Find all archive files
archives = [re.search(archive_regex, name).groupdict() for name in files if re.search(archive_regex, name)]
#Extract all found archives
for archive in archives:
# Check if it has already been processed by CPS
if self.hastagRelease(release_download = {'folder': os.path.dirname(archive['file']), 'files': archive['file']}):
continue
# Find all related archive files
archive['files'] = [name for name in files if re.search(restfile_regex % re.escape(archive['base']), name)]
archive['files'].append(archive['file'])
# Check if archive is fresh and maybe still copying/moving/downloading, ignore files newer than 1 minute
if check_file_date:
files_too_new, time_string = self.checkFilesChanged(archive['files'])
if files_too_new:
log.info('Archive seems to be still copying/moving/downloading or just copied/moved/downloaded (created on %s), ignoring for now: %s', (time_string, os.path.basename(archive['file'])))
continue
log.info('Archive %s found. Extracting...', os.path.basename(archive['file']))
try:
rar_handle = RarFile(archive['file'])
extr_path = os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder))
self.makeDir(extr_path)
for packedinfo in rar_handle.infolist():
if not packedinfo.isdir and not os.path.isfile(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))):
log.debug('Extracting %s...', packedinfo.filename)
rar_handle.extract(condition = [packedinfo.index], path = extr_path, withSubpath = False, overwrite = False)
extr_files.append(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename))))
del rar_handle
except Exception as e:
log.error('Failed to extract %s: %s %s', (archive['file'], e, traceback.format_exc()))
continue
# Delete the archive files
for filename in archive['files']:
if cleanup:
try:
os.remove(filename)
except Exception as e:
log.error('Failed to remove %s: %s %s', (filename, e, traceback.format_exc()))
continue
files.remove(filename)
# Move the rest of the files and folders if any files are extracted to the from folder (only if folder was provided)
if extr_files and folder != from_folder:
for leftoverfile in list(files):
move_to = os.path.join(from_folder, os.path.relpath(leftoverfile, folder))
try:
self.makeDir(os.path.dirname(move_to))
self.moveFile(leftoverfile, move_to, cleanup)
except Exception as e:
log.error('Failed moving left over file %s to %s: %s %s', (leftoverfile, move_to, e, traceback.format_exc()))
# As we probably tried to overwrite the nfo file, check if it exists and then remove the original
if os.path.isfile(move_to):
if cleanup:
log.info('Deleting left over file %s instead...', leftoverfile)
os.unlink(leftoverfile)
else:
continue
files.remove(leftoverfile)
extr_files.append(move_to)
if cleanup:
# Remove all left over folders
log.debug('Removing old movie folder %s...', media_folder)
self.deleteEmptyFolder(media_folder)
media_folder = os.path.join(from_folder, os.path.relpath(media_folder, folder))
folder = from_folder
if extr_files:
files.extend(extr_files)
# Cleanup files and folder if media_folder was not provided
if not media_folder:
files = []
folder = None
return folder, media_folder, files, extr_files