diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 08be4bd..9e24d91 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -66,36 +66,36 @@ class Downloader(Provider): def getAllDownloadStatus(self): return - def _removeFailed(self, item): + def _removeFailed(self, release_download): if self.isDisabled(manual = True, data = {}): return - if item and item.get('downloader') == self.getName(): + if release_download and release_download.get('downloader') == self.getName(): if self.conf('delete_failed'): - return self.removeFailed(item) + return self.removeFailed(release_download) return False return - def removeFailed(self, item): + def removeFailed(self, release_download): return - def _processComplete(self, item): + def _processComplete(self, release_download): if self.isDisabled(manual = True, data = {}): return - if item and item.get('downloader') == self.getName(): + if release_download and release_download.get('downloader') == self.getName(): if self.conf('remove_complete', default = False): - return self.processComplete(item = item, delete_files = self.conf('delete_files', default = False)) + return self.processComplete(release_download = release_download, delete_files = self.conf('delete_files', default = False)) return False return - def processComplete(self, item, delete_files): + def processComplete(self, release_download, delete_files): return - def isCorrectProtocol(self, item_protocol): - is_correct = item_protocol in self.protocol + def isCorrectProtocol(self, protocol): + is_correct = protocol in self.protocol if not is_correct: log.debug("Downloader doesn't support this protocol") @@ -151,20 +151,20 @@ class Downloader(Provider): (d_manual and manual or d_manual is False) and \ (not data or self.isCorrectProtocol(data.get('protocol'))) - def _pause(self, item, pause = True): + def _pause(self, release_download, pause = True): if self.isDisabled(manual = True, data = {}): return - if item and item.get('downloader') == self.getName(): - self.pause(item, pause) + if release_download and release_download.get('downloader') == self.getName(): + self.pause(release_download, pause) return True return False - def pause(self, item, pause): + def pause(self, release_download, pause): return -class StatusList(list): +class ReleaseDownloadList(list): provider = None @@ -173,7 +173,7 @@ class StatusList(list): self.provider = provider self.kwargs = kwargs - super(StatusList, self).__init__() + super(ReleaseDownloadList, self).__init__() def extend(self, results): for r in results: @@ -181,7 +181,7 @@ class StatusList(list): def append(self, result): new_result = self.fillResult(result) - super(StatusList, self).append(new_result) + super(ReleaseDownloadList, self).append(new_result) def fillResult(self, result): @@ -190,6 +190,7 @@ class StatusList(list): 'status': 'busy', 'downloader': self.provider.getName(), 'folder': '', + 'files': '', } return mergeDicts(defaults, result) diff --git a/couchpotato/core/downloaders/deluge/main.py b/couchpotato/core/downloaders/deluge/main.py index 580ed7f..1edb9ad 100644 --- a/couchpotato/core/downloaders/deluge/main.py +++ b/couchpotato/core/downloaders/deluge/main.py @@ -1,5 +1,5 @@ from base64 import b64encode -from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import isInt, ss from couchpotato.core.helpers.variable import tryFloat from couchpotato.core.logger import CPLog @@ -85,14 +85,10 @@ class Deluge(Downloader): log.debug('Checking Deluge download status.') - if not os.path.isdir(Env.setting('from', 'renamer')): - log.error('Renamer "from" folder doesn\'t to exist.') - return - if not self.connect(): return False - statuses = StatusList(self) + release_downloads = ReleaseDownloadList(self) queue = self.drpc.get_alltorrents() @@ -101,50 +97,55 @@ class Deluge(Downloader): return False for torrent_id in queue: - item = queue[torrent_id] - log.debug('name=%s / id=%s / save_path=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (item['name'], item['hash'], item['save_path'], item['move_completed_path'], item['hash'], item['progress'], item['state'], item['eta'], item['ratio'], item['stop_ratio'], item['is_seed'], item['is_finished'], item['paused'])) + torrent = queue[torrent_id] + log.debug('name=%s / id=%s / save_path=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused'])) # Deluge has no easy way to work out if a torrent is stalled or failing. #status = 'failed' status = 'busy' - if item['is_seed'] and tryFloat(item['ratio']) < tryFloat(item['stop_ratio']): - # We have item['seeding_time'] to work out what the seeding time is, but we do not + if torrent['is_seed'] and tryFloat(torrent['ratio']) < tryFloat(torrent['stop_ratio']): + # We have torrent['seeding_time'] to work out what the seeding time is, but we do not # have access to the downloader seed_time, as with deluge we have no way to pass it # when the torrent is added. So Deluge will only look at the ratio. # See above comment in download(). status = 'seeding' - elif item['is_seed'] and item['is_finished'] and item['paused'] and item['state'] == 'Paused': + elif torrent['is_seed'] and torrent['is_finished'] and torrent['paused'] and torrent['state'] == 'Paused': status = 'completed' - download_dir = item['save_path'] - if item['move_on_completed']: - download_dir = item['move_completed_path'] + download_dir = torrent['save_path'] + if torrent['move_on_completed']: + download_dir = torrent['move_completed_path'] + + torrent_files = [] + for file_item in torrent['files']: + torrent_files.append(os.path.join(download_dir, file_item['path'])) - statuses.append({ - 'id': item['hash'], - 'name': item['name'], + release_downloads.append({ + 'id': torrent['hash'], + 'name': torrent['name'], 'status': status, - 'original_status': item['state'], - 'seed_ratio': item['ratio'], - 'timeleft': str(timedelta(seconds = item['eta'])), - 'folder': ss(os.path.join(download_dir, item['name'])), + 'original_status': torrent['state'], + 'seed_ratio': torrent['ratio'], + 'timeleft': str(timedelta(seconds = torrent['eta'])), + 'folder': ss(download_dir) if len(torrent_files) == 1 else ss(os.path.join(download_dir, torrent['name'])), + 'files': ss('|'.join(torrent_files)), }) - return statuses + return release_downloads - def pause(self, item, pause = True): + def pause(self, release_download, pause = True): if pause: - return self.drpc.pause_torrent([item['id']]) + return self.drpc.pause_torrent([release_download['id']]) else: - return self.drpc.resume_torrent([item['id']]) + return self.drpc.resume_torrent([release_download['id']]) - def removeFailed(self, item): - log.info('%s failed downloading, deleting...', item['name']) - return self.drpc.remove_torrent(item['id'], True) + def removeFailed(self, release_download): + log.info('%s failed downloading, deleting...', release_download['name']) + return self.drpc.remove_torrent(release_download['id'], True) - def processComplete(self, item, delete_files = False): - log.debug('Requesting Deluge to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) - return self.drpc.remove_torrent(item['id'], remove_local_data = delete_files) + def processComplete(self, release_download, delete_files = False): + log.debug('Requesting Deluge to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) + return self.drpc.remove_torrent(release_download['id'], remove_local_data = delete_files) class DelugeRPC(object): diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index b7cf026..3de1f12 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -1,5 +1,5 @@ from base64 import standard_b64encode -from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import ss from couchpotato.core.helpers.variable import tryInt, md5 from couchpotato.core.logger import CPLog @@ -99,60 +99,60 @@ class NZBGet(Downloader): log.error('Failed getting data: %s', traceback.format_exc(1)) return False - statuses = StatusList(self) + release_downloads = ReleaseDownloadList(self) - for item in groups: - log.debug('Found %s in NZBGet download queue', item['NZBFilename']) + for nzb in groups: + log.debug('Found %s in NZBGet download queue', nzb['NZBFilename']) try: - nzb_id = [param['Value'] for param in item['Parameters'] if param['Name'] == 'couchpotato'][0] + nzb_id = [param['Value'] for param in nzb['Parameters'] if param['Name'] == 'couchpotato'][0] except: - nzb_id = item['NZBID'] + nzb_id = nzb['NZBID'] timeleft = -1 try: - if item['ActiveDownloads'] > 0 and item['DownloadRate'] > 0 and not (status['DownloadPaused'] or status['Download2Paused']): - timeleft = str(timedelta(seconds = item['RemainingSizeMB'] / status['DownloadRate'] * 2 ^ 20)) + if nzb['ActiveDownloads'] > 0 and nzb['DownloadRate'] > 0 and not (status['DownloadPaused'] or status['Download2Paused']): + timeleft = str(timedelta(seconds = nzb['RemainingSizeMB'] / status['DownloadRate'] * 2 ^ 20)) except: pass - statuses.append({ + release_downloads.append({ 'id': nzb_id, - 'name': item['NZBFilename'], - 'original_status': 'DOWNLOADING' if item['ActiveDownloads'] > 0 else 'QUEUED', + 'name': nzb['NZBFilename'], + 'original_status': 'DOWNLOADING' if nzb['ActiveDownloads'] > 0 else 'QUEUED', # Seems to have no native API function for time left. This will return the time left after NZBGet started downloading this item 'timeleft': timeleft, }) - for item in queue: # 'Parameters' is not passed in rpc.postqueue - log.debug('Found %s in NZBGet postprocessing queue', item['NZBFilename']) - statuses.append({ - 'id': item['NZBID'], - 'name': item['NZBFilename'], - 'original_status': item['Stage'], + for nzb in queue: # 'Parameters' is not passed in rpc.postqueue + log.debug('Found %s in NZBGet postprocessing queue', nzb['NZBFilename']) + release_downloads.append({ + 'id': nzb['NZBID'], + 'name': nzb['NZBFilename'], + 'original_status': nzb['Stage'], 'timeleft': str(timedelta(seconds = 0)) if not status['PostPaused'] else -1, }) - for item in history: - log.debug('Found %s in NZBGet history. ParStatus: %s, ScriptStatus: %s, Log: %s', (item['NZBFilename'] , item['ParStatus'], item['ScriptStatus'] , item['Log'])) + for nzb in history: + log.debug('Found %s in NZBGet history. ParStatus: %s, ScriptStatus: %s, Log: %s', (nzb['NZBFilename'] , nzb['ParStatus'], nzb['ScriptStatus'] , nzb['Log'])) try: - nzb_id = [param['Value'] for param in item['Parameters'] if param['Name'] == 'couchpotato'][0] + nzb_id = [param['Value'] for param in nzb['Parameters'] if param['Name'] == 'couchpotato'][0] except: - nzb_id = item['NZBID'] - statuses.append({ + nzb_id = nzb['NZBID'] + release_downloads.append({ 'id': nzb_id, - 'name': item['NZBFilename'], - 'status': 'completed' if item['ParStatus'] in ['SUCCESS','NONE'] and item['ScriptStatus'] in ['SUCCESS','NONE'] else 'failed', - 'original_status': item['ParStatus'] + ', ' + item['ScriptStatus'], + 'name': nzb['NZBFilename'], + 'status': 'completed' if nzb['ParStatus'] in ['SUCCESS','NONE'] and nzb['ScriptStatus'] in ['SUCCESS','NONE'] else 'failed', + 'original_status': nzb['ParStatus'] + ', ' + nzb['ScriptStatus'], 'timeleft': str(timedelta(seconds = 0)), - 'folder': ss(item['DestDir']) + 'folder': ss(nzb['DestDir']) }) - return statuses + return release_downloads - def removeFailed(self, item): + def removeFailed(self, release_download): - log.info('%s failed downloading, deleting...', item['name']) + log.info('%s failed downloading, deleting...', release_download['name']) url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')} @@ -179,7 +179,7 @@ class NZBGet(Downloader): for hist in history: for param in hist['Parameters']: - if param['Name'] == 'couchpotato' and param['Value'] == item['id']: + if param['Name'] == 'couchpotato' and param['Value'] == release_download['id']: nzb_id = hist['ID'] path = hist['DestDir'] diff --git a/couchpotato/core/downloaders/nzbvortex/main.py b/couchpotato/core/downloaders/nzbvortex/main.py index a652f11..983a8fd 100644 --- a/couchpotato/core/downloaders/nzbvortex/main.py +++ b/couchpotato/core/downloaders/nzbvortex/main.py @@ -1,5 +1,5 @@ from base64 import b64encode -from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import tryUrlencode, ss from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog @@ -33,7 +33,7 @@ class NZBVortex(Downloader): self.call('nzb/add', params = {'file': (ss(nzb_filename), filedata)}, multipart = True) raw_statuses = self.call('nzb') - nzb_id = [item['id'] for item in raw_statuses.get('nzbs', []) if item['name'] == nzb_filename][0] + nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if nzb['name'] == nzb_filename][0] return self.downloadReturnId(nzb_id) except: log.error('Something went wrong sending the NZB file: %s', traceback.format_exc()) @@ -43,33 +43,33 @@ class NZBVortex(Downloader): raw_statuses = self.call('nzb') - statuses = StatusList(self) - for item in raw_statuses.get('nzbs', []): + release_downloads = ReleaseDownloadList(self) + for nzb in raw_statuses.get('nzbs', []): # Check status status = 'busy' - if item['state'] == 20: + if nzb['state'] == 20: status = 'completed' - elif item['state'] in [21, 22, 24]: + elif nzb['state'] in [21, 22, 24]: status = 'failed' - statuses.append({ - 'id': item['id'], - 'name': item['uiTitle'], + release_downloads.append({ + 'id': nzb['id'], + 'name': nzb['uiTitle'], 'status': status, - 'original_status': item['state'], + 'original_status': nzb['state'], 'timeleft':-1, - 'folder': ss(item['destinationPath']), + 'folder': ss(nzb['destinationPath']), }) - return statuses + return release_downloads - def removeFailed(self, item): + def removeFailed(self, release_download): - log.info('%s failed downloading, deleting...', item['name']) + log.info('%s failed downloading, deleting...', release_download['name']) try: - self.call('nzb/%s/cancel' % item['id']) + self.call('nzb/%s/cancel' % release_download['id']) except: log.error('Failed deleting: %s', traceback.format_exc(0)) return False diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index c8c323f..566787b 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -1,6 +1,6 @@ from base64 import b16encode, b32decode from bencode import bencode, bdecode -from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import ss from couchpotato.core.logger import CPLog from datetime import timedelta @@ -151,37 +151,42 @@ class rTorrent(Downloader): try: torrents = self.rt.get_torrents() - statuses = StatusList(self) + release_downloads = ReleaseDownloadList(self) + + for torrent in torrents: + torrent_files = [] + for file_item in torrent.get_files(): + torrent_files.append(os.path.join(torrent.directory, file_item.path)) - for item in torrents: status = 'busy' - if item.complete: - if item.active: + if torrent.complete: + if torrent.active: status = 'seeding' else: status = 'completed' - statuses.append({ - 'id': item.info_hash, - 'name': item.name, + release_downloads.append({ + 'id': torrent.info_hash, + 'name': torrent.name, 'status': status, - 'seed_ratio': item.ratio, - 'original_status': item.state, - 'timeleft': str(timedelta(seconds = float(item.left_bytes) / item.down_rate)) if item.down_rate > 0 else -1, - 'folder': ss(item.directory) + 'seed_ratio': torrent.ratio, + 'original_status': torrent.state, + 'timeleft': str(timedelta(seconds = float(torrent.left_bytes) / torrent.down_rate)) if torrent.down_rate > 0 else -1, + 'folder': ss(torrent.directory), + 'files': ss('|'.join(torrent_files)) }) - return statuses + return release_downloads except Exception, err: log.error('Failed to get status from rTorrent: %s', err) return False - def pause(self, download_info, pause = True): + def pause(self, release_download, pause = True): if not self.connect(): return False - torrent = self.rt.find_torrent(download_info['id']) + torrent = self.rt.find_torrent(release_download['id']) if torrent is None: return False @@ -189,23 +194,23 @@ class rTorrent(Downloader): return torrent.pause() return torrent.resume() - def removeFailed(self, item): - log.info('%s failed downloading, deleting...', item['name']) - return self.processComplete(item, delete_files = True) + def removeFailed(self, release_download): + log.info('%s failed downloading, deleting...', release_download['name']) + return self.processComplete(release_download, delete_files = True) - def processComplete(self, item, delete_files): + def processComplete(self, release_download, delete_files): log.debug('Requesting rTorrent to remove the torrent %s%s.', - (item['name'], ' and cleanup the downloaded files' if delete_files else '')) + (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) if not self.connect(): return False - torrent = self.rt.find_torrent(item['id']) + torrent = self.rt.find_torrent(release_download['id']) if torrent is None: return False torrent.erase() # just removes the torrent, doesn't delete data if delete_files: - shutil.rmtree(item['folder'], True) + shutil.rmtree(release_download['folder'], True) return True diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 41f9f70..dcbe245 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -1,4 +1,4 @@ -from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import tryUrlencode, ss from couchpotato.core.helpers.variable import cleanHost, mergeDicts from couchpotato.core.logger import CPLog @@ -6,6 +6,7 @@ from couchpotato.environment import Env from datetime import timedelta from urllib2 import URLError import json +import os import traceback log = CPLog(__name__) @@ -86,58 +87,58 @@ class Sabnzbd(Downloader): log.error('Failed getting history json: %s', traceback.format_exc(1)) return False - statuses = StatusList(self) + release_downloads = ReleaseDownloadList(self) # Get busy releases - for item in queue.get('slots', []): + for nzb in queue.get('slots', []): status = 'busy' - if 'ENCRYPTED / ' in item['filename']: + if 'ENCRYPTED / ' in nzb['filename']: status = 'failed' - statuses.append({ - 'id': item['nzo_id'], - 'name': item['filename'], + release_downloads.append({ + 'id': nzb['nzo_id'], + 'name': nzb['filename'], 'status': status, - 'original_status': item['status'], - 'timeleft': item['timeleft'] if not queue['paused'] else -1, + 'original_status': nzb['status'], + 'timeleft': nzb['timeleft'] if not queue['paused'] else -1, }) # Get old releases - for item in history.get('slots', []): + for nzb in history.get('slots', []): status = 'busy' - if item['status'] == 'Failed' or (item['status'] == 'Completed' and item['fail_message'].strip()): + if nzb['status'] == 'Failed' or (nzb['status'] == 'Completed' and nzb['fail_message'].strip()): status = 'failed' - elif item['status'] == 'Completed': + elif nzb['status'] == 'Completed': status = 'completed' - statuses.append({ - 'id': item['nzo_id'], - 'name': item['name'], + release_downloads.append({ + 'id': nzb['nzo_id'], + 'name': nzb['name'], 'status': status, - 'original_status': item['status'], + 'original_status': nzb['status'], 'timeleft': str(timedelta(seconds = 0)), - 'folder': ss(item['storage']), + 'folder': os.path.dirname(ss(nzb['storage'])) if os.path.isfile(ss(nzb['storage'])) else ss(nzb['storage']), }) - return statuses + return release_downloads - def removeFailed(self, item): + def removeFailed(self, release_download): - log.info('%s failed downloading, deleting...', item['name']) + log.info('%s failed downloading, deleting...', release_download['name']) try: self.call({ 'mode': 'queue', 'name': 'delete', 'del_files': '1', - 'value': item['id'] + 'value': release_download['id'] }, use_json = False) self.call({ 'mode': 'history', 'name': 'delete', 'del_files': '1', - 'value': item['id'] + 'value': release_download['id'] }, use_json = False) except: log.error('Failed deleting: %s', traceback.format_exc(0)) @@ -145,15 +146,15 @@ class Sabnzbd(Downloader): return True - def processComplete(self, item, delete_files = False): - log.debug('Requesting SabNZBd to remove the NZB %s.', item['name']) + def processComplete(self, release_download, delete_files = False): + log.debug('Requesting SabNZBd to remove the NZB %s.', release_download['name']) try: self.call({ 'mode': 'history', 'name': 'delete', 'del_files': '0', - 'value': item['id'] + 'value': release_download['id'] }, use_json = False) except: log.error('Failed removing: %s', traceback.format_exc(0)) diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 1c35996..10cb954 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -1,9 +1,8 @@ from base64 import b64encode -from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import isInt, ss from couchpotato.core.helpers.variable import tryInt, tryFloat from couchpotato.core.logger import CPLog -from couchpotato.environment import Env from datetime import timedelta import httplib import json @@ -89,10 +88,10 @@ class Transmission(Downloader): if not self.connect(): return False - statuses = StatusList(self) + release_downloads = ReleaseDownloadList(self) return_params = { - 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isStalled', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit'] + 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isStalled', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit', 'files'] } queue = self.trpc.get_alltorrents(return_params) @@ -100,47 +99,48 @@ class Transmission(Downloader): log.debug('Nothing in queue or error') return False - for item in queue['torrents']: + for torrent in queue['torrents']: log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / isFinished=%s', - (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], item['isFinished'])) + (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent['eta'], torrent['uploadRatio'], torrent['isFinished'])) - if not os.path.isdir(Env.setting('from', 'renamer')): - log.error('Renamer "from" folder doesn\'t to exist.') - return + torrent_files = [] + for file_item in torrent['files']: + torrent_files.append(os.path.normpath(os.path.join(ss(torrent['downloadDir']), ss(file_item['name'])))) status = 'busy' - if item['isStalled'] and self.conf('stalled_as_failed'): + if torrent['isStalled'] and self.conf('stalled_as_failed'): status = 'failed' - elif item['status'] == 0 and item['percentDone'] == 1: + elif torrent['status'] == 0 and torrent['percentDone'] == 1: status = 'completed' - elif item['status'] in [5, 6]: + elif torrent['status'] in [5, 6]: status = 'seeding' - statuses.append({ - 'id': item['hashString'], - 'name': item['name'], + release_downloads.append({ + 'id': torrent['hashString'], + 'name': torrent['name'], 'status': status, - 'original_status': item['status'], - 'seed_ratio': item['uploadRatio'], - 'timeleft': str(timedelta(seconds = item['eta'])), - 'folder': ss(os.path.join(item['downloadDir'], item['name'])), + 'original_status': torrent['status'], + 'seed_ratio': torrent['uploadRatio'], + 'timeleft': str(timedelta(seconds = torrent['eta'])), + 'folder': os.path.normpath(ss(torrent['downloadDir'])) if len(torrent_files) == 1 else os.path.normpath(os.path.join(ss(torrent['downloadDir']), ss(torrent['name']))), + 'files': ss('|'.join(torrent_files)) }) - return statuses + return release_downloads - def pause(self, item, pause = True): + def pause(self, release_download, pause = True): if pause: - return self.trpc.stop_torrent(item['id']) + return self.trpc.stop_torrent(release_download['id']) else: - return self.trpc.start_torrent(item['id']) + return self.trpc.start_torrent(release_download['id']) - def removeFailed(self, item): - log.info('%s failed downloading, deleting...', item['name']) - return self.trpc.remove_torrent(item['id'], True) + def removeFailed(self, release_download): + log.info('%s failed downloading, deleting...', release_download['name']) + return self.trpc.remove_torrent(release_download['id'], True) - def processComplete(self, item, delete_files = False): - log.debug('Requesting Transmission to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) - return self.trpc.remove_torrent(item['id'], delete_files) + def processComplete(self, release_download, delete_files = False): + log.debug('Requesting Transmission to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) + return self.trpc.remove_torrent(release_download['id'], delete_files) class TransmissionRPC(object): diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index d5262e2..b0b9c89 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -1,6 +1,6 @@ from base64 import b16encode, b32decode from bencode import bencode as benc, bdecode -from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import isInt, ss from couchpotato.core.helpers.variable import tryInt, tryFloat from couchpotato.core.logger import CPLog @@ -102,39 +102,6 @@ class uTorrent(Downloader): if self.conf('paused', default = 0): self.utorrent_api.pause_torrent(torrent_hash) - count = 0 - while True: - - count += 1 - # Check if torrent is saved in subfolder of torrent name - getfiles_data = self.utorrent_api.get_files(torrent_hash) - - torrent_files = json.loads(getfiles_data) - if torrent_files.get('error'): - log.error('Error getting data from uTorrent: %s', torrent_files.get('error')) - return False - - if (torrent_files.get('files') and len(torrent_files['files'][1]) > 0) or count > 60: - break - - time.sleep(1) - - # Torrent has only one file, so uTorrent wont create a folder for it - if len(torrent_files['files'][1]) == 1: - # Remove torrent and try again - self.utorrent_api.remove_torrent(torrent_hash, remove_data = True) - - # Send request to uTorrent - if data.get('protocol') == 'torrent_magnet': - self.utorrent_api.add_torrent_uri(torrent_filename, data.get('url'), add_folder = True) - else: - self.utorrent_api.add_torrent_file(torrent_filename, filedata, add_folder = True) - - # Change settings of added torrent - self.utorrent_api.set_torrent(torrent_hash, torrent_params) - if self.conf('paused', default = 0): - self.utorrent_api.pause_torrent(torrent_hash) - return self.downloadReturnId(torrent_hash) def getAllDownloadStatus(self): @@ -144,7 +111,7 @@ class uTorrent(Downloader): if not self.connect(): return False - statuses = StatusList(self) + release_downloads = ReleaseDownloadList(self) data = self.utorrent_api.get_status() if not data: @@ -161,52 +128,74 @@ class uTorrent(Downloader): return False # Get torrents - for item in queue['torrents']: + for torrent in queue['torrents']: + + #Get files of the torrent + torrent_files = [] + try: + torrent_files = json.loads(self.utorrent_api.get_files(torrent[0])) + torrent_files = [os.path.join(torrent[26], torrent_file[0]) for torrent_file in torrent_files['files'][1]] + except: + log.debug('Failed getting files from torrent: %s', torrent[2]) + + status_flags = { + "STARTED" : 1, + "CHECKING" : 2, + "CHECK-START" : 4, + "CHECKED" : 8, + "ERROR" : 16, + "PAUSED" : 32, + "QUEUED" : 64, + "LOADED" : 128 + } - # item[21] = Paused | Downloading | Seeding | Finished status = 'busy' - if 'Finished' in item[21]: - status = 'completed' - self.removeReadOnly(item[26]) - elif 'Seeding' in item[21]: + if (torrent[1] & status_flags["STARTED"] or torrent[1] & status_flags["QUEUED"]) and torrent[4] == 1000: status = 'seeding' - self.removeReadOnly(item[26]) - - statuses.append({ - 'id': item[0], - 'name': item[2], - 'status': status, - 'seed_ratio': float(item[7]) / 1000, - 'original_status': item[1], - 'timeleft': str(timedelta(seconds = item[10])), - 'folder': ss(item[26]), + elif (torrent[1] & status_flags["ERROR"]): + status = 'failed' + elif torrent[4] == 1000: + status = 'completed' + + if not status == 'busy': + self.removeReadOnly(torrent_files) + + release_downloads.append({ + 'id': torrent[0], + 'name': torrent[2], + 'status': status, + 'seed_ratio': float(torrent[7]) / 1000, + 'original_status': torrent[1], + 'timeleft': str(timedelta(seconds = torrent[10])), + 'folder': ss(torrent[26]), + 'files': ss('|'.join(torrent_files)) }) - return statuses + return release_downloads - def pause(self, item, pause = True): + def pause(self, release_download, pause = True): if not self.connect(): return False - return self.utorrent_api.pause_torrent(item['id'], pause) + return self.utorrent_api.pause_torrent(release_download['id'], pause) - def removeFailed(self, item): - log.info('%s failed downloading, deleting...', item['name']) + def removeFailed(self, release_download): + log.info('%s failed downloading, deleting...', release_download['name']) if not self.connect(): return False - return self.utorrent_api.remove_torrent(item['id'], remove_data = True) + return self.utorrent_api.remove_torrent(release_download['id'], remove_data = True) - def processComplete(self, item, delete_files = False): - log.debug('Requesting uTorrent to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) + def processComplete(self, release_download, delete_files = False): + log.debug('Requesting uTorrent to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) if not self.connect(): return False - return self.utorrent_api.remove_torrent(item['id'], remove_data = delete_files) + return self.utorrent_api.remove_torrent(release_download['id'], remove_data = delete_files) - def removeReadOnly(self, folder): - #Removes all read-only flags in a folder - if folder and os.path.isdir(folder): - for root, folders, filenames in os.walk(folder): - for filename in filenames: - os.chmod(os.path.join(root, filename), stat.S_IWRITE) + def removeReadOnly(self, files): + #Removes all read-on ly flags in a for all files + for filepath in files: + if os.path.isfile(filepath): + #Windows only needs S_IWRITE, but we bitwise-or with current perms to preserve other permission bits on Linux + os.chmod(filepath, stat.S_IWRITE | os.stat(filepath).st_mode) class uTorrentAPI(object): @@ -304,13 +293,13 @@ class uTorrentAPI(object): utorrent_settings = json.loads(self._request(action)) # Create settings dict - for item in utorrent_settings['settings']: - if item[1] == 0: # int - settings_dict[item[0]] = int(item[2] if not item[2].strip() == '' else '0') - elif item[1] == 1: # bool - settings_dict[item[0]] = True if item[2] == 'true' else False - elif item[1] == 2: # string - settings_dict[item[0]] = item[2] + for setting in utorrent_settings['settings']: + if setting[1] == 0: # int + settings_dict[setting[0]] = int(setting[2] if not setting[2].strip() == '' else '0') + elif setting[1] == 1: # bool + settings_dict[setting[0]] = True if setting[2] == 'true' else False + elif setting[1] == 2: # string + settings_dict[setting[0]] = setting[2] #log.debug('uTorrent settings: %s', settings_dict) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 9df23d9..8d58ad2 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -270,7 +270,7 @@ class Release(Plugin): } def updateStatus(self, id, status = None): - if not status: return + if not status: return False db = get_session() @@ -281,11 +281,20 @@ class Release(Plugin): for info in rel.info: item[info.identifier] = info.value + if rel.files: + for file_item in rel.files: + if file_item.type.identifier == 'movie': + release_name = os.path.basename(file_item.path) + break + else: + release_name = item['name'] #update status in Db - log.debug('Marking release %s as %s', (item['name'], status.get("label"))) + log.debug('Marking release %s as %s', (release_name, status.get("label"))) rel.status_id = status.get('id') rel.last_edit = int(time.time()) db.commit() #Update all movie info as there is no release update function fireEvent('notify.frontend', type = 'release.update_status.%s' % rel.id, data = status.get('id')) + + return True \ No newline at end of file diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 5f5473e..2c62d74 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -3,7 +3,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ - getImdb, link, symlink, tryInt + getImdb, link, symlink, tryInt, splitString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, File, Profile, Release, \ @@ -31,8 +31,9 @@ class Renamer(Plugin): 'params': { 'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'}, 'movie_folder': {'desc': 'Optional: The folder of the movie to scan. Keep empty for default renamer folder.'}, - 'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'}, - 'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'}, + 'downloader' : {'desc': 'Optional: The downloader the release has been downloaded with. \'download_id\' is required with this option.'}, + 'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in movie_folder. \'downloader\' is required with this option.'}, + 'status': {'desc': 'Optional: The status of the release: \'completed\' (default) or \'seeding\''}, }, }) @@ -65,20 +66,21 @@ class Renamer(Plugin): movie_folder = kwargs.get('movie_folder') downloader = kwargs.get('downloader') download_id = kwargs.get('download_id') + status = kwargs.get('status', 'completed') - download_info = {'folder': movie_folder} if movie_folder else None - if download_info: - download_info.update({'id': download_id, 'downloader': downloader} if download_id else {}) + release_download = {'folder': movie_folder} if movie_folder else None + if release_download: + release_download.update({'id': download_id, 'downloader': downloader, 'status': status} if download_id else {}) fire_handle = fireEvent if not async else fireEventAsync - fire_handle('renamer.scan', download_info) + fire_handle('renamer.scan', release_download) return { 'success': True } - def scan(self, download_info = None): + def scan(self, release_download = None): if self.isDisabled(): return @@ -87,22 +89,41 @@ class Renamer(Plugin): log.info('Renamer is already running, if you see this often, check the logs above for errors.') return - movie_folder = download_info and download_info.get('folder') + # Get movie folder to process + movie_folder = release_download and release_download.get('folder') - # Check to see if the "to" folder is inside the "from" folder. - if movie_folder and not os.path.isdir(movie_folder) or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): - l = log.debug if movie_folder else log.error - l('Both the "To" and "From" have to exist.') - return - elif self.conf('from') in self.conf('to'): - log.error('The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.') + # Get all folders that should not be processed + no_process = [self.conf('to')] + cat_list = fireEvent('category.all', single = True) or [] + no_process.extend([item['destination'] for item in cat_list]) + try: + if Env.setting('library', section = 'manage').strip(): + no_process.extend(splitString(Env.setting('library', section = 'manage'), '::')) + except: + pass + + # Check to see if the no_process folders are inside the "from" folder. + if not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): + log.error('Both the "To" and "From" have to exist.') return - elif movie_folder and movie_folder in [self.conf('to'), self.conf('from')]: - log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.') + else: + for item in no_process: + if self.conf('from') in item: + log.error('To protect your data, the movie libraries can\'t be inside of or the same as the "from" folder.') + return + + # Check to see if the no_process folders are inside the provided movie_folder + if movie_folder and not os.path.isdir(movie_folder): + log.error('The provided movie folder %s does not exist.', movie_folder) return + elif movie_folder: + for item in no_process: + if movie_folder in item: + log.error('To protect your data, the movie libraries can\'t be inside of or the same as the provided movie folder.') + return # Make sure a checkSnatched marked all downloads/seeds as such - if not download_info and self.conf('run_every') > 0: + if not release_download and self.conf('run_every') > 0: fireEvent('renamer.check_snatched') self.renaming_started = True @@ -115,26 +136,33 @@ class Renamer(Plugin): movie_folder = movie_folder.rstrip(os.path.sep) folder = os.path.dirname(movie_folder) - # Get all files from the specified folder - try: - for root, folders, names in os.walk(movie_folder): - files.extend([os.path.join(root, name) for name in names]) - except: - log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc())) + if release_download['files']: + files = release_download['files'].split('|') + + # If there is only one file in the torrent, the downloader did not create a subfolder + if len(files) == 1: + folder = movie_folder + else: + # Get all files from the specified folder + try: + for root, folders, names in os.walk(movie_folder): + files.extend([os.path.join(root, name) for name in names]) + except: + log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc())) db = get_session() # Extend the download info with info stored in the downloaded release - download_info = self.extendDownloadInfo(download_info) + release_download = self.extendReleaseDownload(release_download) # Unpack any archives extr_files = None if self.conf('unrar'): folder, movie_folder, files, extr_files = self.extractFiles(folder = folder, movie_folder = movie_folder, files = files, - cleanup = self.conf('cleanup') and not self.downloadIsTorrent(download_info)) + cleanup = self.conf('cleanup') and not self.downloadIsTorrent(release_download)) groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), - files = files, download_info = download_info, return_ignored = False, single = True) or [] + files = files, release_download = release_download, return_ignored = False, single = True) or [] folder_name = self.conf('folder_name') file_name = self.conf('file_name') @@ -142,9 +170,9 @@ class Renamer(Plugin): nfo_name = self.conf('nfo_name') separator = self.conf('separator') - # Statusses - done_status, active_status, downloaded_status, snatched_status = \ - fireEvent('status.get', ['done', 'active', 'downloaded', 'snatched'], single = True) + # Statuses + done_status, active_status, downloaded_status, snatched_status, seeding_status = \ + fireEvent('status.get', ['done', 'active', 'downloaded', 'snatched', 'seeding'], single = True) for group_identifier in groups: @@ -157,7 +185,7 @@ class Renamer(Plugin): # Add _UNKNOWN_ if no library item is connected if not group['library'] or not movie_title: - self.tagDir(group, 'unknown') + self.tagRelease(group = group, tag = 'unknown') continue # Rename the files using the library data else: @@ -195,7 +223,7 @@ class Renamer(Plugin): if extr_files: group['before_rename'].extend(extr_files) - # Remove weird chars from moviename + # Remove weird chars from movie name movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', movie_title) # Put 'The' at the end @@ -231,7 +259,7 @@ class Renamer(Plugin): if file_type is 'nfo' and not self.conf('rename_nfo'): log.debug('Skipping, renaming of %s disabled', file_type) for current_file in group['files'][file_type]: - if self.conf('cleanup') and (not self.downloadIsTorrent(download_info) or self.fileIsAdded(current_file, group)): + if self.conf('cleanup') and (not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)): remove_files.append(current_file) continue @@ -391,7 +419,7 @@ class Renamer(Plugin): log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label)) # Add exists tag to the .ignore file - self.tagDir(group, 'exists') + self.tagRelease(group = group, tag = 'exists') # Notify on rename fail download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label) @@ -399,10 +427,20 @@ class Renamer(Plugin): remove_leftovers = False break - elif release.status_id is snatched_status.get('id'): - if release.quality.id is group['meta_data']['quality']['id']: - # Set the release to downloaded - fireEvent('release.update_status', release.id, status = downloaded_status, single = True) + + elif release.status_id in [snatched_status.get('id'), seeding_status.get('id')]: + if release_download and release_download.get('rls_id'): + if release_download['rls_id'] == release.id: + if release_download['status'] == 'completed': + # Set the release to downloaded + fireEvent('release.update_status', release.id, status = downloaded_status, single = True) + elif release_download['status'] == 'seeding': + # Set the release to seeding + fireEvent('release.update_status', release.id, status = seeding_status, single = True) + + elif release.quality.id is group['meta_data']['quality']['id']: + # Set the release to downloaded + fireEvent('release.update_status', release.id, status = downloaded_status, single = True) # Remove leftover files if not remove_leftovers: # Don't remove anything @@ -411,7 +449,7 @@ class Renamer(Plugin): log.debug('Removing leftover files') for current_file in group['files']['leftover']: if self.conf('cleanup') and not self.conf('move_leftover') and \ - (not self.downloadIsTorrent(download_info) or self.fileIsAdded(current_file, group)): + (not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)): remove_files.append(current_file) # Remove files @@ -437,7 +475,7 @@ class Renamer(Plugin): except: log.error('Failed removing %s: %s', (src, traceback.format_exc())) - self.tagDir(group, 'failed_remove') + self.tagRelease(group = group, tag = 'failed_remove') # Delete leftover folder from older releases for delete_folder in delete_folders: @@ -457,15 +495,15 @@ class Renamer(Plugin): self.makeDir(os.path.dirname(dst)) try: - self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(download_info) or self.fileIsAdded(src, group)) + self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(release_download) or self.fileIsAdded(src, group)) group['renamed_files'].append(dst) except: log.error('Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) - self.tagDir(group, 'failed_rename') + self.tagRelease(group = group, tag = 'failed_rename') # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent - if self.movieInFromFolder(movie_folder) and self.downloadIsTorrent(download_info): - self.tagDir(group, 'renamed_already') + if self.movieInFromFolder(movie_folder) and self.downloadIsTorrent(release_download): + self.tagRelease(group = group, tag = 'renamed_already') # Remove matching releases for release in remove_releases: @@ -475,7 +513,7 @@ class Renamer(Plugin): except: log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc())) - if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(download_info): + if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(release_download): if movie_folder: # Delete the movie folder group_folder = movie_folder @@ -522,18 +560,9 @@ class Renamer(Plugin): return rename_files # This adds a file to ignore / tag a release so it is ignored later - def tagDir(self, group, tag): - - ignore_file = None - if isinstance(group, dict): - for movie_file in sorted(list(group['files']['movie'])): - ignore_file = '%s.%s.ignore' % (os.path.splitext(movie_file)[0], tag) - break - else: - if not os.path.isdir(group) or not tag: - return - ignore_file = os.path.join(group, '%s.ignore' % tag) - + def tagRelease(self, tag, group = None, release_download = None): + if not tag: + return text = """This file is from CouchPotato It has marked this release as "%s" @@ -541,25 +570,88 @@ This file hides the release from the renamer Remove it if you want it to be renamed (again, or at least let it try again) """ % tag - if ignore_file: - self.createFile(ignore_file, text) + tag_files = [] - def untagDir(self, folder, tag = ''): - if not os.path.isdir(folder): + # Tag movie files if they are known + if isinstance(group, dict): + tag_files = [sorted(list(group['files']['movie']))[0]] + + elif isinstance(release_download, dict): + # Tag download_files if they are known + if release_download['files']: + tag_files = release_download['files'].split('|') + + # Tag all files in release folder + else: + for root, folders, names in os.walk(release_download['folder']): + tag_files.extend([os.path.join(root, name) for name in names]) + + for filename in tag_files: + tag_filename = '%s.%s.ignore' % (os.path.splitext(filename)[0], tag) + if not os.path.isfile(tag_filename): + self.createFile(tag_filename, text) + + def untagRelease(self, release_download, tag = ''): + if not release_download: return - # Remove any .ignore files + tag_files = [] + + folder = release_download['folder'] + if not os.path.isdir(folder): + return False + + # Untag download_files if they are known + if release_download['files']: + tag_files = release_download['files'].split('|') + + # Untag all files in release folder + else: + for root, folders, names in os.walk(release_download['folder']): + tag_files.extend([os.path.join(root, name) for name in names if not os.path.splitext(name)[1] == '.ignore']) + + # Find all .ignore files in folder + ignore_files = [] for root, dirnames, filenames in os.walk(folder): - for filename in fnmatch.filter(filenames, '*%s.ignore' % tag): - os.remove((os.path.join(root, filename))) + ignore_files.extend(fnmatch.filter([os.path.join(root, filename) for filename in filenames], '*%s.ignore' % tag)) + + # Match all found ignore files with the tag_files and delete if found + for tag_file in tag_files: + ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*')) + for filename in ignore_file: + try: + os.remove(filename) + except: + log.debug('Unable to remove ignore file: %s. Error: %s.' % (filename, traceback.format_exc())) + + def hastagRelease(self, release_download, tag = ''): + if not release_download: + return False - def hastagDir(self, folder, tag = ''): + folder = release_download['folder'] if not os.path.isdir(folder): return False - # Find any .ignore files + tag_files = [] + ignore_files = [] + + # Find tag on download_files if they are known + if release_download['files']: + tag_files = release_download['files'].split('|') + + # Find tag on all files in release folder + else: + for root, folders, names in os.walk(release_download['folder']): + tag_files.extend([os.path.join(root, name) for name in names if not os.path.splitext(name)[1] == '.ignore']) + + # Find all .ignore files in folder for root, dirnames, filenames in os.walk(folder): - if fnmatch.filter(filenames, '*%s.ignore' % tag): + ignore_files.extend(fnmatch.filter([os.path.join(root, filename) for filename in filenames], '*%s.ignore' % tag)) + + # Match all found ignore files with the tag_files and return True found + for tag_file in tag_files: + ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*')) + if ignore_file: return True return False @@ -578,7 +670,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) link(old, dest) except: # Try to simlink next - log.debug('Couldn\'t hardlink file "%s" to "%s". Simlinking instead. Error: %s. ', (old, dest, traceback.format_exc())) + log.debug('Couldn\'t hardlink file "%s" to "%s". Simlinking instead. Error: %s.', (old, dest, traceback.format_exc())) shutil.copy(old, dest) try: symlink(dest, old + '.link') @@ -671,14 +763,14 @@ Remove it if you want it to be renamed (again, or at least let it try again) Release.status_id.in_([snatched_status.get('id'), seeding_status.get('id'), missing_status.get('id')]) ).all() - scan_items = [] + scan_releases = [] scan_required = False if rels: log.debug('Checking status snatched releases...') - statuses = fireEvent('download.status', merge = True) - if not statuses: + release_downloads = fireEvent('download.status', merge = True) + if not release_downloads: log.debug('Download status functionality is not implemented for active downloaders.') scan_required = True else: @@ -691,85 +783,82 @@ Remove it if you want it to be renamed (again, or at least let it try again) nzbname = self.createNzbName(rel_dict['info'], movie_dict) found = False - for item in statuses: + for release_download in release_downloads: found_release = False if rel_dict['info'].get('download_id'): - if item['id'] == rel_dict['info']['download_id'] and item['downloader'] == rel_dict['info']['download_downloader']: - log.debug('Found release by id: %s', item['id']) + if release_download['id'] == rel_dict['info']['download_id'] and release_download['downloader'] == rel_dict['info']['download_downloader']: + log.debug('Found release by id: %s', release_download['id']) found_release = True else: - if item['name'] == nzbname or rel_dict['info']['name'] in item['name'] or getImdb(item['name']) == movie_dict['library']['identifier']: + if release_download['name'] == nzbname or rel_dict['info']['name'] in release_download['name'] or getImdb(release_download['name']) == movie_dict['library']['identifier']: found_release = True if found_release: - timeleft = 'N/A' if item['timeleft'] == -1 else item['timeleft'] - log.debug('Found %s: %s, time to go: %s', (item['name'], item['status'].upper(), timeleft)) + timeleft = 'N/A' if release_download['timeleft'] == -1 else release_download['timeleft'] + log.debug('Found %s: %s, time to go: %s', (release_download['name'], release_download['status'].upper(), timeleft)) - if item['status'] == 'busy': + if release_download['status'] == 'busy': # Set the release to snatched if it was missing before fireEvent('release.update_status', rel.id, status = snatched_status, single = True) # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading - if item['folder'] and self.conf('from') in item['folder']: - self.tagDir(item['folder'], 'downloading') - - elif item['status'] == 'seeding': - # Set the release to seeding - fireEvent('release.update_status', rel.id, status = seeding_status, single = True) + if release_download['folder'] and self.conf('from') in release_download['folder']: + self.tagRelease(release_download = release_download, tag = 'downloading') + elif release_download['status'] == 'seeding': #If linking setting is enabled, process release - if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(item): - log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (item['name'], item['seed_ratio'])) + if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(release_download): + log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (release_download['name'], release_download['seed_ratio'])) # Remove the downloading tag - self.untagDir(item['folder'], 'downloading') + self.untagRelease(release_download = release_download, tag = 'downloading') # Scan and set the torrent to paused if required - item.update({'pause': True, 'scan': True, 'process_complete': False}) - scan_items.append(item) + release_download.update({'pause': True, 'scan': True, 'process_complete': False}) + scan_releases.append(release_download) else: #let it seed - log.debug('%s is seeding with ratio: %s', (item['name'], item['seed_ratio'])) + log.debug('%s is seeding with ratio: %s', (release_download['name'], release_download['seed_ratio'])) - elif item['status'] == 'failed': + # Set the release to seeding + fireEvent('release.update_status', rel.id, status = seeding_status, single = True) + + elif release_download['status'] == 'failed': # Set the release to failed fireEvent('release.update_status', rel.id, status = failed_status, single = True) - fireEvent('download.remove_failed', item, single = True) + fireEvent('download.remove_failed', release_download, single = True) if self.conf('next_on_failed'): fireEvent('movie.searcher.try_next_release', movie_id = rel.movie_id) - elif item['status'] == 'completed': - log.info('Download of %s completed!', item['name']) - if self.statusInfoComplete(item): + elif release_download['status'] == 'completed': + log.info('Download of %s completed!', release_download['name']) + if self.statusInfoComplete(release_download): # If the release has been seeding, process now the seeding is done if rel.status_id == seeding_status.get('id'): - if rel.movie.status_id == done_status.get('id'): + if self.conf('file_action') != 'move': # Set the release to done as the movie has already been renamed fireEvent('release.update_status', rel.id, status = downloaded_status, single = True) # Allow the downloader to clean-up - item.update({'pause': False, 'scan': False, 'process_complete': True}) - scan_items.append(item) + release_download.update({'pause': False, 'scan': False, 'process_complete': True}) + scan_releases.append(release_download) else: - # Set the release to snatched so that the renamer can process the release as if it was never seeding - fireEvent('release.update_status', rel.id, status = snatched_status, single = True) - # Scan and Allow the downloader to clean-up - item.update({'pause': False, 'scan': True, 'process_complete': True}) - scan_items.append(item) + release_download.update({'pause': False, 'scan': True, 'process_complete': True}) + scan_releases.append(release_download) else: # Set the release to snatched if it was missing before fireEvent('release.update_status', rel.id, status = snatched_status, single = True) # Remove the downloading tag - self.untagDir(item['folder'], 'downloading') + self.untagRelease(release_download = release_download, tag = 'downloading') # Scan and Allow the downloader to clean-up - item.update({'pause': False, 'scan': True, 'process_complete': True}) - scan_items.append(item) + release_download.update({'pause': False, 'scan': True, 'process_complete': True}) + scan_releases.append(release_download) else: scan_required = True @@ -791,21 +880,21 @@ Remove it if you want it to be renamed (again, or at least let it try again) log.error('Failed checking for release in downloader: %s', traceback.format_exc()) # The following can either be done here, or inside the scanner if we pass it scan_items in one go - for item in scan_items: + for release_download in scan_releases: # Ask the renamer to scan the item - if item['scan']: - if item['pause'] and self.conf('file_action') == 'link': - fireEvent('download.pause', item = item, pause = True, single = True) - fireEvent('renamer.scan', download_info = item) - if item['pause'] and self.conf('file_action') == 'link': - fireEvent('download.pause', item = item, pause = False, single = True) - if item['process_complete']: + if release_download['scan']: + if release_download['pause'] and self.conf('file_action') == 'link': + fireEvent('download.pause', release_download = release_download, pause = True, single = True) + fireEvent('renamer.scan', release_download = release_download) + if release_download['pause'] and self.conf('file_action') == 'link': + fireEvent('download.pause', release_download = release_download, pause = False, single = True) + if release_download['process_complete']: #First make sure the files were succesfully processed - if not self.hastagDir(item['folder'], 'failed_rename'): + if not self.hastagRelease(release_download = release_download, tag = 'failed_rename'): # Remove the seeding tag if it exists - self.untagDir(item['folder'], 'renamed_already') + self.untagRelease(release_download = release_download, tag = 'renamed_already') # Ask the downloader to process the item - fireEvent('download.process_complete', item = item, single = True) + fireEvent('download.process_complete', release_download = release_download, single = True) if scan_required: fireEvent('renamer.scan') @@ -814,16 +903,16 @@ Remove it if you want it to be renamed (again, or at least let it try again) return True - def extendDownloadInfo(self, download_info): + def extendReleaseDownload(self, release_download): rls = None - if download_info and download_info.get('id') and download_info.get('downloader'): + if release_download and release_download.get('id') and release_download.get('downloader'): db = get_session() - rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = download_info.get('downloader')).all() - rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_info.get('id')).all() + rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = release_download.get('downloader')).all() + rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = release_download.get('id')).all() for rlsnfo_dwnld in rlsnfo_dwnlds: for rlsnfo_id in rlsnfo_ids: @@ -833,29 +922,30 @@ Remove it if you want it to be renamed (again, or at least let it try again) if rls: break if not rls: - log.error('Download ID %s from downloader %s not found in releases', (download_info.get('id'), download_info.get('downloader'))) + log.error('Download ID %s from downloader %s not found in releases', (release_download.get('id'), release_download.get('downloader'))) if rls: rls_dict = rls.to_dict({'info':{}}) - download_info.update({ + release_download.update({ 'imdb_id': rls.movie.library.identifier, 'quality': rls.quality.identifier, 'protocol': rls_dict.get('info', {}).get('protocol') or rls_dict.get('info', {}).get('type'), + 'rls_id': rls.id, }) - return download_info + return release_download - def downloadIsTorrent(self, download_info): - return download_info and download_info.get('protocol') in ['torrent', 'torrent_magnet'] + def downloadIsTorrent(self, release_download): + return release_download and release_download.get('protocol') in ['torrent', 'torrent_magnet'] def fileIsAdded(self, src, group): if not group or not group.get('before_rename'): return False return src in group['before_rename'] - def statusInfoComplete(self, item): - return item['id'] and item['downloader'] and item['folder'] + def statusInfoComplete(self, release_download): + return release_download['id'] and release_download['downloader'] and release_download['folder'] def movieInFromFolder(self, movie_folder): return movie_folder and self.conf('from') in movie_folder or not movie_folder @@ -886,7 +976,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) #Extract all found archives for archive in archives: # Check if it has already been processed by CPS - if self.hastagDir(os.path.dirname(archive['file'])): + if self.hastagRelease(release_download = {'folder': os.path.dirname(archive['file']), 'files': archive['file']}): continue # Find all related archive files diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 316e81a..6346269 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -104,7 +104,7 @@ class Scanner(Plugin): addEvent('scanner.name_year', self.getReleaseNameYear) addEvent('scanner.partnumber', self.getPartNumber) - def scan(self, folder = None, files = None, download_info = None, simple = False, newer_than = 0, return_ignored = True, on_found = None): + def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, on_found = None): folder = ss(os.path.normpath(folder)) @@ -232,10 +232,6 @@ class Scanner(Plugin): # Remove the found files from the leftover stack leftovers = leftovers - set(found_files) - exts = [getExt(ff) for ff in found_files] - if 'ignore' in exts: - ignored_identifiers.append(identifier) - # Break if CP wants to shut down if self.shuttingDown(): break @@ -262,10 +258,6 @@ class Scanner(Plugin): # Remove the found files from the leftover stack leftovers = leftovers - set([ff]) - ext = getExt(ff) - if ext == 'ignore': - ignored_identifiers.append(new_identifier) - # Break if CP wants to shut down if self.shuttingDown(): break @@ -339,11 +331,11 @@ class Scanner(Plugin): total_found = len(valid_files) # Make sure only one movie was found if a download ID is provided - if download_info and total_found == 0: - log.info('Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).', download_info.get('imdb_id')) - elif download_info and total_found > 1: - log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_info.get('imdb_id'), len(valid_files))) - download_info = None + if release_download and total_found == 0: + log.info('Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).', release_download.get('imdb_id')) + elif release_download and total_found > 1: + log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (release_download.get('imdb_id'), len(valid_files))) + release_download = None # Determine file types db = get_session() @@ -379,7 +371,7 @@ class Scanner(Plugin): continue log.debug('Getting metadata for %s', identifier) - group['meta_data'] = self.getMetaData(group, folder = folder, download_info = download_info) + group['meta_data'] = self.getMetaData(group, folder = folder, release_download = release_download) # Subtitle meta group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {} @@ -411,7 +403,7 @@ class Scanner(Plugin): del group['unsorted_files'] # Determine movie - group['library'] = self.determineMovie(group, download_info = download_info) + group['library'] = self.determineMovie(group, release_download = release_download) if not group['library']: log.error('Unable to determine movie: %s', group['identifiers']) else: @@ -436,7 +428,7 @@ class Scanner(Plugin): return processed_movies - def getMetaData(self, group, folder = '', download_info = None): + def getMetaData(self, group, folder = '', release_download = None): data = {} files = list(group['files']['movie']) @@ -461,8 +453,8 @@ class Scanner(Plugin): # Use the quality guess first, if that failes use the quality we wanted to download data['quality'] = None - if download_info and download_info.get('quality'): - data['quality'] = fireEvent('quality.single', download_info.get('quality'), single = True) + if release_download and release_download.get('quality'): + data['quality'] = fireEvent('quality.single', release_download.get('quality'), single = True) if not data['quality']: data['quality'] = fireEvent('quality.guess', files = files, extra = data, single = True) @@ -546,12 +538,12 @@ class Scanner(Plugin): return detected_languages - def determineMovie(self, group, download_info = None): + def determineMovie(self, group, release_download = None): # Get imdb id from downloader - imdb_id = download_info and download_info.get('imdb_id') + imdb_id = release_download and release_download.get('imdb_id') if imdb_id: - log.debug('Found movie via imdb id from it\'s download id: %s', download_info.get('imdb_id')) + log.debug('Found movie via imdb id from it\'s download id: %s', release_download.get('imdb_id')) files = group['files']