From b8deb49ed30d4192fc61dc6be8b3146cf2219458 Mon Sep 17 00:00:00 2001 From: peerster Date: Wed, 5 Oct 2016 12:58:47 +0200 Subject: [PATCH 01/17] Remove dead project link --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index ea20d2d..618dea1 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,6 @@ Linux: Docker: * You can use [linuxserver.io](https://github.com/linuxserver/docker-couchpotato) or [razorgirl's](https://github.com/razorgirl/docker-couchpotato) to quickly build your own isolated app container. It's based on the Linux instructions above. For more info about Docker check out the [official website](https://www.docker.com). -Ansible: -* You can use [peerster's] (https://github.com/peerster/ansible-couchpotato) [ansible] (http://www.ansible.com) role to deploy couchpotato. - FreeBSD: * Become root with `su` From 84e7bbfd7b11af166dba12f4e5908a3af818f385 Mon Sep 17 00:00:00 2001 From: Harris Borawski Date: Sun, 13 Nov 2016 18:49:12 -0800 Subject: [PATCH 02/17] Add notifiter for Join by joaoapps --- couchpotato/core/notifications/join.py | 84 ++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 couchpotato/core/notifications/join.py diff --git a/couchpotato/core/notifications/join.py b/couchpotato/core/notifications/join.py new file mode 100644 index 0000000..06bf4e5 --- /dev/null +++ b/couchpotato/core/notifications/join.py @@ -0,0 +1,84 @@ +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.helpers.variable import splitString +from couchpotato.core.logger import CPLog +from couchpotato.core.notifications.base import Notification + + +log = CPLog(__name__) + +autoload = 'Join' + + +class Join(Notification): + + # URL for request + url = 'https://joinjoaomgcd.appspot.com/_ah/api/messaging/v1/sendPush?title=%s&text=%s&deviceId=%s&icon=%s' + + # URL for notification icon + icon = tryUrlencode('https://raw.githubusercontent.com/CouchPotato/CouchPotatoServer/master/couchpotato/static/images/icons/android.png') + + def notify(self, message = '', data = None, listener = None): + if not data: data = {} + + # default for devices + device_default = [None] + + apikey = self.conf('apikey') + if apikey is not None: + # Add apikey to request url + self.url = self.url + '&apikey=' + apikey + # If api key is present, default to sending to all devices + device_default = ['group.all'] + + devices = self.getDevices() or device_default + successful = 0 + for device in devices: + response = self.urlopen(self.url % (self.default_title, tryUrlencode(toUnicode(message)), device, self.icon)) + + if response: + successful += 1 + else: + log.error('Unable to push notification to Join device with ID %s' % device) + + return successful == len(devices) + + def getDevices(self): + return splitString(self.conf('devices')) + + +config = [{ + 'name': 'join', + 'groups': [ + { + 'tab': 'notifications', + 'list': 'notification_providers', + 'name': 'join', + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + }, + { + 'name': 'devices', + 'default': '', + 'description': 'IDs of devices to notify, or group to send to if API key is specified (ex: group.all)' + }, + { + 'name': 'apikey', + 'default': '', + 'advanced': True, + 'description': 'API Key for sending to all devices, or group' + }, + { + 'name': 'on_snatch', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Also send message when movie is snatched.', + }, + ], + } + ], +}] From bd801b35d9310102a68e84a22447cfef92d8c3b1 Mon Sep 17 00:00:00 2001 From: etomm Date: Tue, 22 Nov 2016 04:07:57 +0100 Subject: [PATCH 03/17] File names that has a : in position 1 makes guessit split_path to infinite loop --- libs/guessit/fileutils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/guessit/fileutils.py b/libs/guessit/fileutils.py index de30b8d..993952a 100755 --- a/libs/guessit/fileutils.py +++ b/libs/guessit/fileutils.py @@ -23,6 +23,7 @@ from guessit import s, u import os.path import zipfile import io +import re def split_path(path): @@ -45,6 +46,13 @@ def split_path(path): while True: head, tail = os.path.split(path) headlen = len(head) + + # if a string has a : in position 1 it gets splitted in everycase, also if + # there is not a valid drive letter and also if : is not followed by \ + if headlen >= 2 and headlen <= 3 and head[1] == ':' and ( head + tail == path ) and ( head[1:] != ':\\' or not re.match("^[a-zA-Z]:\\\\", head) ): + tail = path + head = '' + headlen = 0 # on Unix systems, the root folder is '/' if head and head == '/'*headlen and tail == '': From c4d0cb17642d2069d15479cf4752acbc12bf84d5 Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Wed, 23 Nov 2016 12:36:58 -0500 Subject: [PATCH 04/17] Pulled new API from PUTIO --- libs/pio/api.py | 230 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 183 insertions(+), 47 deletions(-) diff --git a/libs/pio/api.py b/libs/pio/api.py index ecfc177..01c4d1f 100644 --- a/libs/pio/api.py +++ b/libs/pio/api.py @@ -1,26 +1,53 @@ # -*- coding: utf-8 -*- - -# Changed -# Removed iso8601 library requirement +# Changed +# Removed iso8601 library requirement # Added CP logging import os -import re import json +import binascii import webbrowser -from urllib import urlencode -from couchpotato import CPLog -from dateutil.parser import parse +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode +from datetime import datetime +import tus import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry +from couchpotato import CPLog + + +KB = 1024 +MB = 1024 * KB + +# Read and write operations are limited to this chunk size. +# This can make a big difference when dealing with large files. +CHUNK_SIZE = 256 * KB BASE_URL = 'https://api.put.io/v2' +UPLOAD_URL = 'https://upload.put.io/v2/files/upload' +TUS_UPLOAD_URL = 'https://upload.put.io/files/' ACCESS_TOKEN_URL = 'https://api.put.io/v2/oauth2/access_token' AUTHENTICATION_URL = 'https://api.put.io/v2/oauth2/authenticate' log = CPLog(__name__) +class APIError(Exception): + pass + + +class ClientError(APIError): + pass + + +class ServerError(APIError): + pass + + class AuthHelper(object): def __init__(self, client_id, client_secret, redirect_uri, type='code'): @@ -58,10 +85,21 @@ class AuthHelper(object): class Client(object): - def __init__(self, access_token): + def __init__(self, access_token, use_retry=False): self.access_token = access_token self.session = requests.session() + if use_retry: + # Retry maximum 10 times, backoff on each retry + # Sleeps 1s, 2s, 4s, 8s, etc to a maximum of 120s between retries + # Retries on HTTP status codes 500, 502, 503, 504 + retries = Retry(total=10, + backoff_factor=1, + status_forcelist=[500, 502, 503, 504]) + + # Use the retry strategy for all HTTPS requests + self.session.mount('https://', HTTPAdapter(max_retries=retries)) + # Keep resource classes as attributes of client. # Pass client to resource classes so resource object # can use the client. @@ -71,7 +109,7 @@ class Client(object): self.Account = type('Account', (_Account,), attributes) def request(self, path, method='GET', params=None, data=None, files=None, - headers=None, raw=False, stream=False): + headers=None, raw=False, allow_redirects=True, stream=False): """ Wrapper around requests.request() @@ -91,27 +129,31 @@ class Client(object): headers['Accept'] = 'application/json' - url = BASE_URL + path + if path.startswith('https://'): + url = path + else: + url = BASE_URL + path log.debug('url: %s', url) response = self.session.request( method, url, params=params, data=data, files=files, - headers=headers, allow_redirects=True, stream=stream) + headers=headers, allow_redirects=allow_redirects, stream=stream) log.debug('response: %s', response) if raw: return response log.debug('content: %s', response.content) try: - response = json.loads(response.content) + body = json.loads(response.content.decode()) except ValueError: - raise Exception('Server didn\'t send valid JSON:\n%s\n%s' % ( - response, response.content)) + raise ServerError('InvalidJSON', response.content) - if response['status'] == 'ERROR': - raise Exception(response['error_type']) + if body['status'] == 'ERROR': + log.error("API returned error: %s", body) + exception_class = {'4': ClientError, '5': ServerError}[str(response.status_code)[0]] + raise exception_class(body['error_type'], body['error_message']) - return response + return body class _BaseResource(object): @@ -125,8 +167,8 @@ class _BaseResource(object): self.name = None self.__dict__.update(resource_dict) try: - self.created_at = parse(self.created_at) - except AttributeError: + self.created_at = strptime(self.created_at) + except Exception: self.created_at = None def __str__(self): @@ -135,7 +177,7 @@ class _BaseResource(object): def __repr__(self): # shorten name for display name = self.name[:17] + '...' if len(self.name) > 20 else self.name - return '<%s id=%r, name="%r">' % ( + return '<%s id=%r, name=%r>' % ( self.__class__.__name__, self.id, name) @@ -160,59 +202,113 @@ class _File(_BaseResource): files = {'file': (name, f)} else: files = {'file': f} - d = cls.client.request('/files/upload', method='POST', + d = cls.client.request(UPLOAD_URL, method='POST', data={'parent_id': parent_id}, files=files) f = d['file'] return cls(f) + @classmethod + def upload_tus(cls, path, name=None, parent_id=0): + headers = {'Authorization': 'token %s' % cls.client.access_token} + metadata = {'parent_id': str(parent_id)} + if name: + metadata['name'] = name + with open(path) as f: + tus.upload(f, TUS_UPLOAD_URL, file_name=name, headers=headers, metadata=metadata) + def dir(self): """List the files under directory.""" return self.list(parent_id=self.id) - def download(self, dest='.', delete_after_download=False): + def download(self, dest='.', delete_after_download=False, chunk_size=CHUNK_SIZE): if self.content_type == 'application/x-directory': - self._download_directory(dest, delete_after_download) + self._download_directory(dest, delete_after_download, chunk_size) else: - self._download_file(dest, delete_after_download) + self._download_file(dest, delete_after_download, chunk_size) - def _download_directory(self, dest='.', delete_after_download=False): - name = self.name - if isinstance(name, unicode): - name = name.encode('utf-8', 'replace') + def _download_directory(self, dest, delete_after_download, chunk_size): + name = _str(self.name) dest = os.path.join(dest, name) if not os.path.exists(dest): os.mkdir(dest) for sub_file in self.dir(): - sub_file.download(dest, delete_after_download) + sub_file.download(dest, delete_after_download, chunk_size) if delete_after_download: self.delete() - def _download_file(self, dest='.', delete_after_download=False): - response = self.client.request( - '/files/%s/download' % self.id, raw=True, stream=True) + def _verify_file(self, filepath): + log.info('verifying crc32...') + filesize = os.path.getsize(filepath) + if self.size != filesize: + logging.error('file %s is %d bytes, should be %s bytes' % (filepath, filesize, self.size)) + return False - filename = re.match( - 'attachment; filename=(.*)', - response.headers['content-disposition']).groups()[0] - # If file name has spaces, it must have quotes around. - filename = filename.strip('"') + crcbin = 0 + with open(filepath, 'rb') as f: + while True: + chunk = f.read(CHUNK_SIZE) + if not chunk: + break - with open(os.path.join(dest, filename), 'wb') as f: - for chunk in response.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - f.flush() + crcbin = binascii.crc32(chunk, crcbin) & 0xffffffff - if delete_after_download: - self.delete() + crc32 = '%08x' % crcbin + + if crc32 != self.crc32: + logging.error('file %s CRC32 is %s, should be %s' % (filepath, crc32, self.crc32)) + return False + + return True + + def _download_file(self, dest, delete_after_download, chunk_size): + name = _str(self.name) + + filepath = os.path.join(dest, name) + if os.path.exists(filepath): + first_byte = os.path.getsize(filepath) + + if first_byte == self.size: + log.warning('file %s exists and is the correct size %d' % (filepath, self.size)) + else: + first_byte = 0 + + log.debug('file %s is currently %d, should be %d' % (filepath, first_byte, self.size)) + + if self.size == 0: + # Create an empty file + open(filepath, 'w').close() + log.debug('created empty file %s' % filepath) + else: + if first_byte < self.size: + with open(filepath, 'ab') as f: + headers = {'Range': 'bytes=%d-' % first_byte} + + log.debug('request range: bytes=%d-' % first_byte) + response = self.client.request('/files/%s/download' % self.id, + headers=headers, + raw=True, + stream=True) + + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + + if self._verify_file(filepath): + if delete_after_download: + self.delete() def delete(self): return self.client.request('/files/delete', method='POST', - data={'file_ids': str(self.id)}) + data={'file_id': str(self.id)}) + + @classmethod + def delete_multi(cls, ids): + return cls.client.request('/files/delete', method='POST', + data={'file_ids': ','.join(map(str, ids))}) def move(self, parent_id): return self.client.request('/files/move', method='POST', @@ -239,6 +335,7 @@ class _Transfer(_BaseResource): @classmethod def add_url(cls, url, parent_id=0, extract=False, callback_url=None): + log.debug('callback_url is %s', callback_url) d = cls.client.request('/transfers/add', method='POST', data=dict( url=url, save_parent_id=parent_id, extract=extract, callback_url=callback_url)) @@ -247,10 +344,10 @@ class _Transfer(_BaseResource): @classmethod def add_torrent(cls, path, parent_id=0, extract=False, callback_url=None): - with open(path) as f: + with open(path, 'rb') as f: files = {'file': f} d = cls.client.request('/files/upload', method='POST', files=files, - data=dict(save_parent_id=parent_id, + data=dict(parent_id=parent_id, extract=extract, callback_url=callback_url)) t = d['transfer'] @@ -260,6 +357,17 @@ class _Transfer(_BaseResource): def clean(cls): return cls.client.request('/transfers/clean', method='POST') + def cancel(self): + return self.client.request('/transfers/cancel', + method='POST', + data={'transfer_ids': self.id}) + + @classmethod + def cancel_multi(cls, ids): + return cls.client.request('/transfers/cancel', + method='POST', + data={'transfer_ids': ','.join(map(str, ids))}) + class _Account(_BaseResource): @@ -270,3 +378,31 @@ class _Account(_BaseResource): @classmethod def settings(cls): return cls.client.request('/account/settings', method='GET') + + +# Due to a nasty bug in datetime module, datetime.strptime calls +# are not thread-safe and can throw a TypeError. Details: https://bugs.python.org/issue7980 +# Here we are implementing simple RFC3339 parser which is used in Put.io APIv2. +def strptime(date): + """Returns datetime object from the given date, which is in a specific format: YYYY-MM-ddTHH:mm:ss""" + d = { + 'year': date[0:4], + 'month': date[5:7], + 'day': date[8:10], + 'hour': date[11:13], + 'minute': date[14:16], + 'second': date[17:], + } + + d = dict((k, int(v)) for k, v in d.iteritems()) + return datetime(**d) + + +def _str(s): + """Python 3 compatibility function for converting to str.""" + try: + if isinstance(s, unicode): + return s.encode('utf-8', 'replace') + except NameError: + pass + return s From fdea2377b6f94f5307eb8d9224557f015f1b3de2 Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Wed, 23 Nov 2016 12:38:04 -0500 Subject: [PATCH 05/17] Added https variable --- couchpotato/core/downloaders/putio/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/couchpotato/core/downloaders/putio/__init__.py b/couchpotato/core/downloaders/putio/__init__.py index 6564de7..7ee95eb 100644 --- a/couchpotato/core/downloaders/putio/__init__.py +++ b/couchpotato/core/downloaders/putio/__init__.py @@ -34,6 +34,12 @@ config = [{ 'default': 0, }, { + 'name': 'https', + 'description': 'Set to true if your callback host accepts https instead of http', + 'type': 'bool', + 'default': 0, + }, + { 'name': 'callback_host', 'description': 'External reachable url to CP so put.io can do it\'s thing', }, From 4bc0e2c58b48c26f39589f321692a345de17efcb Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Wed, 23 Nov 2016 12:38:27 -0500 Subject: [PATCH 06/17] Allow setting callbackurl to https --- couchpotato/core/downloaders/putio/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index c568582..a49f870 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -61,9 +61,13 @@ class PutIO(DownloaderBase): # Note callback_host is NOT our address, it's the internet host that putio can call too callbackurl = None if self.conf('download'): - callbackurl = 'http://' + self.conf('callback_host') + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/')) + pre = 'http://' + if self.conf('https'): + pre = 'https://' + callbackurl = pre + self.conf('callback_host') + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/')) + log.debug('callbackurl is %s', callbackurl) resp = client.Transfer.add_url(url, callback_url = callbackurl, parent_id = putioFolder) - log.debug('resp is %s', resp.id); + log.debug('resp is %s', resp.id) return self.downloadReturnId(resp.id) def test(self): From 5b659f85933899b388f634a25502587c6ed4b5b6 Mon Sep 17 00:00:00 2001 From: Christopher Haglund Date: Sat, 26 Nov 2016 23:30:49 +0100 Subject: [PATCH 07/17] Add year to search param for torrentpotato --- couchpotato/core/media/movie/providers/torrent/torrentpotato.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/providers/torrent/torrentpotato.py b/couchpotato/core/media/movie/providers/torrent/torrentpotato.py index e22f4e7..9bc6b85 100644 --- a/couchpotato/core/media/movie/providers/torrent/torrentpotato.py +++ b/couchpotato/core/media/movie/providers/torrent/torrentpotato.py @@ -17,6 +17,6 @@ class TorrentPotato(MovieProvider, Base): 'user': host['name'], 'passkey': host['pass_key'], 'imdbid': getIdentifier(media), - 'search' : getTitle(media), + 'search' : getTitle(media) + ' ' + str(media['info']['year']), }) return '%s?%s' % (host['host'], arguments) From 17ab2561420098b5cdfb37545843428aa51b69c0 Mon Sep 17 00:00:00 2001 From: Red Hodgerson Date: Sun, 4 Dec 2016 14:44:09 -0600 Subject: [PATCH 08/17] Fix BeautifulSoup find_all parsing BeautifulSoup only returns a single TR tag regardless of the amount of results at line 49. Since the first TR tag returned isn't an actual result, you never get any results back. Explicitly calling the html parser fixes this issue. http://stackoverflow.com/questions/16322862/beautiful-soup-findall-doent-find-them-all --- couchpotato/core/media/_base/providers/torrent/bithdtv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/bithdtv.py b/couchpotato/core/media/_base/providers/torrent/bithdtv.py index 7aa0157..69c030e 100644 --- a/couchpotato/core/media/_base/providers/torrent/bithdtv.py +++ b/couchpotato/core/media/_base/providers/torrent/bithdtv.py @@ -39,7 +39,7 @@ class Base(TorrentProvider): if '## SELECT COUNT(' in split_data[0]: data = split_data[2] - html = BeautifulSoup(data) + html = BeautifulSoup(data, 'html.parser') try: result_table = html.find('table', attrs = {'width': '750', 'class': ''}) From 78a474995251264bc6203c2bb374346a059b0d06 Mon Sep 17 00:00:00 2001 From: lattedesu Date: Mon, 5 Dec 2016 12:24:05 +0200 Subject: [PATCH 09/17] PassThePopcorn URL update to prevent API 403 error PassThePopcorn recently changed their api endpoints to tls.passthepopcorn.me to passthepopcorn.me . Unless you change these URLs, you get this issue: ``` 12-05 05:08:17 ERROR [hpotato.core.plugins.base] Failed opening url in PassThePopcorn: https://tls.passthepopcorn.me/ajax.php?action=login Traceback (most recent call last): HTTPError: 403 Client Error: Forbidden for url: https://passthepopcorn.me/ajax.php?action=login 12-05 05:08:17 ERROR [edia._base.providers.base] Failed to login PassThePopcorn: Traceback (most recent call last): File "/home/couchpotato/CouchPotatoServer/couchpotato/core/media/_base/providers/base.py", line 163, in login output = self.urlopen(self.urls['login'], data = self.getLoginParams()) File "/home/couchpotato/CouchPotatoServer/couchpotato/core/plugins/base.py", line 227, in urlopen response.raise_for_status() File "/home/couchpotato/CouchPotatoServer/libs/requests/models.py", line 837, in raise_for_status raise HTTPError(http_error_msg, response=self) HTTPError: 403 Client Error: Forbidden for url: https://passthepopcorn.me/ajax.php?action=login``` There is a thread at their forum; titled: "HTTPS Migration preventing CouchPotato Login", and it's told that this patch fixes this issue. This is required due to migration and their tracker update, and it is shared there. So all credits to the poster. I'm simply making a pull request for the fix, and further investigation if required. Thanks in advance, --- .../core/media/_base/providers/torrent/passthepopcorn.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/passthepopcorn.py b/couchpotato/core/media/_base/providers/torrent/passthepopcorn.py index d4db814..c96afd1 100644 --- a/couchpotato/core/media/_base/providers/torrent/passthepopcorn.py +++ b/couchpotato/core/media/_base/providers/torrent/passthepopcorn.py @@ -18,12 +18,12 @@ log = CPLog(__name__) class Base(TorrentProvider): urls = { - 'domain': 'https://tls.passthepopcorn.me', - 'detail': 'https://tls.passthepopcorn.me/torrents.php?torrentid=%s', - 'torrent': 'https://tls.passthepopcorn.me/torrents.php', - 'login': 'https://tls.passthepopcorn.me/ajax.php?action=login', - 'login_check': 'https://tls.passthepopcorn.me/ajax.php?action=login', - 'search': 'https://tls.passthepopcorn.me/search/%s/0/7/%d' + 'domain': 'https://passthepopcorn.me', + 'detail': 'https://passthepopcorn.me/torrents.php?torrentid=%s', + 'torrent': 'https://passthepopcorn.me/torrents.php', + 'login': 'https://passthepopcorn.me/ajax.php?action=login', + 'login_check': 'https://passthepopcorn.me/ajax.php?action=login', + 'search': 'https://passthepopcorn.me/search/%s/0/7/%d' } login_errors = 0 @@ -218,7 +218,7 @@ config = [{ 'name': 'domain', 'advanced': True, 'label': 'Proxy server', - 'description': 'Domain for requests (HTTPS only!), keep empty to use default (tls.passthepopcorn.me).', + 'description': 'Domain for requests (HTTPS only!), keep empty to use default (passthepopcorn.me).', }, { 'name': 'username', From 02ab82cbecb701e3bbeb5173e8a7eae3a75fbefb Mon Sep 17 00:00:00 2001 From: Thijs Tijsma Date: Tue, 6 Dec 2016 08:46:15 +0100 Subject: [PATCH 10/17] Removed changing of global uTorrent settings. --- .gitignore | 4 ++++ couchpotato/core/downloaders/utorrent.py | 20 +------------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index d53d129..1d21d2e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ .coverage coverage.xml nosetests.xml + +# Visual Studio + +/.vs diff --git a/couchpotato/core/downloaders/utorrent.py b/couchpotato/core/downloaders/utorrent.py index 847eaf1..2bfc64e 100644 --- a/couchpotato/core/downloaders/utorrent.py +++ b/couchpotato/core/downloaders/utorrent.py @@ -1,4 +1,4 @@ -from base64 import b16encode, b32decode +from base64 import b16encode, b32decode from datetime import timedelta from hashlib import sha1 import cookielib @@ -74,24 +74,6 @@ class uTorrent(DownloaderBase): if not self.connect(): return False - settings = self.utorrent_api.get_settings() - if not settings: - return False - - #Fix settings in case they are not set for CPS compatibility - new_settings = {} - if not (settings.get('seed_prio_limitul') == 0 and settings['seed_prio_limitul_flag']): - new_settings['seed_prio_limitul'] = 0 - new_settings['seed_prio_limitul_flag'] = True - log.info('Updated uTorrent settings to set a torrent to complete after it the seeding requirements are met.') - - if settings.get('bt.read_only_on_complete'): #This doesn't work as this option seems to be not available through the api. Mitigated with removeReadOnly function - new_settings['bt.read_only_on_complete'] = False - log.info('Updated uTorrent settings to not set the files to read only after completing.') - - if new_settings: - self.utorrent_api.set_settings(new_settings) - torrent_params = {} if self.conf('label'): torrent_params['label'] = self.conf('label') From d516182fc65ee4084f2c6b66f0e2351859a3c98d Mon Sep 17 00:00:00 2001 From: DaftHonk Date: Fri, 9 Dec 2016 14:41:36 -0400 Subject: [PATCH 11/17] Correct Emby API Call for Movies update Fixes issue of emby notification not triggering a movie library update --- couchpotato/core/notifications/emby.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/emby.py b/couchpotato/core/notifications/emby.py index 30254f0..baba521 100644 --- a/couchpotato/core/notifications/emby.py +++ b/couchpotato/core/notifications/emby.py @@ -18,7 +18,7 @@ class Emby(Notification): apikey = self.conf('apikey') host = cleanHost(host) - url = '%semby/Library/Series/Updated' % (host) + url = '%semby/Library/Movies/Updated' % (host) values = {} data = urllib.urlencode(values) From 57cc63da98601c5e8bcdfc35e7171b1ca6c7bd30 Mon Sep 17 00:00:00 2001 From: whitter Date: Wed, 14 Dec 2016 15:52:50 +0000 Subject: [PATCH 12/17] Updated TorrentLeech provider for new site version The new version of the TorrentLeech website now has absolute paths for the download links --- couchpotato/core/media/_base/providers/torrent/torrentleech.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentleech.py b/couchpotato/core/media/_base/providers/torrent/torrentleech.py index da13822..ffe41f6 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentleech.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentleech.py @@ -50,7 +50,7 @@ class Base(TorrentProvider): results.append({ 'id': link['href'].replace('/torrent/', ''), 'name': six.text_type(link.string), - 'url': self.urls['download'] % url['href'], + 'url': url['href'], 'detail_url': self.urls['download'] % details['href'], 'size': self.parseSize(result.find_all('td')[4].string), 'seeders': tryInt(result.find('td', attrs = {'class': 'seeders'}).string), From dfd368263f5fe12bee8797749424b5ee2af1ed1b Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 17 Dec 2016 13:42:38 +0100 Subject: [PATCH 13/17] Indent fix --- couchpotato/core/downloaders/putio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/putio/__init__.py b/couchpotato/core/downloaders/putio/__init__.py index 7ee95eb..0f3654a 100644 --- a/couchpotato/core/downloaders/putio/__init__.py +++ b/couchpotato/core/downloaders/putio/__init__.py @@ -34,7 +34,7 @@ config = [{ 'default': 0, }, { - 'name': 'https', + 'name': 'https', 'description': 'Set to true if your callback host accepts https instead of http', 'type': 'bool', 'default': 0, From 9ba8cbebece6296f0e418725ad1a2a6ba8537c5a Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 17 Dec 2016 14:01:37 +0100 Subject: [PATCH 14/17] PutIO needs tus lib --- libs/tus/__init__.py | 190 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 libs/tus/__init__.py diff --git a/libs/tus/__init__.py b/libs/tus/__init__.py new file mode 100644 index 0000000..c6fea96 --- /dev/null +++ b/libs/tus/__init__.py @@ -0,0 +1,190 @@ +import os +import base64 +import logging +import argparse + +import requests + +LOG_LEVEL = logging.INFO +DEFAULT_CHUNK_SIZE = 4 * 1024 * 1024 +TUS_VERSION = '1.0.0' + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +logger.addHandler(logging.NullHandler()) + + +class TusError(Exception): + pass + + +def _init(): + fmt = "[%(asctime)s] %(levelname)s %(message)s" + h = logging.StreamHandler() + h.setLevel(LOG_LEVEL) + h.setFormatter(logging.Formatter(fmt)) + logger.addHandler(h) + + +def _create_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('file', type=argparse.FileType('rb')) + parser.add_argument('--chunk-size', type=int, default=DEFAULT_CHUNK_SIZE) + parser.add_argument( + '--header', + action='append', + help="A single key/value pair" + " to be sent with all requests as HTTP header." + " Can be specified multiple times to send more then one header." + " Key and value must be separated with \":\".") + return parser + + +def _cmd_upload(): + _init() + + parser = _create_parser() + parser.add_argument('tus_endpoint') + parser.add_argument('--file_name') + parser.add_argument( + '--metadata', + action='append', + help="A single key/value pair to be sent in Upload-Metadata header." + " Can be specified multiple times to send more than one pair." + " Key and value must be separated with space.") + args = parser.parse_args() + + headers = dict([x.split(':') for x in args.header]) + metadata = dict([x.split(' ') for x in args.metadata]) + + upload( + args.file, + args.tus_endpoint, + chunk_size=args.chunk_size, + file_name=args.file_name, + headers=headers, + metadata=metadata) + + +def _cmd_resume(): + _init() + + parser = _create_parser() + parser.add_argument('file_endpoint') + args = parser.parse_args() + + headers = dict([x.split(':') for x in args.header]) + + resume( + args.file, + args.file_endpoint, + chunk_size=args.chunk_size, + headers=headers) + + +def upload(file_obj, + tus_endpoint, + chunk_size=DEFAULT_CHUNK_SIZE, + file_name=None, + headers=None, + metadata=None): + file_name = os.path.basename(file_obj.name) + file_size = _get_file_size(file_obj) + location = _create_file( + tus_endpoint, + file_name, + file_size, + extra_headers=headers, + metadata=metadata) + resume( + file_obj, location, chunk_size=chunk_size, headers=headers, offset=0) + + +def _get_file_size(f): + pos = f.tell() + f.seek(0, 2) + size = f.tell() + f.seek(pos) + return size + + +def _create_file(tus_endpoint, + file_name, + file_size, + extra_headers=None, + metadata=None): + logger.info("Creating file endpoint") + + headers = { + "Tus-Resumable": TUS_VERSION, + "Upload-Length": str(file_size), + } + + if extra_headers: + headers.update(extra_headers) + + if metadata: + l = [k + ' ' + base64.b64encode(v) for k, v in metadata.items()] + headers["Upload-Metadata"] = ','.join(l) + + response = requests.post(tus_endpoint, headers=headers) + if response.status_code != 201: + raise TusError("Create failed: %s" % response) + + location = response.headers["Location"] + logger.info("Created: %s", location) + return location + + +def resume(file_obj, + file_endpoint, + chunk_size=DEFAULT_CHUNK_SIZE, + headers=None, + offset=None): + if offset is None: + offset = _get_offset(file_endpoint, extra_headers=headers) + + total_sent = 0 + file_size = _get_file_size(file_obj) + while offset < file_size: + file_obj.seek(offset) + data = file_obj.read(chunk_size) + offset = _upload_chunk( + data, offset, file_endpoint, extra_headers=headers) + total_sent += len(data) + logger.info("Total bytes sent: %i", total_sent) + + +def _get_offset(file_endpoint, extra_headers=None): + logger.info("Getting offset") + + headers = {"Tus-Resumable": TUS_VERSION} + + if extra_headers: + headers.update(extra_headers) + + response = requests.head(file_endpoint, headers=headers) + response.raise_for_status() + + offset = int(response.headers["Upload-Offset"]) + logger.info("offset=%i", offset) + return offset + + +def _upload_chunk(data, offset, file_endpoint, extra_headers=None): + logger.info("Uploading chunk from offset: %i", offset) + + headers = { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': str(offset), + 'Tus-Resumable': TUS_VERSION, + } + + if extra_headers: + headers.update(extra_headers) + + response = requests.patch(file_endpoint, headers=headers, data=data) + if response.status_code != 204: + raise TusError("Upload chunk failed: %s" % response) + + return int(response.headers["Upload-Offset"]) From 6be3e0f4baa8c6732bd13c9c10da5d593889bad9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 17 Dec 2016 14:02:04 +0100 Subject: [PATCH 15/17] Don't try to remove language from themoviedb if it isn't in the list --- .../core/media/movie/providers/info/themoviedb.py | 57 +++++++++++----------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/couchpotato/core/media/movie/providers/info/themoviedb.py b/couchpotato/core/media/movie/providers/info/themoviedb.py index 31fb06b..9a242e7 100644 --- a/couchpotato/core/media/movie/providers/info/themoviedb.py +++ b/couchpotato/core/media/movie/providers/info/themoviedb.py @@ -27,7 +27,7 @@ class TheMovieDb(MovieProvider): ak = ['ZjdmNTE3NzU4NzdlMGJiNjcwMzUyMDk1MmIzYzc4NDA=', 'ZTIyNGZlNGYzZmVjNWY3YjU1NzA2NDFmN2NkM2RmM2E=', 'YTNkYzExMWU2NjEwNWY2Mzg3ZTk5MzkzODEzYWU0ZDU=', 'ZjZiZDY4N2ZmYTYzY2QyODJiNmZmMmM2ODc3ZjI2Njk='] - + languages = [ 'en' ] default_language = 'en' @@ -45,19 +45,20 @@ class TheMovieDb(MovieProvider): self.conf('api_key', '') languages = self.getLanguages() - + # languages should never be empty, the first language is the default language used for all the description details self.default_language = languages[0] - + # en is always downloaded and it is the fallback if 'en' in languages: languages.remove('en') - + # default language has a special management - languages.remove(self.default_language) - + if self.default_language in languages: + languages.remove(self.default_language) + self.languages = languages - + configuration = self.request('configuration') if configuration: self.configuration = configuration @@ -124,19 +125,19 @@ class TheMovieDb(MovieProvider): }) if not movie: return - + movie_default = movie if self.default_language == 'en' else self.request('movie/%s' % movie.get('id'), { 'append_to_response': 'alternative_titles' + (',images,casts' if extended else ''), 'language': self.default_language }) - + movie_default = movie_default or movie - + movie_others = [ self.request('movie/%s' % movie.get('id'), { 'append_to_response': 'alternative_titles' + (',images,casts' if extended else ''), 'language': language }) for language in self.languages] if self.languages else [] - + # Images poster = self.getImage(movie, type = 'poster', size = 'w154') poster_original = self.getImage(movie, type = 'poster', size = 'original') @@ -176,7 +177,7 @@ class TheMovieDb(MovieProvider): images['actors'][toUnicode(cast_item.get('name'))] = self.getImage(cast_item, type = 'profile', size = 'original') except: log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc())) - + movie_data = { 'type': 'movie', 'via_tmdb': True, @@ -199,17 +200,17 @@ class TheMovieDb(MovieProvider): # Add alternative names movies = [ movie ] + movie_others if movie == movie_default else [ movie, movie_default ] + movie_others movie_titles = [ self.getTitles(movie) for movie in movies ] - + all_titles = sorted(list(itertools.chain.from_iterable(movie_titles))) - + alternate_titles = movie_data['titles'] - + for title in all_titles: if title and title not in alternate_titles and title.lower() != 'none' and title is not None: alternate_titles.append(title) - - movie_data['titles'] = alternate_titles - + + movie_data['titles'] = alternate_titles + return movie_data def getImage(self, movie, type = 'poster', size = 'poster'): @@ -261,26 +262,26 @@ class TheMovieDb(MovieProvider): def getApiKey(self): key = self.conf('api_key') return bd(random.choice(self.ak)) if key == '' else key - + def getLanguages(self): languages = splitString(Env.setting('languages', section = 'core')) if len(languages): return languages - + return [ 'en' ] - + def getTitles(self, movie): # add the title to the list - title = toUnicode(movie.get('title')) - + title = toUnicode(movie.get('title')) + titles = [title] if title else [] - - # add the original_title to the list + + # add the original_title to the list alternate_title = toUnicode(movie.get('original_title')) - + if alternate_title and alternate_title not in titles: titles.append(alternate_title) - + # Add alternative titles alternate_titles = movie.get('alternative_titles', {}).get('titles', []) @@ -288,7 +289,7 @@ class TheMovieDb(MovieProvider): alt_name = toUnicode(alt.get('title')) if alt_name and alt_name not in titles and alt_name.lower() != 'none' and alt_name is not None: titles.append(alt_name) - + return titles; From 42dcd35689244e03f45c49b311c14343890b7a85 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 17 Dec 2016 14:02:23 +0100 Subject: [PATCH 16/17] Value -1 for quality counts as not selected --- couchpotato/core/plugins/profile/static/profile.js | 2 +- couchpotato/static/scripts/combined.plugins.min.js | 2 +- couchpotato/static/scripts/couchpotato.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/profile/static/profile.js b/couchpotato/core/plugins/profile/static/profile.js index 01fad1e..36c8ad3 100644 --- a/couchpotato/core/plugins/profile/static/profile.js +++ b/couchpotato/core/plugins/profile/static/profile.js @@ -140,7 +140,7 @@ var Profile = new Class({ }; Array.each(self.type_container.getElements('.type'), function(type){ - if(!type.hasClass('deleted') && type.getElement('select').get('value') != -1) + if(!type.hasClass('deleted') && type.getElement('select').get('value') != -1 && type.getElement('select').get('value') != "") data.types.include({ 'quality': type.getElement('select').get('value'), 'finish': +type.getElement('input.finish[type=checkbox]').checked, diff --git a/couchpotato/static/scripts/combined.plugins.min.js b/couchpotato/static/scripts/combined.plugins.min.js index dcc7da3..d193db4 100644 --- a/couchpotato/static/scripts/combined.plugins.min.js +++ b/couchpotato/static/scripts/combined.plugins.min.js @@ -3223,7 +3223,7 @@ var Profile = new Class({ types: [] }; Array.each(self.type_container.getElements(".type"), function(type) { - if (!type.hasClass("deleted") && type.getElement("select").get("value") != -1) data.types.include({ + if (!type.hasClass("deleted") && type.getElement("select").get("value") != -1 && type.getElement("select").get("value") != "") data.types.include({ quality: type.getElement("select").get("value"), finish: +type.getElement("input.finish[type=checkbox]").checked, "3d": +type.getElement("input.3d[type=checkbox]").checked diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index 6f5c184..9f6bbfd 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -182,7 +182,7 @@ 'click': self.checkForUpdate.bind(self, null) } })); - }; + } setting_links.each(function(a){ self.block.more.addLink(a); From d8a555c921df44c89a4da2588eff97717ad65ca8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 17 Dec 2016 14:32:32 +0100 Subject: [PATCH 17/17] Fix broken profiles with empty qualities --- couchpotato/core/plugins/profile/main.py | 12 ++++++++++++ couchpotato/core/plugins/profile/static/profile.js | 9 +++++---- couchpotato/static/scripts/combined.plugins.min.js | 9 +++++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index eed5f9d..aa112c5 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -62,6 +62,18 @@ class ProfilePlugin(Plugin): except: log.error('Failed: %s', traceback.format_exc()) + # Cleanup profiles that have empty qualites + profiles = self.all() + for profile in profiles: + try: + if '' in profile.get('qualities') or '-1' in profile.get('qualities'): + log.warning('Found profile with empty qualities, cleaning it up') + p = db.get('id', profile.get('_id')) + p['qualities'] = [x for x in p['qualities'] if (x != '' and x != '-1')] + db.update(p) + except: + log.error('Failed: %s', traceback.format_exc()) + def allView(self, **kwargs): return { diff --git a/couchpotato/core/plugins/profile/static/profile.js b/couchpotato/core/plugins/profile/static/profile.js index 36c8ad3..457aaa9 100644 --- a/couchpotato/core/plugins/profile/static/profile.js +++ b/couchpotato/core/plugins/profile/static/profile.js @@ -258,9 +258,10 @@ Profile.Type = new Class({ self.create(); self.addEvent('change', function(){ - self.el[self.qualities.get('value') == '-1' ? 'addClass' : 'removeClass']('is_empty'); - self.el[Quality.getQuality(self.qualities.get('value')).allow_3d ? 'addClass': 'removeClass']('allow_3d'); - self.deleted = self.qualities.get('value') == '-1'; + var has_quality = !(self.qualities.get('value') == '-1' || self.qualities.get('value') == ''); + self.el[!has_quality ? 'addClass' : 'removeClass']('is_empty'); + self.el[has_quality && Quality.getQuality(self.qualities.get('value')).allow_3d ? 'addClass': 'removeClass']('allow_3d'); + self.deleted = !has_quality; }); }, @@ -337,7 +338,7 @@ Profile.Type = new Class({ }).inject(self.qualities); }); - self.qualities.set('value', self.data.quality); + self.qualities.set('value', self.data.quality || -1); return self.qualities; diff --git a/couchpotato/static/scripts/combined.plugins.min.js b/couchpotato/static/scripts/combined.plugins.min.js index d193db4..475c053 100644 --- a/couchpotato/static/scripts/combined.plugins.min.js +++ b/couchpotato/static/scripts/combined.plugins.min.js @@ -3313,9 +3313,10 @@ Profile.Type = new Class({ self.data = data || {}; self.create(); self.addEvent("change", function() { - self.el[self.qualities.get("value") == "-1" ? "addClass" : "removeClass"]("is_empty"); - self.el[Quality.getQuality(self.qualities.get("value")).allow_3d ? "addClass" : "removeClass"]("allow_3d"); - self.deleted = self.qualities.get("value") == "-1"; + var has_quality = !(self.qualities.get("value") == "-1" || self.qualities.get("value") == ""); + self.el[!has_quality ? "addClass" : "removeClass"]("is_empty"); + self.el[has_quality && Quality.getQuality(self.qualities.get("value")).allow_3d ? "addClass" : "removeClass"]("allow_3d"); + self.deleted = !has_quality; }); }, create: function() { @@ -3364,7 +3365,7 @@ Profile.Type = new Class({ "data-allow_3d": q.allow_3d }).inject(self.qualities); }); - self.qualities.set("value", self.data.quality); + self.qualities.set("value", self.data.quality || -1); return self.qualities; }, getData: function() {