diff --git a/libs/pio/api.py b/libs/pio/api.py index ecfc177..01c4d1f 100644 --- a/libs/pio/api.py +++ b/libs/pio/api.py @@ -1,26 +1,53 @@ # -*- coding: utf-8 -*- - -# Changed -# Removed iso8601 library requirement +# Changed +# Removed iso8601 library requirement # Added CP logging import os -import re import json +import binascii import webbrowser -from urllib import urlencode -from couchpotato import CPLog -from dateutil.parser import parse +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode +from datetime import datetime +import tus import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry +from couchpotato import CPLog + + +KB = 1024 +MB = 1024 * KB + +# Read and write operations are limited to this chunk size. +# This can make a big difference when dealing with large files. +CHUNK_SIZE = 256 * KB BASE_URL = 'https://api.put.io/v2' +UPLOAD_URL = 'https://upload.put.io/v2/files/upload' +TUS_UPLOAD_URL = 'https://upload.put.io/files/' ACCESS_TOKEN_URL = 'https://api.put.io/v2/oauth2/access_token' AUTHENTICATION_URL = 'https://api.put.io/v2/oauth2/authenticate' log = CPLog(__name__) +class APIError(Exception): + pass + + +class ClientError(APIError): + pass + + +class ServerError(APIError): + pass + + class AuthHelper(object): def __init__(self, client_id, client_secret, redirect_uri, type='code'): @@ -58,10 +85,21 @@ class AuthHelper(object): class Client(object): - def __init__(self, access_token): + def __init__(self, access_token, use_retry=False): self.access_token = access_token self.session = requests.session() + if use_retry: + # Retry maximum 10 times, backoff on each retry + # Sleeps 1s, 2s, 4s, 8s, etc to a maximum of 120s between retries + # Retries on HTTP status codes 500, 502, 503, 504 + retries = Retry(total=10, + backoff_factor=1, + status_forcelist=[500, 502, 503, 504]) + + # Use the retry strategy for all HTTPS requests + self.session.mount('https://', HTTPAdapter(max_retries=retries)) + # Keep resource classes as attributes of client. # Pass client to resource classes so resource object # can use the client. @@ -71,7 +109,7 @@ class Client(object): self.Account = type('Account', (_Account,), attributes) def request(self, path, method='GET', params=None, data=None, files=None, - headers=None, raw=False, stream=False): + headers=None, raw=False, allow_redirects=True, stream=False): """ Wrapper around requests.request() @@ -91,27 +129,31 @@ class Client(object): headers['Accept'] = 'application/json' - url = BASE_URL + path + if path.startswith('https://'): + url = path + else: + url = BASE_URL + path log.debug('url: %s', url) response = self.session.request( method, url, params=params, data=data, files=files, - headers=headers, allow_redirects=True, stream=stream) + headers=headers, allow_redirects=allow_redirects, stream=stream) log.debug('response: %s', response) if raw: return response log.debug('content: %s', response.content) try: - response = json.loads(response.content) + body = json.loads(response.content.decode()) except ValueError: - raise Exception('Server didn\'t send valid JSON:\n%s\n%s' % ( - response, response.content)) + raise ServerError('InvalidJSON', response.content) - if response['status'] == 'ERROR': - raise Exception(response['error_type']) + if body['status'] == 'ERROR': + log.error("API returned error: %s", body) + exception_class = {'4': ClientError, '5': ServerError}[str(response.status_code)[0]] + raise exception_class(body['error_type'], body['error_message']) - return response + return body class _BaseResource(object): @@ -125,8 +167,8 @@ class _BaseResource(object): self.name = None self.__dict__.update(resource_dict) try: - self.created_at = parse(self.created_at) - except AttributeError: + self.created_at = strptime(self.created_at) + except Exception: self.created_at = None def __str__(self): @@ -135,7 +177,7 @@ class _BaseResource(object): def __repr__(self): # shorten name for display name = self.name[:17] + '...' if len(self.name) > 20 else self.name - return '<%s id=%r, name="%r">' % ( + return '<%s id=%r, name=%r>' % ( self.__class__.__name__, self.id, name) @@ -160,59 +202,113 @@ class _File(_BaseResource): files = {'file': (name, f)} else: files = {'file': f} - d = cls.client.request('/files/upload', method='POST', + d = cls.client.request(UPLOAD_URL, method='POST', data={'parent_id': parent_id}, files=files) f = d['file'] return cls(f) + @classmethod + def upload_tus(cls, path, name=None, parent_id=0): + headers = {'Authorization': 'token %s' % cls.client.access_token} + metadata = {'parent_id': str(parent_id)} + if name: + metadata['name'] = name + with open(path) as f: + tus.upload(f, TUS_UPLOAD_URL, file_name=name, headers=headers, metadata=metadata) + def dir(self): """List the files under directory.""" return self.list(parent_id=self.id) - def download(self, dest='.', delete_after_download=False): + def download(self, dest='.', delete_after_download=False, chunk_size=CHUNK_SIZE): if self.content_type == 'application/x-directory': - self._download_directory(dest, delete_after_download) + self._download_directory(dest, delete_after_download, chunk_size) else: - self._download_file(dest, delete_after_download) + self._download_file(dest, delete_after_download, chunk_size) - def _download_directory(self, dest='.', delete_after_download=False): - name = self.name - if isinstance(name, unicode): - name = name.encode('utf-8', 'replace') + def _download_directory(self, dest, delete_after_download, chunk_size): + name = _str(self.name) dest = os.path.join(dest, name) if not os.path.exists(dest): os.mkdir(dest) for sub_file in self.dir(): - sub_file.download(dest, delete_after_download) + sub_file.download(dest, delete_after_download, chunk_size) if delete_after_download: self.delete() - def _download_file(self, dest='.', delete_after_download=False): - response = self.client.request( - '/files/%s/download' % self.id, raw=True, stream=True) + def _verify_file(self, filepath): + log.info('verifying crc32...') + filesize = os.path.getsize(filepath) + if self.size != filesize: + logging.error('file %s is %d bytes, should be %s bytes' % (filepath, filesize, self.size)) + return False - filename = re.match( - 'attachment; filename=(.*)', - response.headers['content-disposition']).groups()[0] - # If file name has spaces, it must have quotes around. - filename = filename.strip('"') + crcbin = 0 + with open(filepath, 'rb') as f: + while True: + chunk = f.read(CHUNK_SIZE) + if not chunk: + break - with open(os.path.join(dest, filename), 'wb') as f: - for chunk in response.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - f.flush() + crcbin = binascii.crc32(chunk, crcbin) & 0xffffffff - if delete_after_download: - self.delete() + crc32 = '%08x' % crcbin + + if crc32 != self.crc32: + logging.error('file %s CRC32 is %s, should be %s' % (filepath, crc32, self.crc32)) + return False + + return True + + def _download_file(self, dest, delete_after_download, chunk_size): + name = _str(self.name) + + filepath = os.path.join(dest, name) + if os.path.exists(filepath): + first_byte = os.path.getsize(filepath) + + if first_byte == self.size: + log.warning('file %s exists and is the correct size %d' % (filepath, self.size)) + else: + first_byte = 0 + + log.debug('file %s is currently %d, should be %d' % (filepath, first_byte, self.size)) + + if self.size == 0: + # Create an empty file + open(filepath, 'w').close() + log.debug('created empty file %s' % filepath) + else: + if first_byte < self.size: + with open(filepath, 'ab') as f: + headers = {'Range': 'bytes=%d-' % first_byte} + + log.debug('request range: bytes=%d-' % first_byte) + response = self.client.request('/files/%s/download' % self.id, + headers=headers, + raw=True, + stream=True) + + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + + if self._verify_file(filepath): + if delete_after_download: + self.delete() def delete(self): return self.client.request('/files/delete', method='POST', - data={'file_ids': str(self.id)}) + data={'file_id': str(self.id)}) + + @classmethod + def delete_multi(cls, ids): + return cls.client.request('/files/delete', method='POST', + data={'file_ids': ','.join(map(str, ids))}) def move(self, parent_id): return self.client.request('/files/move', method='POST', @@ -239,6 +335,7 @@ class _Transfer(_BaseResource): @classmethod def add_url(cls, url, parent_id=0, extract=False, callback_url=None): + log.debug('callback_url is %s', callback_url) d = cls.client.request('/transfers/add', method='POST', data=dict( url=url, save_parent_id=parent_id, extract=extract, callback_url=callback_url)) @@ -247,10 +344,10 @@ class _Transfer(_BaseResource): @classmethod def add_torrent(cls, path, parent_id=0, extract=False, callback_url=None): - with open(path) as f: + with open(path, 'rb') as f: files = {'file': f} d = cls.client.request('/files/upload', method='POST', files=files, - data=dict(save_parent_id=parent_id, + data=dict(parent_id=parent_id, extract=extract, callback_url=callback_url)) t = d['transfer'] @@ -260,6 +357,17 @@ class _Transfer(_BaseResource): def clean(cls): return cls.client.request('/transfers/clean', method='POST') + def cancel(self): + return self.client.request('/transfers/cancel', + method='POST', + data={'transfer_ids': self.id}) + + @classmethod + def cancel_multi(cls, ids): + return cls.client.request('/transfers/cancel', + method='POST', + data={'transfer_ids': ','.join(map(str, ids))}) + class _Account(_BaseResource): @@ -270,3 +378,31 @@ class _Account(_BaseResource): @classmethod def settings(cls): return cls.client.request('/account/settings', method='GET') + + +# Due to a nasty bug in datetime module, datetime.strptime calls +# are not thread-safe and can throw a TypeError. Details: https://bugs.python.org/issue7980 +# Here we are implementing simple RFC3339 parser which is used in Put.io APIv2. +def strptime(date): + """Returns datetime object from the given date, which is in a specific format: YYYY-MM-ddTHH:mm:ss""" + d = { + 'year': date[0:4], + 'month': date[5:7], + 'day': date[8:10], + 'hour': date[11:13], + 'minute': date[14:16], + 'second': date[17:], + } + + d = dict((k, int(v)) for k, v in d.iteritems()) + return datetime(**d) + + +def _str(s): + """Python 3 compatibility function for converting to str.""" + try: + if isinstance(s, unicode): + return s.encode('utf-8', 'replace') + except NameError: + pass + return s