Browse Source

Merge branch 'master' into desktop

Conflicts:
	couchpotato/core/database.py
tags/build/2.6.0
Ruud 11 years ago
parent
commit
c657d6d70b
  1. 1
      CouchPotato.py
  2. 20
      README.md
  3. 6
      contributing.md
  4. 2
      couchpotato/api.py
  5. 10
      couchpotato/core/_base/_core.py
  6. 25
      couchpotato/core/_base/updater/main.py
  7. 596
      couchpotato/core/database.py
  8. 103
      couchpotato/core/downloaders/nzbvortex.py
  9. 24
      couchpotato/core/downloaders/transmission.py
  10. 2
      couchpotato/core/event.py
  11. 9
      couchpotato/core/helpers/encoding.py
  12. 34
      couchpotato/core/helpers/variable.py
  13. 17
      couchpotato/core/logger.py
  14. 23
      couchpotato/core/media/__init__.py
  15. 110
      couchpotato/core/media/_base/library/main.py
  16. 6
      couchpotato/core/media/_base/matcher/main.py
  17. 117
      couchpotato/core/media/_base/media/main.py
  18. 23
      couchpotato/core/media/_base/providers/nzb/newznab.py
  19. 126
      couchpotato/core/media/_base/providers/nzb/nzbindex.py
  20. 2
      couchpotato/core/media/_base/providers/torrent/awesomehd.py
  21. 13
      couchpotato/core/media/_base/providers/torrent/bitsoup.py
  22. 3
      couchpotato/core/media/_base/providers/torrent/kickasstorrents.py
  23. 12
      couchpotato/core/media/_base/providers/torrent/passthepopcorn.py
  24. 12
      couchpotato/core/media/_base/providers/torrent/thepiratebay.py
  25. 12
      couchpotato/core/media/_base/providers/torrent/torrentleech.py
  26. 19
      couchpotato/core/media/_base/providers/torrent/torrentshack.py
  27. 20
      couchpotato/core/media/_base/searcher/__init__.py
  28. 51
      couchpotato/core/media/_base/searcher/main.py
  29. 59
      couchpotato/core/media/movie/_base/main.py
  30. 15
      couchpotato/core/media/movie/_base/static/movie.actions.js
  31. 16
      couchpotato/core/media/movie/_base/static/movie.js
  32. 7
      couchpotato/core/media/movie/charts/__init__.py
  33. 6
      couchpotato/core/media/movie/charts/main.py
  34. 32
      couchpotato/core/media/movie/charts/static/charts.js
  35. 55
      couchpotato/core/media/movie/providers/automation/bluray.py
  36. 4
      couchpotato/core/media/movie/providers/info/couchpotatoapi.py
  37. 5
      couchpotato/core/media/movie/providers/info/fanarttv.py
  38. 273
      couchpotato/core/media/movie/providers/info/themoviedb.py
  39. 2
      couchpotato/core/media/movie/providers/metadata/base.py
  40. 30
      couchpotato/core/media/movie/providers/nzb/nzbindex.py
  41. 2
      couchpotato/core/media/movie/providers/torrent/iptorrents.py
  42. 4
      couchpotato/core/media/movie/providers/torrent/passthepopcorn.py
  43. 2
      couchpotato/core/media/movie/providers/torrent/torrentleech.py
  44. 10
      couchpotato/core/media/movie/providers/trailer/hdtrailers.py
  45. 4
      couchpotato/core/media/movie/providers/userscript/filmstarts.py
  46. 39
      couchpotato/core/media/movie/searcher.py
  47. 2
      couchpotato/core/media/movie/suggestion/main.py
  48. 31
      couchpotato/core/media/movie/suggestion/static/suggest.js
  49. 25
      couchpotato/core/notifications/core/main.py
  50. 4
      couchpotato/core/notifications/core/static/notification.js
  51. 2
      couchpotato/core/notifications/email_.py
  52. 8
      couchpotato/core/notifications/growl.py
  53. 68
      couchpotato/core/notifications/notifymywp.py
  54. 3
      couchpotato/core/notifications/pushbullet.py
  55. 6
      couchpotato/core/notifications/pushover.py
  56. 7
      couchpotato/core/notifications/trakt.py
  57. 8
      couchpotato/core/notifications/xbmc.py
  58. 3
      couchpotato/core/plugins/automation.py
  59. 121
      couchpotato/core/plugins/base.py
  60. 29
      couchpotato/core/plugins/browser.py
  61. 2
      couchpotato/core/plugins/category/main.py
  62. 24
      couchpotato/core/plugins/dashboard.py
  63. 13
      couchpotato/core/plugins/file.py
  64. 2
      couchpotato/core/plugins/log/static/log.js
  65. 10
      couchpotato/core/plugins/manage.py
  66. 2
      couchpotato/core/plugins/profile/main.py
  67. 5
      couchpotato/core/plugins/profile/static/profile.css
  68. 12
      couchpotato/core/plugins/profile/static/profile.js
  69. 123
      couchpotato/core/plugins/quality/main.py
  70. 69
      couchpotato/core/plugins/release/main.py
  71. 157
      couchpotato/core/plugins/renamer.py
  72. 34
      couchpotato/core/plugins/scanner.py
  73. 101
      couchpotato/core/plugins/score/scores.py
  74. 32
      couchpotato/runner.py
  75. 15
      couchpotato/static/scripts/couchpotato.js
  76. 84
      couchpotato/static/scripts/page/home.js
  77. 136
      couchpotato/static/scripts/page/settings.js
  78. 13
      couchpotato/static/style/settings.css
  79. 2
      couchpotato/templates/index.html
  80. 12
      couchpotato/templates/login.html
  81. 6
      init/ubuntu
  82. 6
      libs/axl/axel.py
  83. 14
      libs/chardet/__init__.py
  84. 20
      libs/chardet/big5freq.py
  85. 17
      libs/chardet/big5prober.py
  86. 46
      libs/chardet/chardetect.py
  87. 159
      libs/chardet/chardistribution.py
  88. 36
      libs/chardet/charsetgroupprober.py
  89. 24
      libs/chardet/charsetprober.py
  90. 15
      libs/chardet/codingstatemachine.py
  91. 34
      libs/chardet/compat.py
  92. 8
      libs/chardet/constants.py
  93. 44
      libs/chardet/cp949prober.py
  94. 39
      libs/chardet/escprober.py
  95. 338
      libs/chardet/escsm.py
  96. 45
      libs/chardet/eucjpprober.py
  97. 2
      libs/chardet/euckrfreq.py
  98. 13
      libs/chardet/euckrprober.py
  99. 16
      libs/chardet/euctwfreq.py
  100. 8
      libs/chardet/euctwprober.py

1
CouchPotato.py

@ -10,7 +10,6 @@ import socket
import subprocess import subprocess
import sys import sys
import traceback import traceback
import time
# Root path # Root path
base_path = dirname(os.path.abspath(__file__)) base_path = dirname(os.path.abspath(__file__))

20
README.md

@ -29,19 +29,25 @@ OS X:
* Then do `python CouchPotatoServer/CouchPotato.py` * Then do `python CouchPotatoServer/CouchPotato.py`
* Your browser should open up, but if it doesn't go to `http://localhost:5050/` * Your browser should open up, but if it doesn't go to `http://localhost:5050/`
Linux (Ubuntu / Debian): Linux:
* Install [GIT](http://git-scm.com/) with `apt-get install git-core` * (Ubuntu / Debian) Install [GIT](http://git-scm.com/) with `apt-get install git-core`
* (Fedora / CentOS) Install [GIT](http://git-scm.com/) with `yum install git`
* 'cd' to the folder of your choosing. * 'cd' to the folder of your choosing.
* Run `git clone https://github.com/RuudBurger/CouchPotatoServer.git` * Run `git clone https://github.com/RuudBurger/CouchPotatoServer.git`
* Then do `python CouchPotatoServer/CouchPotato.py` to start * Then do `python CouchPotatoServer/CouchPotato.py` to start
* To run on boot copy the init script `sudo cp CouchPotatoServer/init/ubuntu /etc/init.d/couchpotato` * (Ubuntu / Debian) To run on boot copy the init script `sudo cp CouchPotatoServer/init/ubuntu /etc/init.d/couchpotato`
* Copy the default paths file `sudo cp CouchPotatoServer/init/ubuntu.default /etc/default/couchpotato` * (Ubuntu / Debian) Copy the default paths file `sudo cp CouchPotatoServer/init/ubuntu.default /etc/default/couchpotato`
* Change the paths inside the default file `sudo nano /etc/default/couchpotato` * (Ubuntu / Debian) Change the paths inside the default file `sudo nano /etc/default/couchpotato`
* Make it executable `sudo chmod +x /etc/init.d/couchpotato` * (Ubuntu / Debian) Make it executable `sudo chmod +x /etc/init.d/couchpotato`
* Add it to defaults `sudo update-rc.d couchpotato defaults` * (Ubuntu / Debian) Add it to defaults `sudo update-rc.d couchpotato defaults`
* (systemd) To run on boot copy the systemd config `sudo cp CouchPotatoServer/init/couchpotato.fedora.service /etc/systemd/system/couchpotato.service`
* (systemd) Update the systemd config file with your user and path to CouchPotato.py
* (systemd) Enable it at boot with `sudo systemctl enable couchpotato`
* Open your browser and go to `http://localhost:5050/` * Open your browser and go to `http://localhost:5050/`
Docker:
* You can use [razorgirl's Dockerfile](https://github.com/razorgirl/docker-couchpotato) to quickly build your own isolated app container. It's based on the Linux instructions above. For more info about Docker check out the [official website](https://www.docker.com).
FreeBSD : FreeBSD :

6
contributing.md

@ -13,6 +13,8 @@ Lastly, for anything related to CouchPotato, feel free to stop by the [forum](ht
## Issues ## Issues
Issues are intended for reporting bugs and weird behaviour or suggesting improvements to CouchPotatoServer. Issues are intended for reporting bugs and weird behaviour or suggesting improvements to CouchPotatoServer.
Before you submit an issue, please go through the following checklist: Before you submit an issue, please go through the following checklist:
* **FILL IN ALL THE FIELDS ASKED FOR**
* **POST MORE THAN A SINGLE LINE LOG**, if you do, you'd better have a easy reproducable bug
* Search through existing issues (*including closed issues!*) first: you might be able to get your answer there. * Search through existing issues (*including closed issues!*) first: you might be able to get your answer there.
* Double check your issue manually, because it could be an external issue. * Double check your issue manually, because it could be an external issue.
* Post logs with your issue: Without seeing what is going on, the developers can't reproduce the error. * Post logs with your issue: Without seeing what is going on, the developers can't reproduce the error.
@ -25,12 +27,14 @@ Before you submit an issue, please go through the following checklist:
* What hardware / OS are you using and what are its limitations? For example: NAS can be slow and maybe have a different version of python installed than when you use CP on OS X or Windows. * What hardware / OS are you using and what are its limitations? For example: NAS can be slow and maybe have a different version of python installed than when you use CP on OS X or Windows.
* Your issue might be marked with the "can't reproduce" tag. Don't ask why your issue was closed if it says so in the tag. * Your issue might be marked with the "can't reproduce" tag. Don't ask why your issue was closed if it says so in the tag.
* If you're running on a NAS (QNAP, Austor, Synology etc.) with pre-made packages, make sure these are set up to use our source repository (RuudBurger/CouchPotatoServer) and nothing else! * If you're running on a NAS (QNAP, Austor, Synology etc.) with pre-made packages, make sure these are set up to use our source repository (RuudBurger/CouchPotatoServer) and nothing else!
* Do not "bump" issues with "Any updates on this" or whatever. Yes I've seen it, you don't have to remind me of it. There will be an update when the code is done or I need information. If you feel the need to do so, you'd better have more info on the issue.
The more relevant information you provide, the more likely that your issue will be resolved. The more relevant information you provide, the more likely that your issue will be resolved.
If you don't follow any of the checks above, I'll close the issue. If you are wondering why (and ask) I'll block you from posting new issues and the repo.
## Pull Requests ## Pull Requests
Pull requests are intended for contributing code or documentation to the project. Before you submit a pull request, consider the following: Pull requests are intended for contributing code or documentation to the project. Before you submit a pull request, consider the following:
* Make sure your pull request is made for the *develop* branch (or relevant feature branch). * Make sure your pull request is made for the *develop* branch (or relevant feature branch).
* Have you tested your PR? If not, why? * Have you tested your PR? If not, why?
* Does your PR have any limitations we should know of? * Does your PR have any limitations I should know of?
* Is your PR up-to-date with the branch you're trying to push into? * Is your PR up-to-date with the branch you're trying to push into?

2
couchpotato/api.py

@ -143,6 +143,8 @@ class ApiHandler(RequestHandler):
else: else:
self.write(result) self.write(result)
self.finish() self.finish()
except UnicodeDecodeError:
log.error('Failed proper encode: %s', traceback.format_exc())
except: except:
log.debug('Failed doing request, probably already closed: %s', (traceback.format_exc())) log.debug('Failed doing request, probably already closed: %s', (traceback.format_exc()))
try: self.finish({'success': False, 'error': 'Failed returning results'}) try: self.finish({'success': False, 'error': 'Failed returning results'})

10
couchpotato/core/_base/_core.py

@ -181,13 +181,13 @@ class Core(Plugin):
return '%sapi/%s' % (self.createBaseUrl(), Env.setting('api_key')) return '%sapi/%s' % (self.createBaseUrl(), Env.setting('api_key'))
def version(self): def version(self):
ver = fireEvent('updater.info', single = True) ver = fireEvent('updater.info', single = True) or {'version': {}}
if os.name == 'nt': platf = 'windows' if os.name == 'nt': platf = 'windows'
elif 'Darwin' in platform.platform(): platf = 'osx' elif 'Darwin' in platform.platform(): platf = 'osx'
else: platf = 'linux' else: platf = 'linux'
return '%s - %s-%s - v2' % (platf, ver.get('version')['type'], ver.get('version')['hash']) return '%s - %s-%s - v2' % (platf, ver.get('version').get('type') or 'unknown', ver.get('version').get('hash') or 'unknown')
def versionView(self, **kwargs): def versionView(self, **kwargs):
return { return {
@ -286,13 +286,13 @@ config = [{
'name': 'permission_folder', 'name': 'permission_folder',
'default': '0755', 'default': '0755',
'label': 'Folder CHMOD', 'label': 'Folder CHMOD',
'description': 'Can be either decimal (493) or octal (leading zero: 0755)', 'description': 'Can be either decimal (493) or octal (leading zero: 0755). <a target="_blank" href="http://permissions-calculator.org/">Calculate the correct value</a>',
}, },
{ {
'name': 'permission_file', 'name': 'permission_file',
'default': '0755', 'default': '0644',
'label': 'File CHMOD', 'label': 'File CHMOD',
'description': 'Same as Folder CHMOD but for files', 'description': 'See Folder CHMOD description, but for files',
}, },
], ],
}, },

25
couchpotato/core/_base/updater/main.py

@ -205,19 +205,28 @@ class GitUpdater(BaseUpdater):
def getVersion(self): def getVersion(self):
if not self.version: if not self.version:
hash = None
date = None
branch = self.branch
try: try:
output = self.repo.getHead() # Yes, please output = self.repo.getHead() # Yes, please
log.debug('Git version output: %s', output.hash) log.debug('Git version output: %s', output.hash)
self.version = {
'repr': 'git:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, self.repo.getCurrentBranch().name or self.branch, output.hash[:8], datetime.fromtimestamp(output.getDate())), hash = output.hash[:8]
'hash': output.hash[:8], date = output.getDate()
'date': output.getDate(), branch = self.repo.getCurrentBranch().name
'type': 'git',
'branch': self.repo.getCurrentBranch().name
}
except Exception as e: except Exception as e:
log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s', e) log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s', e)
return 'No GIT'
self.version = {
'repr': 'git:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, branch, hash or 'unknown_hash', datetime.fromtimestamp(date) if date else 'unknown_date'),
'hash': hash,
'date': date,
'type': 'git',
'branch': branch
}
return self.version return self.version

596
couchpotato/core/database.py

@ -2,6 +2,7 @@ import json
import os import os
import time import time
import traceback import traceback
from sqlite3 import OperationalError
from CodernityDB.database import RecordNotFound from CodernityDB.database import RecordNotFound
from CodernityDB.index import IndexException, IndexNotFoundException, IndexConflict from CodernityDB.index import IndexException, IndexNotFoundException, IndexConflict
@ -9,7 +10,7 @@ from couchpotato import CPLog
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import toUnicode, sp from couchpotato.core.helpers.encoding import toUnicode, sp
from couchpotato.core.helpers.variable import getImdb, tryInt from couchpotato.core.helpers.variable import getImdb, tryInt, randomString
log = CPLog(__name__) log = CPLog(__name__)
@ -32,6 +33,7 @@ class Database(object):
addEvent('database.setup.after', self.startup_compact) addEvent('database.setup.after', self.startup_compact)
addEvent('database.setup_index', self.setupIndex) addEvent('database.setup_index', self.setupIndex)
addEvent('database.delete_corrupted', self.deleteCorrupted)
addEvent('app.migrate', self.migrate) addEvent('app.migrate', self.migrate)
addEvent('app.after_shutdown', self.close) addEvent('app.after_shutdown', self.close)
@ -147,6 +149,17 @@ class Database(object):
return results return results
def deleteCorrupted(self, _id, traceback_error = ''):
db = self.getDB()
try:
log.debug('Deleted corrupted document "%s": %s', (_id, traceback_error))
corrupted = db.get('id', _id, with_storage = False)
db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None)
except:
log.debug('Failed deleting corrupted: %s', traceback.format_exc())
def reindex(self, **kwargs): def reindex(self, **kwargs):
success = True success = True
@ -299,309 +312,326 @@ class Database(object):
} }
migrate_data = {} migrate_data = {}
rename_old = False
c = conn.cursor() try:
for ml in migrate_list: c = conn.cursor()
migrate_data[ml] = {}
rows = migrate_list[ml]
try: for ml in migrate_list:
c.execute('SELECT %s FROM `%s`' % ('`' + '`,`'.join(rows) + '`', ml)) migrate_data[ml] = {}
except: rows = migrate_list[ml]
# ignore faulty destination_id database
if ml == 'category':
migrate_data[ml] = {}
else:
raise
for p in c.fetchall():
columns = {}
for row in migrate_list[ml]:
columns[row] = p[rows.index(row)]
if not migrate_data[ml].get(p[0]): try:
migrate_data[ml][p[0]] = columns c.execute('SELECT %s FROM `%s`' % ('`' + '`,`'.join(rows) + '`', ml))
except:
# ignore faulty destination_id database
if ml == 'category':
migrate_data[ml] = {}
else:
rename_old = True
raise
for p in c.fetchall():
columns = {}
for row in migrate_list[ml]:
columns[row] = p[rows.index(row)]
if not migrate_data[ml].get(p[0]):
migrate_data[ml][p[0]] = columns
else:
if not isinstance(migrate_data[ml][p[0]], list):
migrate_data[ml][p[0]] = [migrate_data[ml][p[0]]]
migrate_data[ml][p[0]].append(columns)
conn.close()
log.info('Getting data took %s', time.time() - migrate_start)
db = self.getDB()
if not db.opened:
return
# Use properties
properties = migrate_data['properties']
log.info('Importing %s properties', len(properties))
for x in properties:
property = properties[x]
Env.prop(property.get('identifier'), property.get('value'))
# Categories
categories = migrate_data.get('category', [])
log.info('Importing %s categories', len(categories))
category_link = {}
for x in categories:
c = categories[x]
new_c = db.insert({
'_t': 'category',
'order': c.get('order', 999),
'label': toUnicode(c.get('label', '')),
'ignored': toUnicode(c.get('ignored', '')),
'preferred': toUnicode(c.get('preferred', '')),
'required': toUnicode(c.get('required', '')),
'destination': toUnicode(c.get('destination', '')),
})
category_link[x] = new_c.get('_id')
# Profiles
log.info('Importing profiles')
new_profiles = db.all('profile', with_doc = True)
new_profiles_by_label = {}
for x in new_profiles:
# Remove default non core profiles
if not x['doc'].get('core'):
db.delete(x['doc'])
else: else:
if not isinstance(migrate_data[ml][p[0]], list): new_profiles_by_label[x['doc']['label']] = x['_id']
migrate_data[ml][p[0]] = [migrate_data[ml][p[0]]]
migrate_data[ml][p[0]].append(columns)
conn.close() profiles = migrate_data['profile']
profile_link = {}
for x in profiles:
p = profiles[x]
log.info('Getting data took %s', time.time() - migrate_start) exists = new_profiles_by_label.get(p.get('label'))
db = self.getDB() # Update existing with order only
if not db.opened: if exists and p.get('core'):
return profile = db.get('id', exists)
profile['order'] = tryInt(p.get('order'))
profile['hide'] = p.get('hide') in [1, True, 'true', 'True']
db.update(profile)
# Use properties profile_link[x] = profile.get('_id')
properties = migrate_data['properties'] else:
log.info('Importing %s properties', len(properties))
for x in properties:
property = properties[x]
Env.prop(property.get('identifier'), property.get('value'))
# Categories
categories = migrate_data.get('category', [])
log.info('Importing %s categories', len(categories))
category_link = {}
for x in categories:
c = categories[x]
new_c = db.insert({
'_t': 'category',
'order': c.get('order', 999),
'label': toUnicode(c.get('label', '')),
'ignored': toUnicode(c.get('ignored', '')),
'preferred': toUnicode(c.get('preferred', '')),
'required': toUnicode(c.get('required', '')),
'destination': toUnicode(c.get('destination', '')),
})
category_link[x] = new_c.get('_id')
# Profiles
log.info('Importing profiles')
new_profiles = db.all('profile', with_doc = True)
new_profiles_by_label = {}
for x in new_profiles:
# Remove default non core profiles
if not x['doc'].get('core'):
db.delete(x['doc'])
else:
new_profiles_by_label[x['doc']['label']] = x['_id']
profiles = migrate_data['profile']
profile_link = {}
for x in profiles:
p = profiles[x]
exists = new_profiles_by_label.get(p.get('label'))
# Update existing with order only new_profile = {
if exists and p.get('core'): '_t': 'profile',
profile = db.get('id', exists) 'label': p.get('label'),
profile['order'] = tryInt(p.get('order')) 'order': int(p.get('order', 999)),
profile['hide'] = p.get('hide') in [1, True, 'true', 'True'] 'core': p.get('core', False),
db.update(profile) 'qualities': [],
'wait_for': [],
'finish': []
}
profile_link[x] = profile.get('_id') types = migrate_data['profiletype']
else: for profile_type in types:
p_type = types[profile_type]
if types[profile_type]['profile_id'] == p['id']:
if p_type['quality_id']:
new_profile['finish'].append(p_type['finish'])
new_profile['wait_for'].append(p_type['wait_for'])
new_profile['qualities'].append(migrate_data['quality'][p_type['quality_id']]['identifier'])
if len(new_profile['qualities']) > 0:
new_profile.update(db.insert(new_profile))
profile_link[x] = new_profile.get('_id')
else:
log.error('Corrupt profile list for "%s", using default.', p.get('label'))
# Qualities
log.info('Importing quality sizes')
new_qualities = db.all('quality', with_doc = True)
new_qualities_by_identifier = {}
for x in new_qualities:
new_qualities_by_identifier[x['doc']['identifier']] = x['_id']
qualities = migrate_data['quality']
quality_link = {}
for x in qualities:
q = qualities[x]
q_id = new_qualities_by_identifier[q.get('identifier')]
quality = db.get('id', q_id)
quality['order'] = q.get('order')
quality['size_min'] = tryInt(q.get('size_min'))
quality['size_max'] = tryInt(q.get('size_max'))
db.update(quality)
quality_link[x] = quality
# Titles
titles = migrate_data['librarytitle']
titles_by_library = {}
for x in titles:
title = titles[x]
if title.get('default'):
titles_by_library[title.get('libraries_id')] = title.get('title')
# Releases
releaseinfos = migrate_data['releaseinfo']
for x in releaseinfos:
info = releaseinfos[x]
# Skip if release doesn't exist for this info
if not migrate_data['release'].get(info.get('release_id')):
continue
new_profile = { if not migrate_data['release'][info.get('release_id')].get('info'):
'_t': 'profile', migrate_data['release'][info.get('release_id')]['info'] = {}
'label': p.get('label'),
'order': int(p.get('order', 999)), migrate_data['release'][info.get('release_id')]['info'][info.get('identifier')] = info.get('value')
'core': p.get('core', False),
'qualities': [], releases = migrate_data['release']
'wait_for': [], releases_by_media = {}
'finish': [] for x in releases:
} release = releases[x]
if not releases_by_media.get(release.get('movie_id')):
types = migrate_data['profiletype'] releases_by_media[release.get('movie_id')] = []
for profile_type in types:
p_type = types[profile_type] releases_by_media[release.get('movie_id')].append(release)
if types[profile_type]['profile_id'] == p['id']:
if p_type['quality_id']: # Type ids
new_profile['finish'].append(p_type['finish']) types = migrate_data['filetype']
new_profile['wait_for'].append(p_type['wait_for']) type_by_id = {}
new_profile['qualities'].append(migrate_data['quality'][p_type['quality_id']]['identifier']) for t in types:
type = types[t]
if len(new_profile['qualities']) > 0: type_by_id[type.get('id')] = type
new_profile.update(db.insert(new_profile))
profile_link[x] = new_profile.get('_id') # Media
else: log.info('Importing %s media items', len(migrate_data['movie']))
log.error('Corrupt profile list for "%s", using default.', p.get('label')) statuses = migrate_data['status']
libraries = migrate_data['library']
# Qualities library_files = migrate_data['library_files__file_library']
log.info('Importing quality sizes') releases_files = migrate_data['release_files__file_release']
new_qualities = db.all('quality', with_doc = True) all_files = migrate_data['file']
new_qualities_by_identifier = {} poster_type = migrate_data['filetype']['poster']
for x in new_qualities: medias = migrate_data['movie']
new_qualities_by_identifier[x['doc']['identifier']] = x['_id'] for x in medias:
m = medias[x]
qualities = migrate_data['quality']
quality_link = {} status = statuses.get(m['status_id']).get('identifier')
for x in qualities: l = libraries.get(m['library_id'])
q = qualities[x]
q_id = new_qualities_by_identifier[q.get('identifier')] # Only migrate wanted movies, Skip if no identifier present
if not l or not getImdb(l.get('identifier')): continue
quality = db.get('id', q_id)
quality['order'] = q.get('order') profile_id = profile_link.get(m['profile_id'])
quality['size_min'] = tryInt(q.get('size_min')) category_id = category_link.get(m['category_id'])
quality['size_max'] = tryInt(q.get('size_max')) title = titles_by_library.get(m['library_id'])
db.update(quality) releases = releases_by_media.get(x, [])
info = json.loads(l.get('info', ''))
quality_link[x] = quality
files = library_files.get(m['library_id'], [])
# Titles if not isinstance(files, list):
titles = migrate_data['librarytitle'] files = [files]
titles_by_library = {}
for x in titles: added_media = fireEvent('movie.add', {
title = titles[x] 'info': info,
if title.get('default'): 'identifier': l.get('identifier'),
titles_by_library[title.get('libraries_id')] = title.get('title') 'profile_id': profile_id,
'category_id': category_id,
# Releases 'title': title
releaseinfos = migrate_data['releaseinfo'] }, force_readd = False, search_after = False, update_after = False, notify_after = False, status = status, single = True)
for x in releaseinfos:
info = releaseinfos[x] if not added_media:
log.error('Failed adding media %s: %s', (l.get('identifier'), info))
# Skip if release doesn't exist for this info
if not migrate_data['release'].get(info.get('release_id')):
continue
if not migrate_data['release'][info.get('release_id')].get('info'):
migrate_data['release'][info.get('release_id')]['info'] = {}
migrate_data['release'][info.get('release_id')]['info'][info.get('identifier')] = info.get('value')
releases = migrate_data['release']
releases_by_media = {}
for x in releases:
release = releases[x]
if not releases_by_media.get(release.get('movie_id')):
releases_by_media[release.get('movie_id')] = []
releases_by_media[release.get('movie_id')].append(release)
# Type ids
types = migrate_data['filetype']
type_by_id = {}
for t in types:
type = types[t]
type_by_id[type.get('id')] = type
# Media
log.info('Importing %s media items', len(migrate_data['movie']))
statuses = migrate_data['status']
libraries = migrate_data['library']
library_files = migrate_data['library_files__file_library']
releases_files = migrate_data['release_files__file_release']
all_files = migrate_data['file']
poster_type = migrate_data['filetype']['poster']
medias = migrate_data['movie']
for x in medias:
m = medias[x]
status = statuses.get(m['status_id']).get('identifier')
l = libraries.get(m['library_id'])
# Only migrate wanted movies, Skip if no identifier present
if not l or not getImdb(l.get('identifier')): continue
profile_id = profile_link.get(m['profile_id'])
category_id = category_link.get(m['category_id'])
title = titles_by_library.get(m['library_id'])
releases = releases_by_media.get(x, [])
info = json.loads(l.get('info', ''))
files = library_files.get(m['library_id'], [])
if not isinstance(files, list):
files = [files]
added_media = fireEvent('movie.add', {
'info': info,
'identifier': l.get('identifier'),
'profile_id': profile_id,
'category_id': category_id,
'title': title
}, force_readd = False, search_after = False, update_after = False, notify_after = False, status = status, single = True)
if not added_media:
log.error('Failed adding media %s: %s', (l.get('identifier'), info))
continue
added_media['files'] = added_media.get('files', {})
for f in files:
ffile = all_files[f.get('file_id')]
# Only migrate posters
if ffile.get('type_id') == poster_type.get('id'):
if ffile.get('path') not in added_media['files'].get('image_poster', []) and os.path.isfile(ffile.get('path')):
added_media['files']['image_poster'] = [ffile.get('path')]
break
if 'image_poster' in added_media['files']:
db.update(added_media)
for rel in releases:
empty_info = False
if not rel.get('info'):
empty_info = True
rel['info'] = {}
quality = quality_link.get(rel.get('quality_id'))
if not quality:
continue continue
release_status = statuses.get(rel.get('status_id')).get('identifier') added_media['files'] = added_media.get('files', {})
for f in files:
ffile = all_files[f.get('file_id')]
if rel['info'].get('download_id'): # Only migrate posters
status_support = rel['info'].get('download_status_support', False) in [True, 'true', 'True'] if ffile.get('type_id') == poster_type.get('id'):
rel['info']['download_info'] = { if ffile.get('path') not in added_media['files'].get('image_poster', []) and os.path.isfile(ffile.get('path')):
'id': rel['info'].get('download_id'), added_media['files']['image_poster'] = [ffile.get('path')]
'downloader': rel['info'].get('download_downloader'), break
'status_support': status_support,
}
# Add status to keys if 'image_poster' in added_media['files']:
rel['info']['status'] = release_status db.update(added_media)
if not empty_info:
fireEvent('release.create_from_search', [rel['info']], added_media, quality, single = True)
else:
release = {
'_t': 'release',
'identifier': rel.get('identifier'),
'media_id': added_media.get('_id'),
'quality': quality.get('identifier'),
'status': release_status,
'last_edit': int(time.time()),
'files': {}
}
# Add downloader info if provided for rel in releases:
try:
release['download_info'] = rel['info']['download_info']
del rel['download_info']
except:
pass
# Add files empty_info = False
release_files = releases_files.get(rel.get('id'), []) if not rel.get('info'):
if not isinstance(release_files, list): empty_info = True
release_files = [release_files] rel['info'] = {}
if len(release_files) == 0: quality = quality_link.get(rel.get('quality_id'))
if not quality:
continue continue
for f in release_files: release_status = statuses.get(rel.get('status_id')).get('identifier')
rfile = all_files[f.get('file_id')]
file_type = type_by_id.get(rfile.get('type_id')).get('identifier') if rel['info'].get('download_id'):
status_support = rel['info'].get('download_status_support', False) in [True, 'true', 'True']
if not release['files'].get(file_type): rel['info']['download_info'] = {
release['files'][file_type] = [] 'id': rel['info'].get('download_id'),
'downloader': rel['info'].get('download_downloader'),
'status_support': status_support,
}
# Add status to keys
rel['info']['status'] = release_status
if not empty_info:
fireEvent('release.create_from_search', [rel['info']], added_media, quality, single = True)
else:
release = {
'_t': 'release',
'identifier': rel.get('identifier'),
'media_id': added_media.get('_id'),
'quality': quality.get('identifier'),
'status': release_status,
'last_edit': int(time.time()),
'files': {}
}
# Add downloader info if provided
try:
release['download_info'] = rel['info']['download_info']
del rel['download_info']
except:
pass
# Add files
release_files = releases_files.get(rel.get('id'), [])
if not isinstance(release_files, list):
release_files = [release_files]
if len(release_files) == 0:
continue
for f in release_files:
rfile = all_files.get(f.get('file_id'))
if not rfile:
continue
file_type = type_by_id.get(rfile.get('type_id')).get('identifier')
if not release['files'].get(file_type):
release['files'][file_type] = []
release['files'][file_type].append(rfile.get('path'))
try:
rls = db.get('release_identifier', rel.get('identifier'), with_doc = True)['doc']
rls.update(release)
db.update(rls)
except:
db.insert(release)
log.info('Total migration took %s', time.time() - migrate_start)
log.info('=' * 30)
rename_old = True
except OperationalError:
log.error('Migrating from faulty database, probably a (too) old version: %s', traceback.format_exc())
except:
log.error('Migration failed: %s', traceback.format_exc())
release['files'][file_type].append(rfile.get('path'))
try:
rls = db.get('release_identifier', rel.get('identifier'), with_doc = True)['doc']
rls.update(release)
db.update(rls)
except:
db.insert(release)
log.info('Total migration took %s', time.time() - migrate_start)
log.info('=' * 30)
# rename old database # rename old database
log.info('Renaming old database to %s ', old_db + '.old') if rename_old:
os.rename(old_db, old_db + '.old') random = randomString()
log.info('Renaming old database to %s ', '%s.%s_old' % (old_db, random))
if os.path.isfile(old_db + '-wal'): os.rename(old_db, '%s.%s_old' % (old_db, random))
os.rename(old_db + '-wal', old_db + '-wal.old')
if os.path.isfile(old_db + '-shm'): if os.path.isfile(old_db + '-wal'):
os.rename(old_db + '-shm', old_db + '-shm.old') os.rename(old_db + '-wal', '%s-wal.%s_old' % (old_db, random))
if os.path.isfile(old_db + '-shm'):
os.rename(old_db + '-shm', '%s-shm.%s_old' % (old_db, random))

103
couchpotato/core/downloaders/nzbvortex.py

@ -1,16 +1,10 @@
from base64 import b64encode from base64 import b64encode
from urllib2 import URLError import os
from uuid import uuid4 from uuid import uuid4
import hashlib import hashlib
import httplib
import json
import os
import socket
import ssl
import sys
import time
import traceback import traceback
import urllib2
from requests import HTTPError
from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList
from couchpotato.core.helpers.encoding import tryUrlencode, sp from couchpotato.core.helpers.encoding import tryUrlencode, sp
@ -35,13 +29,17 @@ class NZBVortex(DownloaderBase):
# Send the nzb # Send the nzb
try: try:
nzb_filename = self.createFileName(data, filedata, media) nzb_filename = self.createFileName(data, filedata, media, unique_tag = True)
self.call('nzb/add', files = {'file': (nzb_filename, filedata)}) response = self.call('nzb/add', files = {'file': (nzb_filename, filedata, 'application/octet-stream')}, parameters = {
'name': nzb_filename,
'groupname': self.conf('group')
})
time.sleep(10) if response and response.get('result', '').lower() == 'ok':
raw_statuses = self.call('nzb') return self.downloadReturnId(nzb_filename)
nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if os.path.basename(nzb['nzbFileName']) == nzb_filename][0]
return self.downloadReturnId(nzb_id) log.error('Something went wrong sending the NZB file. Response: %s', response)
return False
except: except:
log.error('Something went wrong sending the NZB file: %s', traceback.format_exc()) log.error('Something went wrong sending the NZB file: %s', traceback.format_exc())
return False return False
@ -60,7 +58,8 @@ class NZBVortex(DownloaderBase):
release_downloads = ReleaseDownloadList(self) release_downloads = ReleaseDownloadList(self)
for nzb in raw_statuses.get('nzbs', []): for nzb in raw_statuses.get('nzbs', []):
if nzb['id'] in ids: nzb_id = os.path.basename(nzb['nzbFileName'])
if nzb_id in ids:
# Check status # Check status
status = 'busy' status = 'busy'
@ -70,7 +69,8 @@ class NZBVortex(DownloaderBase):
status = 'failed' status = 'failed'
release_downloads.append({ release_downloads.append({
'id': nzb['id'], 'temp_id': nzb['id'],
'id': nzb_id,
'name': nzb['uiTitle'], 'name': nzb['uiTitle'],
'status': status, 'status': status,
'original_status': nzb['state'], 'original_status': nzb['state'],
@ -85,7 +85,7 @@ class NZBVortex(DownloaderBase):
log.info('%s failed downloading, deleting...', release_download['name']) log.info('%s failed downloading, deleting...', release_download['name'])
try: try:
self.call('nzb/%s/cancel' % release_download['id']) self.call('nzb/%s/cancel' % release_download['temp_id'])
except: except:
log.error('Failed deleting: %s', traceback.format_exc(0)) log.error('Failed deleting: %s', traceback.format_exc(0))
return False return False
@ -114,7 +114,7 @@ class NZBVortex(DownloaderBase):
log.error('Login failed, please check you api-key') log.error('Login failed, please check you api-key')
return False return False
def call(self, call, parameters = None, repeat = False, auth = True, *args, **kwargs): def call(self, call, parameters = None, is_repeat = False, auth = True, *args, **kwargs):
# Login first # Login first
if not parameters: parameters = {} if not parameters: parameters = {}
@ -127,19 +127,20 @@ class NZBVortex(DownloaderBase):
params = tryUrlencode(parameters) params = tryUrlencode(parameters)
url = cleanHost(self.conf('host'), ssl = self.conf('ssl')) + 'api/' + call url = cleanHost(self.conf('host')) + 'api/' + call
try: try:
data = self.urlopen('%s?%s' % (url, params), *args, **kwargs) data = self.getJsonData('%s%s' % (url, '?' + params if params else ''), *args, cache_timeout = 0, show_error = False, **kwargs)
if data: if data:
return json.loads(data) return data
except URLError as e: except HTTPError as e:
if hasattr(e, 'code') and e.code == 403: sc = e.response.status_code
if sc == 403:
# Try login and do again # Try login and do again
if not repeat: if not is_repeat:
self.login() self.login()
return self.call(call, parameters = parameters, repeat = True, **kwargs) return self.call(call, parameters = parameters, is_repeat = True, **kwargs)
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
except: except:
@ -151,13 +152,12 @@ class NZBVortex(DownloaderBase):
if not self.api_level: if not self.api_level:
url = cleanHost(self.conf('host')) + 'api/app/apilevel'
try: try:
data = self.urlopen(url, show_error = False) data = self.call('app/apilevel', auth = False)
self.api_level = float(json.loads(data).get('apilevel')) self.api_level = float(data.get('apilevel'))
except URLError as e: except HTTPError as e:
if hasattr(e, 'code') and e.code == 403: sc = e.response.status_code
if sc == 403:
log.error('This version of NZBVortex isn\'t supported. Please update to 2.8.6 or higher') log.error('This version of NZBVortex isn\'t supported. Please update to 2.8.6 or higher')
else: else:
log.error('NZBVortex doesn\'t seem to be running or maybe the remote option isn\'t enabled yet: %s', traceback.format_exc(1)) log.error('NZBVortex doesn\'t seem to be running or maybe the remote option isn\'t enabled yet: %s', traceback.format_exc(1))
@ -169,29 +169,6 @@ class NZBVortex(DownloaderBase):
return super(NZBVortex, self).isEnabled(manual, data) and self.getApiLevel() return super(NZBVortex, self).isEnabled(manual, data) and self.getApiLevel()
class HTTPSConnection(httplib.HTTPSConnection):
def __init__(self, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
def connect(self):
sock = socket.create_connection((self.host, self.port), self.timeout)
if sys.version_info < (2, 6, 7):
if hasattr(self, '_tunnel_host'):
self.sock = sock
self._tunnel()
else:
if self._tunnel_host:
self.sock = sock
self._tunnel()
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version = ssl.PROTOCOL_TLSv1)
class HTTPSHandler(urllib2.HTTPSHandler):
def https_open(self, req):
return self.do_open(HTTPSConnection, req)
config = [{ config = [{
'name': 'nzbvortex', 'name': 'nzbvortex',
'groups': [ 'groups': [
@ -211,21 +188,19 @@ config = [{
}, },
{ {
'name': 'host', 'name': 'host',
'default': 'localhost:4321', 'default': 'https://localhost:4321',
'description': 'Hostname with port. Usually <strong>localhost:4321</strong>', 'description': 'Hostname with port. Usually <strong>https://localhost:4321</strong>',
},
{
'name': 'ssl',
'default': 1,
'type': 'bool',
'advanced': True,
'description': 'Use HyperText Transfer Protocol Secure, or <strong>https</strong>',
}, },
{ {
'name': 'api_key', 'name': 'api_key',
'label': 'Api Key', 'label': 'Api Key',
}, },
{ {
'name': 'group',
'label': 'Group',
'description': 'The group CP places the nzb in. Make sure to create it in NZBVortex.',
},
{
'name': 'manual', 'name': 'manual',
'default': False, 'default': False,
'type': 'bool', 'type': 'bool',

24
couchpotato/core/downloaders/transmission.py

@ -23,16 +23,14 @@ class Transmission(DownloaderBase):
log = CPLog(__name__) log = CPLog(__name__)
trpc = None trpc = None
def connect(self, reconnect = False): def connect(self):
# Load host from config and split out port. # Load host from config and split out port.
host = cleanHost(self.conf('host'), protocol = False).split(':') host = cleanHost(self.conf('host')).rstrip('/').rsplit(':', 1)
if not isInt(host[1]): if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.') log.error('Config properties are not filled in correctly, port is missing.')
return False return False
if not self.trpc or reconnect: self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password'))
self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password'))
return self.trpc return self.trpc
def download(self, data = None, media = None, filedata = None): def download(self, data = None, media = None, filedata = None):
@ -80,15 +78,17 @@ class Transmission(DownloaderBase):
log.error('Failed sending torrent to Transmission') log.error('Failed sending torrent to Transmission')
return False return False
data = remote_torrent.get('torrent-added') or remote_torrent.get('torrent-duplicate')
# Change settings of added torrents # Change settings of added torrents
if torrent_params: if torrent_params:
self.trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) self.trpc.set_torrent(data['hashString'], torrent_params)
log.info('Torrent sent to Transmission successfully.') log.info('Torrent sent to Transmission successfully.')
return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) return self.downloadReturnId(data['hashString'])
def test(self): def test(self):
if self.connect(True) and self.trpc.get_session(): if self.connect() and self.trpc.get_session():
return True return True
return False return False
@ -164,11 +164,11 @@ class Transmission(DownloaderBase):
class TransmissionRPC(object): class TransmissionRPC(object):
"""TransmissionRPC lite library""" """TransmissionRPC lite library"""
def __init__(self, host = 'localhost', port = 9091, rpc_url = 'transmission', username = None, password = None): def __init__(self, host = 'http://localhost', port = 9091, rpc_url = 'transmission', username = None, password = None):
super(TransmissionRPC, self).__init__() super(TransmissionRPC, self).__init__()
self.url = 'http://' + host + ':' + str(port) + '/' + rpc_url + '/rpc' self.url = host + ':' + str(port) + '/' + rpc_url + '/rpc'
self.tag = 0 self.tag = 0
self.session_id = 0 self.session_id = 0
self.session = {} self.session = {}
@ -276,8 +276,8 @@ config = [{
}, },
{ {
'name': 'host', 'name': 'host',
'default': 'localhost:9091', 'default': 'http://localhost:9091',
'description': 'Hostname with port. Usually <strong>localhost:9091</strong>', 'description': 'Hostname with port. Usually <strong>http://localhost:9091</strong>',
}, },
{ {
'name': 'rpc_url', 'name': 'rpc_url',

2
couchpotato/core/event.py

@ -90,7 +90,7 @@ def fireEvent(name, *args, **kwargs):
else: else:
e = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock()) e = Event(name = name, threads = 10, exc_info = True, traceback = True)
for event in events[name]: for event in events[name]:
e.handle(event['handler'], priority = event['priority']) e.handle(event['handler'], priority = event['priority'])

9
couchpotato/core/helpers/encoding.py

@ -5,6 +5,7 @@ import re
import traceback import traceback
import unicodedata import unicodedata
from chardet import detect
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
import six import six
@ -35,6 +36,9 @@ def toUnicode(original, *args):
return six.text_type(original, *args) return six.text_type(original, *args)
except: except:
try: try:
detected = detect(original)
if detected.get('encoding') == 'utf-8':
return original.decode('utf-8')
return ek(original, *args) return ek(original, *args)
except: except:
raise raise
@ -52,7 +56,10 @@ def ss(original, *args):
return u_original.encode(Env.get('encoding')) return u_original.encode(Env.get('encoding'))
except Exception as e: except Exception as e:
log.debug('Failed ss encoding char, force UTF8: %s', e) log.debug('Failed ss encoding char, force UTF8: %s', e)
return u_original.encode('UTF-8') try:
return u_original.encode(Env.get('encoding'), 'replace')
except:
return u_original.encode('utf-8', 'replace')
def sp(path, *args): def sp(path, *args):

34
couchpotato/core/helpers/variable.py

@ -41,11 +41,11 @@ def symlink(src, dst):
def getUserDir(): def getUserDir():
try: try:
import pwd import pwd
os.environ['HOME'] = pwd.getpwuid(os.geteuid()).pw_dir os.environ['HOME'] = sp(pwd.getpwuid(os.geteuid()).pw_dir)
except: except:
pass pass
return os.path.expanduser('~') return sp(os.path.expanduser('~'))
def getDownloadDir(): def getDownloadDir():
@ -380,3 +380,33 @@ def getFreeSpace(directories):
free_space[folder] = size free_space[folder] = size
return free_space return free_space
def getSize(paths):
single = not isinstance(paths, (tuple, list))
if single:
paths = [paths]
total_size = 0
for path in paths:
path = sp(path)
if os.path.isdir(path):
total_size = 0
for dirpath, _, filenames in os.walk(path):
for f in filenames:
total_size += os.path.getsize(sp(os.path.join(dirpath, f)))
elif os.path.isfile(path):
total_size += os.path.getsize(path)
return total_size / 1048576 # MB
def find(func, iterable):
for item in iterable:
if func(item):
return item
return None

17
couchpotato/core/logger.py

@ -59,15 +59,14 @@ class CPLog(object):
msg = ss(msg) msg = ss(msg)
try: try:
msg = msg % replace_tuple if isinstance(replace_tuple, tuple):
except: msg = msg % tuple([ss(x) if not isinstance(x, (int, float)) else x for x in list(replace_tuple)])
try: elif isinstance(replace_tuple, dict):
if isinstance(replace_tuple, tuple): msg = msg % dict((k, ss(v)) for k, v in replace_tuple.iteritems())
msg = msg % tuple([ss(x) for x in list(replace_tuple)]) else:
else: msg = msg % ss(replace_tuple)
msg = msg % ss(replace_tuple) except Exception as e:
except Exception as e: self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, e))
self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, e))
self.setup() self.setup()
if not self.is_develop: if not self.is_develop:

23
couchpotato/core/media/__init__.py

@ -26,9 +26,9 @@ class MediaBase(Plugin):
def onComplete(): def onComplete():
try: try:
media = fireEvent('media.get', media_id, single = True) media = fireEvent('media.get', media_id, single = True)
event_name = '%s.searcher.single' % media.get('type') if media:
event_name = '%s.searcher.single' % media.get('type')
fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True) fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True)
except: except:
log.error('Failed creating onComplete: %s', traceback.format_exc()) log.error('Failed creating onComplete: %s', traceback.format_exc())
@ -39,9 +39,9 @@ class MediaBase(Plugin):
def notifyFront(): def notifyFront():
try: try:
media = fireEvent('media.get', media_id, single = True) media = fireEvent('media.get', media_id, single = True)
event_name = '%s.update' % media.get('type') if media:
event_name = '%s.update' % media.get('type')
fireEvent('notify.frontend', type = event_name, data = media) fireEvent('notify.frontend', type = event_name, data = media)
except: except:
log.error('Failed creating onComplete: %s', traceback.format_exc()) log.error('Failed creating onComplete: %s', traceback.format_exc())
@ -65,10 +65,13 @@ class MediaBase(Plugin):
return def_title or 'UNKNOWN' return def_title or 'UNKNOWN'
def getPoster(self, image_urls, existing_files): def getPoster(self, media, image_urls):
image_type = 'poster' if 'files' not in media:
media['files'] = {}
# Remove non-existing files existing_files = media['files']
image_type = 'poster'
file_type = 'image_%s' % image_type file_type = 'image_%s' % image_type
# Make existing unique # Make existing unique
@ -92,7 +95,7 @@ class MediaBase(Plugin):
if file_type not in existing_files or len(existing_files.get(file_type, [])) == 0: if file_type not in existing_files or len(existing_files.get(file_type, [])) == 0:
file_path = fireEvent('file.download', url = image, single = True) file_path = fireEvent('file.download', url = image, single = True)
if file_path: if file_path:
existing_files[file_type] = [file_path] existing_files[file_type] = [toUnicode(file_path)]
break break
else: else:
break break

110
couchpotato/core/media/_base/library/main.py

@ -1,10 +1,47 @@
from couchpotato import get_db
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.logger import CPLog
from couchpotato.core.media._base.library.base import LibraryBase from couchpotato.core.media._base.library.base import LibraryBase
log = CPLog(__name__)
class Library(LibraryBase): class Library(LibraryBase):
def __init__(self): def __init__(self):
addEvent('library.title', self.title) addEvent('library.title', self.title)
addEvent('library.related', self.related)
addEvent('library.tree', self.tree)
addEvent('library.root', self.root)
addApiView('library.query', self.queryView)
addApiView('library.related', self.relatedView)
addApiView('library.tree', self.treeView)
def queryView(self, media_id, **kwargs):
db = get_db()
media = db.get('id', media_id)
return {
'result': fireEvent('library.query', media, single = True)
}
def relatedView(self, media_id, **kwargs):
db = get_db()
media = db.get('id', media_id)
return {
'result': fireEvent('library.related', media, single = True)
}
def treeView(self, media_id, **kwargs):
db = get_db()
media = db.get('id', media_id)
return {
'result': fireEvent('library.tree', media, single = True)
}
def title(self, library): def title(self, library):
return fireEvent( return fireEvent(
@ -16,3 +53,76 @@ class Library(LibraryBase):
include_identifier = False, include_identifier = False,
single = True single = True
) )
def related(self, media):
result = {self.key(media['type']): media}
db = get_db()
cur = media
while cur and cur.get('parent_id'):
cur = db.get('id', cur['parent_id'])
result[self.key(cur['type'])] = cur
children = db.get_many('media_children', media['_id'], with_doc = True)
for item in children:
key = self.key(item['doc']['type']) + 's'
if key not in result:
result[key] = []
result[key].append(item['doc'])
return result
def root(self, media):
db = get_db()
cur = media
while cur and cur.get('parent_id'):
cur = db.get('id', cur['parent_id'])
return cur
def tree(self, media = None, media_id = None):
db = get_db()
if media:
result = media
elif media_id:
result = db.get('id', media_id, with_doc = True)
else:
return None
# Find children
items = db.get_many('media_children', result['_id'], with_doc = True)
keys = []
# Build children arrays
for item in items:
key = self.key(item['doc']['type']) + 's'
if key not in result:
result[key] = {}
elif type(result[key]) is not dict:
result[key] = {}
if key not in keys:
keys.append(key)
result[key][item['_id']] = fireEvent('library.tree', item['doc'], single = True)
# Unique children
for key in keys:
result[key] = result[key].values()
# Include releases
result['releases'] = fireEvent('release.for_media', result['_id'], single = True)
return result
def key(self, media_type):
parts = media_type.split('.')
return parts[-1]

6
couchpotato/core/media/_base/matcher/main.py

@ -40,7 +40,7 @@ class Matcher(MatcherBase):
return False return False
def correctTitle(self, chain, media): def correctTitle(self, chain, media):
root_library = media['library']['root_library'] root = fireEvent('library.root', media, single = True)
if 'show_name' not in chain.info or not len(chain.info['show_name']): if 'show_name' not in chain.info or not len(chain.info['show_name']):
log.info('Wrong: missing show name in parsed result') log.info('Wrong: missing show name in parsed result')
@ -50,10 +50,10 @@ class Matcher(MatcherBase):
chain_words = [x.lower() for x in chain.info['show_name']] chain_words = [x.lower() for x in chain.info['show_name']]
# Build a list of possible titles of the media we are searching for # Build a list of possible titles of the media we are searching for
titles = root_library['info']['titles'] titles = root['info']['titles']
# Add year suffix titles (will result in ['<name_one>', '<name_one> <suffix_one>', '<name_two>', ...]) # Add year suffix titles (will result in ['<name_one>', '<name_one> <suffix_one>', '<name_two>', ...])
suffixes = [None, root_library['info']['year']] suffixes = [None, root['info']['year']]
titles = [ titles = [
title + ((' %s' % suffix) if suffix else '') title + ((' %s' % suffix) if suffix else '')

117
couchpotato/core/media/_base/media/main.py

@ -1,10 +1,9 @@
from datetime import timedelta from datetime import timedelta
from operator import itemgetter
import time import time
import traceback import traceback
from string import ascii_lowercase from string import ascii_lowercase
from CodernityDB.database import RecordNotFound from CodernityDB.database import RecordNotFound, RecordDeleted
from couchpotato import tryInt, get_db from couchpotato import tryInt, get_db
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.event import fireEvent, fireEventAsync, addEvent from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
@ -44,15 +43,15 @@ class MediaPlugin(MediaBase):
'desc': 'List media', 'desc': 'List media',
'params': { 'params': {
'type': {'type': 'string', 'desc': 'Media type to filter on.'}, 'type': {'type': 'string', 'desc': 'Media type to filter on.'},
'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'}, 'status': {'type': 'array or csv', 'desc': 'Filter media by status. Example:"active,done"'},
'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'}, 'release_status': {'type': 'array or csv', 'desc': 'Filter media by status of its releases. Example:"snatched,available"'},
'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'}, 'limit_offset': {'desc': 'Limit and offset the media list. Examples: "50" or "50,30"'},
'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'}, 'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all media starting with the letter "a"'},
'search': {'desc': 'Search movie title'}, 'search': {'desc': 'Search media title'},
}, },
'return': {'type': 'object', 'example': """{ 'return': {'type': 'object', 'example': """{
'success': True, 'success': True,
'empty': bool, any movies returned or not, 'empty': bool, any media returned or not,
'media': array, media found, 'media': array, media found,
}"""} }"""}
}) })
@ -78,6 +77,7 @@ class MediaPlugin(MediaBase):
addEvent('app.load', self.addSingleListView, priority = 100) addEvent('app.load', self.addSingleListView, priority = 100)
addEvent('app.load', self.addSingleCharView, priority = 100) addEvent('app.load', self.addSingleCharView, priority = 100)
addEvent('app.load', self.addSingleDeleteView, priority = 100) addEvent('app.load', self.addSingleDeleteView, priority = 100)
addEvent('app.load', self.cleanupFaults)
addEvent('media.get', self.get) addEvent('media.get', self.get)
addEvent('media.with_status', self.withStatus) addEvent('media.with_status', self.withStatus)
@ -88,6 +88,18 @@ class MediaPlugin(MediaBase):
addEvent('media.tag', self.tag) addEvent('media.tag', self.tag)
addEvent('media.untag', self.unTag) addEvent('media.untag', self.unTag)
# Wrongly tagged media files
def cleanupFaults(self):
medias = fireEvent('media.with_status', 'ignored', single = True) or []
db = get_db()
for media in medias:
try:
media['status'] = 'done'
db.update(media)
except:
pass
def refresh(self, id = '', **kwargs): def refresh(self, id = '', **kwargs):
handlers = [] handlers = []
ids = splitString(id) ids = splitString(id)
@ -109,7 +121,7 @@ class MediaPlugin(MediaBase):
try: try:
media = get_db().get('id', media_id) media = get_db().get('id', media_id)
event = '%s.update_info' % media.get('type') event = '%s.update' % media.get('type')
def handler(): def handler():
fireEvent(event, media_id = media_id, on_complete = self.createOnComplete(media_id)) fireEvent(event, media_id = media_id, on_complete = self.createOnComplete(media_id))
@ -146,7 +158,7 @@ class MediaPlugin(MediaBase):
return media return media
except RecordNotFound: except (RecordNotFound, RecordDeleted):
log.error('Media with id "%s" not found', media_id) log.error('Media with id "%s" not found', media_id)
except: except:
raise raise
@ -160,10 +172,13 @@ class MediaPlugin(MediaBase):
'media': media, 'media': media,
} }
def withStatus(self, status, with_doc = True): def withStatus(self, status, types = None, with_doc = True):
db = get_db() db = get_db()
if types and not isinstance(types, (list, tuple)):
types = [types]
status = list(status if isinstance(status, (list, tuple)) else [status]) status = list(status if isinstance(status, (list, tuple)) else [status])
for s in status: for s in status:
@ -171,24 +186,29 @@ class MediaPlugin(MediaBase):
if with_doc: if with_doc:
try: try:
doc = db.get('id', ms['_id']) doc = db.get('id', ms['_id'])
if types and doc.get('type') not in types:
continue
yield doc yield doc
except RecordNotFound: except (RecordDeleted, RecordNotFound):
log.debug('Record not found, skipping: %s', ms['_id']) log.debug('Record not found, skipping: %s', ms['_id'])
except (ValueError, EOFError):
fireEvent('database.delete_corrupted', ms.get('_id'), traceback_error = traceback.format_exc(0))
else: else:
yield ms yield ms
def withIdentifiers(self, identifiers, with_doc = False): def withIdentifiers(self, identifiers, with_doc = False):
db = get_db() db = get_db()
for x in identifiers: for x in identifiers:
try: try:
media = db.get('media', '%s-%s' % (x, identifiers[x]), with_doc = with_doc) return db.get('media', '%s-%s' % (x, identifiers[x]), with_doc = with_doc)
return media
except: except:
pass pass
log.debug('No media found with identifiers: %s', identifiers) log.debug('No media found with identifiers: %s', identifiers)
return False
def list(self, types = None, status = None, release_status = None, status_or = False, limit_offset = None, with_tags = None, starts_with = None, search = None): def list(self, types = None, status = None, release_status = None, status_or = False, limit_offset = None, with_tags = None, starts_with = None, search = None):
@ -275,6 +295,10 @@ class MediaPlugin(MediaBase):
media = fireEvent('media.get', media_id, single = True) media = fireEvent('media.get', media_id, single = True)
# Skip if no media has been found
if not media:
continue
# Merge releases with movie dict # Merge releases with movie dict
medias.append(media) medias.append(media)
@ -307,9 +331,22 @@ class MediaPlugin(MediaBase):
def addSingleListView(self): def addSingleListView(self):
for media_type in fireEvent('media.types', merge = True): for media_type in fireEvent('media.types', merge = True):
def tempList(*args, **kwargs): tempList = lambda *args, **kwargs : self.listView(type = media_type, **kwargs)
return self.listView(types = media_type, **kwargs) addApiView('%s.list' % media_type, tempList, docs = {
addApiView('%s.list' % media_type, tempList) 'desc': 'List media',
'params': {
'status': {'type': 'array or csv', 'desc': 'Filter ' + media_type + ' by status. Example:"active,done"'},
'release_status': {'type': 'array or csv', 'desc': 'Filter ' + media_type + ' by status of its releases. Example:"snatched,available"'},
'limit_offset': {'desc': 'Limit and offset the ' + media_type + ' list. Examples: "50" or "50,30"'},
'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all ' + media_type + 's starting with the letter "a"'},
'search': {'desc': 'Search ' + media_type + ' title'},
},
'return': {'type': 'object', 'example': """{
'success': True,
'empty': bool, any """ + media_type + """s returned or not,
'media': array, media found,
}"""}
})
def availableChars(self, types = None, status = None, release_status = None): def availableChars(self, types = None, status = None, release_status = None):
@ -355,7 +392,7 @@ class MediaPlugin(MediaBase):
if x['_id'] in media_ids: if x['_id'] in media_ids:
chars.add(x['key']) chars.add(x['key'])
if len(chars) == 25: if len(chars) == 27:
break break
return list(chars) return list(chars)
@ -376,8 +413,7 @@ class MediaPlugin(MediaBase):
def addSingleCharView(self): def addSingleCharView(self):
for media_type in fireEvent('media.types', merge = True): for media_type in fireEvent('media.types', merge = True):
def tempChar(*args, **kwargs): tempChar = lambda *args, **kwargs : self.charView(type = media_type, **kwargs)
return self.charView(types = media_type, **kwargs)
addApiView('%s.available_chars' % media_type, tempChar) addApiView('%s.available_chars' % media_type, tempChar)
def delete(self, media_id, delete_from = None): def delete(self, media_id, delete_from = None):
@ -415,11 +451,16 @@ class MediaPlugin(MediaBase):
db.delete(release) db.delete(release)
total_deleted += 1 total_deleted += 1
if (total_releases == total_deleted and media['status'] != 'active') or (total_releases == 0 and not new_media_status) or (not new_media_status and delete_from == 'late'): if (total_releases == total_deleted) or (total_releases == 0 and not new_media_status) or (not new_media_status and delete_from == 'late'):
db.delete(media) db.delete(media)
deleted = True deleted = True
elif new_media_status: elif new_media_status:
media['status'] = new_media_status media['status'] = new_media_status
# Remove profile (no use for in manage)
if new_media_status == 'done':
media['profile_id'] = None
db.update(media) db.update(media)
fireEvent('media.untag', media['_id'], 'recent', single = True) fireEvent('media.untag', media['_id'], 'recent', single = True)
@ -446,11 +487,16 @@ class MediaPlugin(MediaBase):
def addSingleDeleteView(self): def addSingleDeleteView(self):
for media_type in fireEvent('media.types', merge = True): for media_type in fireEvent('media.types', merge = True):
def tempDelete(*args, **kwargs): tempDelete = lambda *args, **kwargs : self.deleteView(type = media_type, **kwargs)
return self.deleteView(types = media_type, *args, **kwargs) addApiView('%s.delete' % media_type, tempDelete, docs = {
addApiView('%s.delete' % media_type, tempDelete) 'desc': 'Delete a ' + media_type + ' from the wanted list',
'params': {
'id': {'desc': 'Media ID(s) you want to delete.', 'type': 'int (comma separated)'},
'delete_from': {'desc': 'Delete ' + media_type + ' from this page', 'type': 'string: all (default), wanted, manage'},
}
})
def restatus(self, media_id): def restatus(self, media_id, tag_recent = True, allowed_restatus = None):
try: try:
db = get_db() db = get_db()
@ -470,12 +516,13 @@ class MediaPlugin(MediaBase):
done_releases = [release for release in media_releases if release.get('status') == 'done'] done_releases = [release for release in media_releases if release.get('status') == 'done']
if done_releases: if done_releases:
# Only look at latest added release
release = sorted(done_releases, key = itemgetter('last_edit'), reverse = True)[0]
# Check if we are finished with the media # Check if we are finished with the media
if fireEvent('quality.isfinish', {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, timedelta(seconds = time.time() - release['last_edit']).days, single = True): for release in done_releases:
m['status'] = 'done' if fireEvent('quality.isfinish', {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, timedelta(seconds = time.time() - release['last_edit']).days, single = True):
m['status'] = 'done'
break
elif previous_status == 'done': elif previous_status == 'done':
m['status'] = 'done' m['status'] = 'done'
@ -484,22 +531,26 @@ class MediaPlugin(MediaBase):
m['status'] = previous_status m['status'] = previous_status
# Only update when status has changed # Only update when status has changed
if previous_status != m['status']: if previous_status != m['status'] and (not allowed_restatus or m['status'] in allowed_restatus):
db.update(m) db.update(m)
# Tag media as recent # Tag media as recent
self.tag(media_id, 'recent') if tag_recent:
self.tag(media_id, 'recent', update_edited = True)
return m['status'] return m['status']
except: except:
log.error('Failed restatus: %s', traceback.format_exc()) log.error('Failed restatus: %s', traceback.format_exc())
def tag(self, media_id, tag): def tag(self, media_id, tag, update_edited = False):
try: try:
db = get_db() db = get_db()
m = db.get('id', media_id) m = db.get('id', media_id)
if update_edited:
m['last_edit'] = int(time.time())
tags = m.get('tags') or [] tags = m.get('tags') or []
if tag not in tags: if tag not in tags:
tags.append(tag) tags.append(tag)

23
couchpotato/core/media/_base/providers/nzb/newznab.py

@ -45,7 +45,7 @@ class Base(NZBProvider, RSS):
def _searchOnHost(self, host, media, quality, results): def _searchOnHost(self, host, media, quality, results):
query = self.buildUrl(media, host) query = self.buildUrl(media, host)
url = '%s&%s' % (self.getUrl(host['host']), query) url = '%s%s' % (self.getUrl(host['host']), query)
nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})
for nzb in nzbs: for nzb in nzbs:
@ -83,7 +83,7 @@ class Base(NZBProvider, RSS):
try: try:
# Get details for extended description to retrieve passwords # Get details for extended description to retrieve passwords
query = self.buildDetailsUrl(nzb_id, host['api_key']) query = self.buildDetailsUrl(nzb_id, host['api_key'])
url = '%s&%s' % (self.getUrl(host['host']), query) url = '%s%s' % (self.getUrl(host['host']), query)
nzb_details = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})[0] nzb_details = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})[0]
description = self.getTextElement(nzb_details, 'description') description = self.getTextElement(nzb_details, 'description')
@ -187,11 +187,12 @@ class Base(NZBProvider, RSS):
self.limits_reached[host] = False self.limits_reached[host] = False
return data return data
except HTTPError as e: except HTTPError as e:
if e.code == 503: sc = e.response.status_code
if sc in [503, 429]:
response = e.read().lower() response = e.read().lower()
if 'maximum api' in response or 'download limit' in response: if sc == 429 or 'maximum api' in response or 'download limit' in response:
if not self.limits_reached.get(host): if not self.limits_reached.get(host):
log.error('Limit reached for newznab provider: %s', host) log.error('Limit reached / to many requests for newznab provider: %s', host)
self.limits_reached[host] = time.time() self.limits_reached[host] = time.time()
return 'try_next' return 'try_next'
@ -220,7 +221,7 @@ config = [{
'description': 'Enable <a href="http://newznab.com/" target="_blank">NewzNab</a> such as <a href="https://nzb.su" target="_blank">NZB.su</a>, \ 'description': 'Enable <a href="http://newznab.com/" target="_blank">NewzNab</a> such as <a href="https://nzb.su" target="_blank">NZB.su</a>, \
<a href="https://nzbs.org" target="_blank">NZBs.org</a>, <a href="http://dognzb.cr/" target="_blank">DOGnzb.cr</a>, \ <a href="https://nzbs.org" target="_blank">NZBs.org</a>, <a href="http://dognzb.cr/" target="_blank">DOGnzb.cr</a>, \
<a href="https://github.com/spotweb/spotweb" target="_blank">Spotweb</a>, <a href="https://nzbgeek.info/" target="_blank">NZBGeek</a>, \ <a href="https://github.com/spotweb/spotweb" target="_blank">Spotweb</a>, <a href="https://nzbgeek.info/" target="_blank">NZBGeek</a>, \
<a href="https://smackdownonyou.com" target="_blank">SmackDown</a>, <a href="https://www.nzbfinder.ws" target="_blank">NZBFinder</a>', <a href="https://www.nzbfinder.ws" target="_blank">NZBFinder</a>',
'wizard': True, 'wizard': True,
'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAgMAAABinRfyAAAACVBMVEVjhwD///86aRovd/sBAAAAMklEQVQI12NgAIPQUCCRmQkjssDEShiRuRIqwZqZGcDAGBrqANUhGgIkWAOABKMDxCAA24UK50b26SAAAAAASUVORK5CYII=', 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAgMAAABinRfyAAAACVBMVEVjhwD///86aRovd/sBAAAAMklEQVQI12NgAIPQUCCRmQkjssDEShiRuRIqwZqZGcDAGBrqANUhGgIkWAOABKMDxCAA24UK50b26SAAAAAASUVORK5CYII=',
'options': [ 'options': [
@ -231,30 +232,30 @@ config = [{
}, },
{ {
'name': 'use', 'name': 'use',
'default': '0,0,0,0,0,0' 'default': '0,0,0,0,0'
}, },
{ {
'name': 'host', 'name': 'host',
'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws', 'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://api.nzbgeek.info,https://www.nzbfinder.ws',
'description': 'The hostname of your newznab provider', 'description': 'The hostname of your newznab provider',
}, },
{ {
'name': 'extra_score', 'name': 'extra_score',
'advanced': True, 'advanced': True,
'label': 'Extra Score', 'label': 'Extra Score',
'default': '0,0,0,0,0,0', 'default': '0,0,0,0,0',
'description': 'Starting score for each release found via this provider.', 'description': 'Starting score for each release found via this provider.',
}, },
{ {
'name': 'custom_tag', 'name': 'custom_tag',
'advanced': True, 'advanced': True,
'label': 'Custom tag', 'label': 'Custom tag',
'default': ',,,,,', 'default': ',,,,',
'description': 'Add custom tags, for example add rls=1 to get only scene releases from nzbs.org', 'description': 'Add custom tags, for example add rls=1 to get only scene releases from nzbs.org',
}, },
{ {
'name': 'api_key', 'name': 'api_key',
'default': ',,,,,', 'default': ',,,,',
'label': 'Api Key', 'label': 'Api Key',
'description': 'Can be found on your profile page', 'description': 'Can be found on your profile page',
'type': 'combined', 'type': 'combined',

126
couchpotato/core/media/_base/providers/nzb/nzbindex.py

@ -1,126 +0,0 @@
import re
import time
from bs4 import BeautifulSoup
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.event import fireEvent
from couchpotato.core.media._base.providers.nzb.base import NZBProvider
from dateutil.parser import parse
log = CPLog(__name__)
class Base(NZBProvider, RSS):
urls = {
'download': 'https://www.nzbindex.com/download/',
'search': 'https://www.nzbindex.com/rss/?%s',
}
http_time_between_calls = 1 # Seconds
def _search(self, media, quality, results):
nzbs = self.getRSSData(self.urls['search'] % self.buildUrl(media, quality))
for nzb in nzbs:
enclosure = self.getElement(nzb, 'enclosure').attrib
nzbindex_id = int(self.getTextElement(nzb, "link").split('/')[4])
title = self.getTextElement(nzb, "title")
match = fireEvent('matcher.parse', title, parser='usenet', single = True)
if not match.chains:
log.info('Unable to parse release with title "%s"', title)
continue
# TODO should we consider other lower-weight chains here?
info = fireEvent('matcher.flatten_info', match.chains[0].info, single = True)
release_name = fireEvent('matcher.construct_from_raw', info.get('release_name'), single = True)
file_name = info.get('detail', {}).get('file_name')
file_name = file_name[0] if file_name else None
title = release_name or file_name
# Strip extension from parsed title (if one exists)
ext_pos = title.rfind('.')
# Assume extension if smaller than 4 characters
# TODO this should probably be done a better way
if len(title[ext_pos + 1:]) <= 4:
title = title[:ext_pos]
if not title:
log.info('Unable to find release name from match')
continue
try:
description = self.getTextElement(nzb, "description")
except:
description = ''
def extra_check(item):
if '#c20000' in item['description'].lower():
log.info('Wrong: Seems to be passworded: %s', item['name'])
return False
return True
results.append({
'id': nzbindex_id,
'name': title,
'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, "pubDate")).timetuple()))),
'size': tryInt(enclosure['length']) / 1024 / 1024,
'url': enclosure['url'],
'detail_url': enclosure['url'].replace('/download/', '/release/'),
'description': description,
'get_more_info': self.getMoreInfo,
'extra_check': extra_check,
})
def getMoreInfo(self, item):
try:
if '/nfo/' in item['description'].lower():
nfo_url = re.search('href=\"(?P<nfo>.+)\" ', item['description']).group('nfo')
full_description = self.getCache('nzbindex.%s' % item['id'], url = nfo_url, cache_timeout = 25920000)
html = BeautifulSoup(full_description)
item['description'] = toUnicode(html.find('pre', attrs = {'id': 'nfo0'}).text)
except:
pass
config = [{
'name': 'nzbindex',
'groups': [
{
'tab': 'searcher',
'list': 'nzb_providers',
'name': 'nzbindex',
'description': 'Free provider, less accurate. See <a href="https://www.nzbindex.com/">NZBIndex</a>',
'wizard': True,
'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAo0lEQVR42t2SQQ2AMBAEcUCwUAv94QMLfHliAQtYqIVawEItYAG6yZFMLkUANNlk79Kbbtp2P1j9uKxVV9VWFeStl+Wh3fWK9hNwEoADZkJtMD49AqS5AUjWGx6A+m+ARICGrM5W+wSTB0gETKzdHZwCEZAJ8PGZQN4AiQAmkR9s06EBAugJiBoAAPFfAQcBgZcIHzwA6TYP4JsXeSg3P9L31w3eksbH3zMb/wAAAABJRU5ErkJggg==',
'options': [
{
'name': 'enabled',
'type': 'enabler',
'default': True,
},
{
'name': 'extra_score',
'advanced': True,
'label': 'Extra Score',
'type': 'int',
'default': 0,
'description': 'Starting score for each release found via this provider.',
}
],
},
],
}]

2
couchpotato/core/media/_base/providers/torrent/awesomehd.py

@ -61,7 +61,7 @@ class Base(TorrentProvider):
'name': re.sub('[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) %s' % (name, year, torrent_desc)), 'name': re.sub('[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) %s' % (name, year, torrent_desc)),
'url': self.urls['download'] % (torrent_id, authkey, self.conf('passkey')), 'url': self.urls['download'] % (torrent_id, authkey, self.conf('passkey')),
'detail_url': self.urls['detail'] % torrent_id, 'detail_url': self.urls['detail'] % torrent_id,
'size': self.parseSize(entry.find('size').get_text()), 'size': tryInt(entry.find('size').get_text()) / 1048576,
'seeders': tryInt(entry.find('seeders').get_text()), 'seeders': tryInt(entry.find('seeders').get_text()),
'leechers': tryInt(entry.find('leechers').get_text()), 'leechers': tryInt(entry.find('leechers').get_text()),
'score': torrentscore 'score': torrentscore

13
couchpotato/core/media/_base/providers/torrent/bitsoup.py

@ -22,6 +22,9 @@ class Base(TorrentProvider):
http_time_between_calls = 1 # Seconds http_time_between_calls = 1 # Seconds
only_tables_tags = SoupStrainer('table') only_tables_tags = SoupStrainer('table')
torrent_name_cell = 1
torrent_download_cell = 2
def _searchOnTitle(self, title, movie, quality, results): def _searchOnTitle(self, title, movie, quality, results):
url = self.urls['search'] % self.buildUrl(title, movie, quality) url = self.urls['search'] % self.buildUrl(title, movie, quality)
@ -40,8 +43,8 @@ class Base(TorrentProvider):
all_cells = result.find_all('td') all_cells = result.find_all('td')
torrent = all_cells[1].find('a') torrent = all_cells[self.torrent_name_cell].find('a')
download = all_cells[3].find('a') download = all_cells[self.torrent_download_cell].find('a')
torrent_id = torrent['href'] torrent_id = torrent['href']
torrent_id = torrent_id.replace('details.php?id=', '') torrent_id = torrent_id.replace('details.php?id=', '')
@ -49,9 +52,9 @@ class Base(TorrentProvider):
torrent_name = torrent.getText() torrent_name = torrent.getText()
torrent_size = self.parseSize(all_cells[7].getText()) torrent_size = self.parseSize(all_cells[8].getText())
torrent_seeders = tryInt(all_cells[9].getText()) torrent_seeders = tryInt(all_cells[10].getText())
torrent_leechers = tryInt(all_cells[10].getText()) torrent_leechers = tryInt(all_cells[11].getText())
torrent_url = self.urls['baseurl'] % download['href'] torrent_url = self.urls['baseurl'] % download['href']
torrent_detail_url = self.urls['baseurl'] % torrent['href'] torrent_detail_url = self.urls['baseurl'] % torrent['href']

3
couchpotato/core/media/_base/providers/torrent/kickasstorrents.py

@ -34,8 +34,7 @@ class Base(TorrentMagnetProvider):
'http://kickass.pw', 'http://kickass.pw',
'http://kickassto.come.in', 'http://kickassto.come.in',
'http://katproxy.ws', 'http://katproxy.ws',
'http://www.kickassunblock.info', 'http://kickass.bitproxy.eu',
'http://www.kickassproxy.info',
'http://katph.eu', 'http://katph.eu',
'http://kickassto.come.in', 'http://kickassto.come.in',
] ]

12
couchpotato/core/media/_base/providers/torrent/passthepopcorn.py

@ -64,6 +64,10 @@ class Base(TorrentProvider):
torrentdesc += ' HQ' torrentdesc += ' HQ'
if self.conf('prefer_golden'): if self.conf('prefer_golden'):
torrentscore += 5000 torrentscore += 5000
if 'FreeleechType' in torrent:
torrentdesc += ' Freeleech'
if self.conf('prefer_freeleech'):
torrentscore += 7000
if 'Scene' in torrent and torrent['Scene']: if 'Scene' in torrent and torrent['Scene']:
torrentdesc += ' Scene' torrentdesc += ' Scene'
if self.conf('prefer_scene'): if self.conf('prefer_scene'):
@ -224,6 +228,14 @@ config = [{
'description': 'Favors Golden Popcorn-releases over all other releases.' 'description': 'Favors Golden Popcorn-releases over all other releases.'
}, },
{ {
'name': 'prefer_freeleech',
'advanced': True,
'type': 'bool',
'label': 'Prefer Freeleech',
'default': 1,
'description': 'Favors torrents marked as freeleech over all other releases.'
},
{
'name': 'prefer_scene', 'name': 'prefer_scene',
'advanced': True, 'advanced': True,
'type': 'bool', 'type': 'bool',

12
couchpotato/core/media/_base/providers/torrent/thepiratebay.py

@ -24,16 +24,16 @@ class Base(TorrentMagnetProvider):
http_time_between_calls = 0 http_time_between_calls = 0
proxy_list = [ proxy_list = [
'https://nobay.net', 'https://dieroschtibay.org',
'https://thebay.al', 'https://thebay.al',
'https://thepiratebay.se', 'https://thepiratebay.se',
'http://thepiratebay.cd', 'http://thepiratebay.se.net',
'http://thebootlegbay.com', 'http://thebootlegbay.com',
'http://www.tpb.gr', 'http://tpb.ninja.so',
'http://tpbproxy.co.uk', 'http://proxybay.fr',
'http://pirateproxy.in', 'http://pirateproxy.in',
'http://www.getpirate.com', 'http://piratebay.skey.sk',
'http://piratebay.io', 'http://pirateproxy.be',
'http://bayproxy.li', 'http://bayproxy.li',
'http://proxybay.pw', 'http://proxybay.pw',
] ]

12
couchpotato/core/media/_base/providers/torrent/torrentleech.py

@ -13,12 +13,12 @@ log = CPLog(__name__)
class Base(TorrentProvider): class Base(TorrentProvider):
urls = { urls = {
'test': 'http://www.torrentleech.org/', 'test': 'https://www.torrentleech.org/',
'login': 'http://www.torrentleech.org/user/account/login/', 'login': 'https://www.torrentleech.org/user/account/login/',
'login_check': 'http://torrentleech.org/user/messages', 'login_check': 'https://torrentleech.org/user/messages',
'detail': 'http://www.torrentleech.org/torrent/%s', 'detail': 'https://www.torrentleech.org/torrent/%s',
'search': 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d', 'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d',
'download': 'http://www.torrentleech.org%s', 'download': 'https://www.torrentleech.org%s',
} }
http_time_between_calls = 1 # Seconds http_time_between_calls = 1 # Seconds

19
couchpotato/core/media/_base/providers/torrent/torrentshack.py

@ -13,12 +13,12 @@ log = CPLog(__name__)
class Base(TorrentProvider): class Base(TorrentProvider):
urls = { urls = {
'test': 'https://torrentshack.net/', 'test': 'http://torrentshack.eu/',
'login': 'https://torrentshack.net/login.php', 'login': 'http://torrentshack.eu/login.php',
'login_check': 'https://torrentshack.net/inbox.php', 'login_check': 'http://torrentshack.eu/inbox.php',
'detail': 'https://torrentshack.net/torrent/%s', 'detail': 'http://torrentshack.eu/torrent/%s',
'search': 'https://torrentshack.net/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', 'search': 'http://torrentshack.eu/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1',
'download': 'https://torrentshack.net/%s', 'download': 'http://torrentshack.eu/%s',
} }
http_time_between_calls = 1 # Seconds http_time_between_calls = 1 # Seconds
@ -42,6 +42,7 @@ class Base(TorrentProvider):
link = result.find('span', attrs = {'class': 'torrent_name_link'}).parent link = result.find('span', attrs = {'class': 'torrent_name_link'}).parent
url = result.find('td', attrs = {'class': 'torrent_td'}).find('a') url = result.find('td', attrs = {'class': 'torrent_td'}).find('a')
tds = result.find_all('td')
results.append({ results.append({
'id': link['href'].replace('torrents.php?torrentid=', ''), 'id': link['href'].replace('torrents.php?torrentid=', ''),
@ -49,8 +50,8 @@ class Base(TorrentProvider):
'url': self.urls['download'] % url['href'], 'url': self.urls['download'] % url['href'],
'detail_url': self.urls['download'] % link['href'], 'detail_url': self.urls['download'] % link['href'],
'size': self.parseSize(result.find_all('td')[5].string), 'size': self.parseSize(result.find_all('td')[5].string),
'seeders': tryInt(result.find_all('td')[7].string), 'seeders': tryInt(tds[len(tds)-2].string),
'leechers': tryInt(result.find_all('td')[8].string), 'leechers': tryInt(tds[len(tds)-1].string),
}) })
except: except:
@ -80,7 +81,7 @@ config = [{
'tab': 'searcher', 'tab': 'searcher',
'list': 'torrent_providers', 'list': 'torrent_providers',
'name': 'TorrentShack', 'name': 'TorrentShack',
'description': '<a href="https://www.torrentshack.net/">TorrentShack</a>', 'description': '<a href="http://torrentshack.eu/">TorrentShack</a>',
'wizard': True, 'wizard': True,
'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAABmElEQVQoFQXBzY2cVRiE0afqvd84CQiAnxWWtyxsS6ThINBYg2Dc7mZBMEjE4mzs6e9WcY5+ePNuVFJJodQAoLo+SaWCy9rcV8cmjah3CI6iYu7oRU30kE5xxELRfamklY3k1NL19sSm7vPzP/ZdNZzKVDaY2sPZJBh9fv5ITrmG2+Vp4e1sPchVqTCQZJnVXi+/L4uuAJGly1+Pw8CprLbi8Om7tbT19/XRqJUk11JP9uHj9ulxhXbvJbI9qJvr5YkGXFG2IBT8tXczt+sfzDZCp3765f3t9tHEHGEDACma77+8o4oATKk+/PfW9YmHruRFjWoVSFsVsGu1YSKq6Oc37+n98unPZSRlY7vsKDqN+92X3yR9+PdXee3iJNKMStqdcZqoTJbUSi5JOkpfRlhSI0mSpEmCFKoU7FqSNOLAk54uGwCStMUCgLrVic62g7oDoFmmdI+P3S0pDe1xvDqb6XrZqbtzShWNoh9fv/XQHaDdM9OqrZi2M7M3UrB2vlkPS1IbdEBk7UiSoD6VlZ6aKWer4aH4f/AvKoHUTjuyAAAAAElFTkSuQmCC', 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAABmElEQVQoFQXBzY2cVRiE0afqvd84CQiAnxWWtyxsS6ThINBYg2Dc7mZBMEjE4mzs6e9WcY5+ePNuVFJJodQAoLo+SaWCy9rcV8cmjah3CI6iYu7oRU30kE5xxELRfamklY3k1NL19sSm7vPzP/ZdNZzKVDaY2sPZJBh9fv5ITrmG2+Vp4e1sPchVqTCQZJnVXi+/L4uuAJGly1+Pw8CprLbi8Om7tbT19/XRqJUk11JP9uHj9ulxhXbvJbI9qJvr5YkGXFG2IBT8tXczt+sfzDZCp3765f3t9tHEHGEDACma77+8o4oATKk+/PfW9YmHruRFjWoVSFsVsGu1YSKq6Oc37+n98unPZSRlY7vsKDqN+92X3yR9+PdXee3iJNKMStqdcZqoTJbUSi5JOkpfRlhSI0mSpEmCFKoU7FqSNOLAk54uGwCStMUCgLrVic62g7oDoFmmdI+P3S0pDe1xvDqb6XrZqbtzShWNoh9fv/XQHaDdM9OqrZi2M7M3UrB2vlkPS1IbdEBk7UiSoD6VlZ6aKWer4aH4f/AvKoHUTjuyAAAAAElFTkSuQmCC',
'options': [ 'options': [

20
couchpotato/core/media/_base/searcher/__init__.py

@ -73,4 +73,24 @@ config = [{
], ],
}, },
], ],
}, {
'name': 'torrent',
'groups': [
{
'tab': 'searcher',
'name': 'searcher',
'wizard': True,
'options': [
{
'name': 'minimum_seeders',
'advanced': True,
'label': 'Minimum seeders',
'description': 'Ignore torrents with seeders below this number',
'default': 1,
'type': 'int',
'unit': 'seeders'
},
],
},
],
}] }]

51
couchpotato/core/media/_base/searcher/main.py

@ -129,7 +129,11 @@ class Searcher(SearcherBase):
# Try guessing via quality tags # Try guessing via quality tags
guess = fireEvent('quality.guess', [nzb.get('name')], single = True) guess = fireEvent('quality.guess', [nzb.get('name')], single = True)
return threed == guess.get('is_3d') if guess:
return threed == guess.get('is_3d')
# If no quality guess, assume not 3d
else:
return threed == False
def correctYear(self, haystack, year, year_range): def correctYear(self, haystack, year, year_range):
@ -174,6 +178,25 @@ class Searcher(SearcherBase):
return False return False
def containsWords(self, rel_name, rel_words, conf, media):
# Make sure it has required words
words = splitString(self.conf('%s_words' % conf, section = 'searcher').lower())
try: words = removeDuplicate(words + splitString(media['category'][conf].lower()))
except: pass
req_match = 0
for req_set in words:
if len(req_set) >= 2 and (req_set[:1] + req_set[-1:]) == '//':
if re.search(req_set[1:-1], rel_name):
log.debug('Regex match: %s', req_set[1:-1])
req_match += 1
else:
req = splitString(req_set, '&')
req_match += len(list(set(rel_words) & set(req))) == len(req)
return words, req_match > 0
def correctWords(self, rel_name, media): def correctWords(self, rel_name, media):
media_title = fireEvent('searcher.get_search_title', media, single = True) media_title = fireEvent('searcher.get_search_title', media, single = True)
media_words = re.split('\W+', simplifyString(media_title)) media_words = re.split('\W+', simplifyString(media_title))
@ -181,31 +204,13 @@ class Searcher(SearcherBase):
rel_name = simplifyString(rel_name) rel_name = simplifyString(rel_name)
rel_words = re.split('\W+', rel_name) rel_words = re.split('\W+', rel_name)
# Make sure it has required words required_words, contains_required = self.containsWords(rel_name, rel_words, 'required', media)
required_words = splitString(self.conf('required_words', section = 'searcher').lower()) if len(required_words) > 0 and not contains_required:
try: required_words = removeDuplicate(required_words + splitString(media['category']['required'].lower()))
except: pass
req_match = 0
for req_set in required_words:
req = splitString(req_set, '&')
req_match += len(list(set(rel_words) & set(req))) == len(req)
if len(required_words) > 0 and req_match == 0:
log.info2('Wrong: Required word missing: %s', rel_name) log.info2('Wrong: Required word missing: %s', rel_name)
return False return False
# Ignore releases ignored_words, contains_ignored = self.containsWords(rel_name, rel_words, 'ignored', media)
ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower()) if len(ignored_words) > 0 and contains_ignored:
try: ignored_words = removeDuplicate(ignored_words + splitString(media['category']['ignored'].lower()))
except: pass
ignored_match = 0
for ignored_set in ignored_words:
ignored = splitString(ignored_set, '&')
ignored_match += len(list(set(rel_words) & set(ignored))) == len(ignored)
if len(ignored_words) > 0 and ignored_match:
log.info2("Wrong: '%s' contains 'ignored words'", rel_name) log.info2("Wrong: '%s' contains 'ignored words'", rel_name)
return False return False

59
couchpotato/core/media/movie/_base/main.py

@ -1,4 +1,3 @@
import os
import traceback import traceback
import time import time
@ -28,6 +27,10 @@ class MovieBase(MovieTypeBase):
addApiView('movie.add', self.addView, docs = { addApiView('movie.add', self.addView, docs = {
'desc': 'Add new movie to the wanted list', 'desc': 'Add new movie to the wanted list',
'return': {'type': 'object', 'example': """{
'success': True,
'movie': object
}"""},
'params': { 'params': {
'identifier': {'desc': 'IMDB id of the movie your want to add.'}, 'identifier': {'desc': 'IMDB id of the movie your want to add.'},
'profile_id': {'desc': 'ID of quality profile you want the add the movie in. If empty will use the default profile.'}, 'profile_id': {'desc': 'ID of quality profile you want the add the movie in. If empty will use the default profile.'},
@ -46,7 +49,7 @@ class MovieBase(MovieTypeBase):
}) })
addEvent('movie.add', self.add) addEvent('movie.add', self.add)
addEvent('movie.update_info', self.updateInfo) addEvent('movie.update', self.update)
addEvent('movie.update_release_dates', self.updateReleaseDate) addEvent('movie.update_release_dates', self.updateReleaseDate)
def add(self, params = None, force_readd = True, search_after = True, update_after = True, notify_after = True, status = None): def add(self, params = None, force_readd = True, search_after = True, update_after = True, notify_after = True, status = None):
@ -151,8 +154,7 @@ class MovieBase(MovieTypeBase):
for release in fireEvent('release.for_media', m['_id'], single = True): for release in fireEvent('release.for_media', m['_id'], single = True):
if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']: if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']:
if params.get('ignore_previous', False): if params.get('ignore_previous', False):
release['status'] = 'ignored' fireEvent('release.update_status', release['_id'], status = 'ignored')
db.update(release)
else: else:
fireEvent('release.delete', release['_id'], single = True) fireEvent('release.delete', release['_id'], single = True)
@ -172,7 +174,7 @@ class MovieBase(MovieTypeBase):
# Trigger update info # Trigger update info
if added and update_after: if added and update_after:
# Do full update to get images etc # Do full update to get images etc
fireEventAsync('movie.update_info', m['_id'], default_title = params.get('title'), on_complete = onComplete) fireEventAsync('movie.update', m['_id'], default_title = params.get('title'), on_complete = onComplete)
# Remove releases # Remove releases
for rel in fireEvent('release.for_media', m['_id'], single = True): for rel in fireEvent('release.for_media', m['_id'], single = True):
@ -180,6 +182,9 @@ class MovieBase(MovieTypeBase):
db.delete(rel) db.delete(rel)
movie_dict = fireEvent('media.get', m['_id'], single = True) movie_dict = fireEvent('media.get', m['_id'], single = True)
if not movie_dict:
log.debug('Failed adding media, can\'t find it anymore')
return False
if do_search and search_after: if do_search and search_after:
onComplete = self.createOnComplete(m['_id']) onComplete = self.createOnComplete(m['_id'])
@ -256,7 +261,7 @@ class MovieBase(MovieTypeBase):
'success': False, 'success': False,
} }
def updateInfo(self, media_id = None, identifier = None, default_title = None, extended = False): def update(self, media_id = None, identifier = None, default_title = None, extended = False):
""" """
Update movie information inside media['doc']['info'] Update movie information inside media['doc']['info']
@ -269,6 +274,10 @@ class MovieBase(MovieTypeBase):
if self.shuttingDown(): if self.shuttingDown():
return return
lock_key = 'media.get.%s' % media_id if media_id else identifier
self.acquireLock(lock_key)
media = {}
try: try:
db = get_db() db = get_db()
@ -312,42 +321,16 @@ class MovieBase(MovieTypeBase):
media['title'] = def_title media['title'] = def_title
# Files # Files
images = info.get('images', []) image_urls = info.get('images', [])
media['files'] = media.get('files', {})
for image_type in ['poster']:
# Remove non-existing files
file_type = 'image_%s' % image_type
existing_files = list(set(media['files'].get(file_type, [])))
for ef in media['files'].get(file_type, []):
if not os.path.isfile(ef):
existing_files.remove(ef)
# Replace new files list
media['files'][file_type] = existing_files
if len(existing_files) == 0:
del media['files'][file_type]
# Loop over type
for image in images.get(image_type, []):
if not isinstance(image, (str, unicode)):
continue
if file_type not in media['files'] or len(media['files'].get(file_type, [])) == 0:
file_path = fireEvent('file.download', url = image, single = True)
if file_path:
media['files'][file_type] = [file_path]
break
else:
break
db.update(media) self.getPoster(media, image_urls)
return media db.update(media)
except: except:
log.error('Failed update media: %s', traceback.format_exc()) log.error('Failed update media: %s', traceback.format_exc())
return {} self.releaseLock(lock_key)
return media
def updateReleaseDate(self, media_id): def updateReleaseDate(self, media_id):
""" """
@ -363,7 +346,7 @@ class MovieBase(MovieTypeBase):
media = db.get('id', media_id) media = db.get('id', media_id)
if not media.get('info'): if not media.get('info'):
media = self.updateInfo(media_id) media = self.update(media_id)
dates = media.get('info', {}).get('release_date') dates = media.get('info', {}).get('release_date')
else: else:
dates = media.get('info').get('release_date') dates = media.get('info').get('release_date')

15
couchpotato/core/media/movie/_base/static/movie.actions.js

@ -115,8 +115,15 @@ MA.Release = new Class({
self.releases = null; self.releases = null;
if(self.options_container){ if(self.options_container){
self.options_container.destroy(); // Releases are currently displayed
self.options_container = null; if(self.options_container.isDisplayed()){
self.options_container.destroy();
self.createReleases();
}
else {
self.options_container.destroy();
self.options_container = null;
}
} }
}); });
@ -131,10 +138,10 @@ MA.Release = new Class({
}, },
createReleases: function(){ createReleases: function(refresh){
var self = this; var self = this;
if(!self.options_container){ if(!self.options_container || refresh){
self.options_container = new Element('div.options').grab( self.options_container = new Element('div.options').grab(
self.release_container = new Element('div.releases.table') self.release_container = new Element('div.releases.table')
); );

16
couchpotato/core/media/movie/_base/static/movie.js

@ -54,13 +54,21 @@ var Movie = new Class({
// Reload when releases have updated // Reload when releases have updated
self.global_events['release.update_status'] = function(notification){ self.global_events['release.update_status'] = function(notification){
var data = notification.data; var data = notification.data;
if(data && self.data._id == data.movie_id){ if(data && self.data._id == data.media_id){
if(!self.data.releases) if(!self.data.releases)
self.data.releases = []; self.data.releases = [];
self.data.releases.push({'quality': data.quality, 'status': data.status}); var updated = false;
self.updateReleases(); self.data.releases.each(function(release){
if(release._id == data._id){
release['status'] = data.status;
updated = true;
}
});
if(updated)
self.updateReleases();
} }
}; };
@ -159,7 +167,7 @@ var Movie = new Class({
} }
} }
}), }),
self.thumbnail = (self.data.files && self.data.files.image_poster) ? new Element('img', { self.thumbnail = (self.data.files && self.data.files.image_poster && self.data.files.image_poster.length > 0) ? new Element('img', {
'class': 'type_image poster', 'class': 'type_image poster',
'src': Api.createUrl('file.cache') + self.data.files.image_poster[0].split(Api.getOption('path_sep')).pop() 'src': Api.createUrl('file.cache') + self.data.files.image_poster[0].split(Api.getOption('path_sep')).pop()
}): null, }): null,

7
couchpotato/core/media/movie/charts/__init__.py

@ -22,13 +22,6 @@ config = [{
'description': 'Maximum number of items displayed from each chart.', 'description': 'Maximum number of items displayed from each chart.',
}, },
{ {
'name': 'update_interval',
'default': 12,
'type': 'int',
'advanced': True,
'description': '(hours)',
},
{
'name': 'hide_wanted', 'name': 'hide_wanted',
'default': False, 'default': False,
'type': 'bool', 'type': 'bool',

6
couchpotato/core/media/movie/charts/main.py

@ -1,6 +1,5 @@
import time import time
from couchpotato import tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.event import addEvent,fireEvent from couchpotato.core.event import addEvent,fireEvent
@ -13,13 +12,14 @@ log = CPLog(__name__)
class Charts(Plugin): class Charts(Plugin):
update_in_progress = False update_in_progress = False
update_interval = 72 # hours
def __init__(self): def __init__(self):
addApiView('charts.view', self.automationView) addApiView('charts.view', self.automationView)
addEvent('app.load', self.setCrons) addEvent('app.load', self.setCrons)
def setCrons(self): def setCrons(self):
fireEvent('schedule.interval', 'charts.update_cache', self.updateViewCache, hours = self.conf('update_interval', default = 12)) fireEvent('schedule.interval', 'charts.update_cache', self.updateViewCache, hours = self.update_interval)
def automationView(self, force_update = False, **kwargs): def automationView(self, force_update = False, **kwargs):
@ -52,7 +52,7 @@ class Charts(Plugin):
for chart in charts: for chart in charts:
chart['hide_wanted'] = self.conf('hide_wanted') chart['hide_wanted'] = self.conf('hide_wanted')
chart['hide_library'] = self.conf('hide_library') chart['hide_library'] = self.conf('hide_library')
self.setCache('charts_cached', charts, timeout = 7200 * tryInt(self.conf('update_interval', default = 12))) self.setCache('charts_cached', charts, timeout = self.update_interval * 3600)
except: except:
log.error('Failed refreshing charts') log.error('Failed refreshing charts')

32
couchpotato/core/media/movie/charts/static/charts.js

@ -2,6 +2,8 @@ var Charts = new Class({
Implements: [Options, Events], Implements: [Options, Events],
shown_once: false,
initialize: function(options){ initialize: function(options){
var self = this; var self = this;
self.setOptions(options); self.setOptions(options);
@ -40,17 +42,13 @@ var Charts = new Class({
) )
); );
if( Cookie.read('suggestions_charts_menu_selected') === 'charts') if( Cookie.read('suggestions_charts_menu_selected') === 'charts'){
self.el.show(); self.show();
self.fireEvent.delay(0, self, 'created');
}
else else
self.el.hide(); self.el.hide();
self.api_request = Api.request('charts.view', {
'onComplete': self.fill.bind(self)
});
self.fireEvent.delay(0, self, 'created');
}, },
fill: function(json){ fill: function(json){
@ -157,6 +155,24 @@ var Charts = new Class({
}, },
show: function(){
var self = this;
self.el.show();
if(!self.shown_once){
self.api_request = Api.request('charts.view', {
'onComplete': self.fill.bind(self)
});
self.shown_once = true;
}
},
hide: function(){
this.el.hide();
},
afterAdded: function(m){ afterAdded: function(m){
$(m).getElement('div.chart_number') $(m).getElement('div.chart_number')

55
couchpotato/core/media/movie/providers/automation/bluray.py

@ -1,3 +1,5 @@
import traceback
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from couchpotato import fireEvent from couchpotato import fireEvent
from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.rss import RSS
@ -5,6 +7,7 @@ from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.media.movie.providers.automation.base import Automation from couchpotato.core.media.movie.providers.automation.base import Automation
log = CPLog(__name__) log = CPLog(__name__)
autoload = 'Bluray' autoload = 'Bluray'
@ -34,27 +37,49 @@ class Bluray(Automation, RSS):
try: try:
# Stop if the release year is before the minimal year # Stop if the release year is before the minimal year
page_year = soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].h3.get_text().split(', ')[1] brk = False
if tryInt(page_year) < self.getMinimal('year'): h3s = soup.body.find_all('h3')
for h3 in h3s:
if h3.parent.name != 'a':
try:
page_year = tryInt(h3.get_text()[-4:])
if page_year > 0 and page_year < self.getMinimal('year'):
brk = True
except:
log.error('Failed determining page year: %s', traceback.format_exc())
brk = True
break
if brk:
break break
for table in soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].find_all('table')[1:20]: for h3 in h3s:
name = table.h3.get_text().lower().split('blu-ray')[0].strip() try:
year = table.small.get_text().split('|')[1].strip() if h3.parent.name == 'a':
name = h3.get_text().lower().split('blu-ray')[0].strip()
if not name.find('/') == -1: # make sure it is not a double movie release
continue
if not h3.parent.parent.small: # ignore non-movie tables
continue
if not name.find('/') == -1: # make sure it is not a double movie release year = h3.parent.parent.small.get_text().split('|')[1].strip()
continue
if tryInt(year) < self.getMinimal('year'): if tryInt(year) < self.getMinimal('year'):
continue continue
imdb = self.search(name, year) imdb = self.search(name, year)
if imdb: if imdb:
if self.isMinimalMovie(imdb): if self.isMinimalMovie(imdb):
movies.append(imdb['imdb']) movies.append(imdb['imdb'])
except:
log.debug('Error parsing movie html: %s', traceback.format_exc())
break
except: except:
log.debug('Error loading page: %s', page) log.debug('Error loading page %s: %s', (page, traceback.format_exc()))
break break
self.conf('backlog', value = False) self.conf('backlog', value = False)
@ -134,7 +159,7 @@ config = [{
{ {
'name': 'backlog', 'name': 'backlog',
'advanced': True, 'advanced': True,
'description': 'Parses the history until the minimum movie year is reached. (Will be disabled once it has completed)', 'description': ('Parses the history until the minimum movie year is reached. (Takes a while)', 'Will be disabled once it has completed'),
'default': False, 'default': False,
'type': 'bool', 'type': 'bool',
}, },

4
couchpotato/core/media/movie/providers/info/couchpotatoapi.py

@ -2,7 +2,7 @@ import base64
import time import time
from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.encoding import tryUrlencode, ss
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.media.movie.providers.base import MovieProvider from couchpotato.core.media.movie.providers.base import MovieProvider
from couchpotato.environment import Env from couchpotato.environment import Env
@ -66,7 +66,7 @@ class CouchPotatoApi(MovieProvider):
if not name: if not name:
return return
name_enc = base64.b64encode(name) name_enc = base64.b64encode(ss(name))
return self.getJsonData(self.urls['validate'] % name_enc, headers = self.getRequestHeaders()) return self.getJsonData(self.urls['validate'] % name_enc, headers = self.getRequestHeaders())
def isMovie(self, identifier = None): def isMovie(self, identifier = None):

5
couchpotato/core/media/movie/providers/info/fanarttv.py

@ -23,10 +23,9 @@ class FanartTV(MovieProvider):
def __init__(self): def __init__(self):
addEvent('movie.info', self.getArt, priority = 1) addEvent('movie.info', self.getArt, priority = 1)
def getArt(self, identifier = None, **kwargs): def getArt(self, identifier = None, extended = True, **kwargs):
log.debug("Getting Extra Artwork from Fanart.tv...") if not identifier or not extended:
if not identifier:
return {} return {}
images = {} images = {}

273
couchpotato/core/media/movie/providers/info/themoviedb.py

@ -1,11 +1,10 @@
import traceback import traceback
from couchpotato.core.event import addEvent from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss from couchpotato.core.helpers.encoding import toUnicode, ss, tryUrlencode
from couchpotato.core.helpers.variable import tryInt from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.media.movie.providers.base import MovieProvider from couchpotato.core.media.movie.providers.base import MovieProvider
import tmdb3
log = CPLog(__name__) log = CPLog(__name__)
@ -13,54 +12,65 @@ autoload = 'TheMovieDb'
class TheMovieDb(MovieProvider): class TheMovieDb(MovieProvider):
MAX_EXTRATHUMBS = 4
http_time_between_calls = .35
configuration = {
'images': {
'secure_base_url': 'https://image.tmdb.org/t/p/',
},
}
def __init__(self): def __init__(self):
addEvent('info.search', self.search, priority = 3)
addEvent('movie.search', self.search, priority = 3)
addEvent('movie.info', self.getInfo, priority = 3) addEvent('movie.info', self.getInfo, priority = 3)
addEvent('movie.info_by_tmdb', self.getInfo) addEvent('movie.info_by_tmdb', self.getInfo)
addEvent('app.load', self.config)
# Configure TMDB settings def config(self):
tmdb3.set_key(self.conf('api_key')) configuration = self.request('configuration')
tmdb3.set_cache('null') if configuration:
self.configuration = configuration
def search(self, q, limit = 12): def search(self, q, limit = 3):
""" Find movie by name """ """ Find movie by name """
if self.isDisabled(): if self.isDisabled():
return False return False
search_string = simplifyString(q) log.debug('Searching for movie: %s', q)
cache_key = 'tmdb.cache.%s.%s' % (search_string, limit)
results = self.getCache(cache_key)
if not results: raw = None
log.debug('Searching for movie: %s', q) try:
name_year = fireEvent('scanner.name_year', q, single = True)
raw = self.request('search/movie', {
'query': name_year.get('name', q),
'year': name_year.get('year'),
'search_type': 'ngram' if limit > 1 else 'phrase'
}, return_key = 'results')
except:
log.error('Failed searching TMDB for "%s": %s', (q, traceback.format_exc()))
raw = None results = []
if raw:
try: try:
raw = tmdb3.searchMovie(search_string) nr = 0
except:
log.error('Failed searching TMDB for "%s": %s', (search_string, traceback.format_exc()))
results = [] for movie in raw:
if raw: parsed_movie = self.parseMovie(movie, extended = False)
try: results.append(parsed_movie)
nr = 0
for movie in raw: nr += 1
results.append(self.parseMovie(movie, extended = False)) if nr == limit:
break
nr += 1 log.info('Found: %s', [result['titles'][0] + ' (' + str(result.get('year', 0)) + ')' for result in results])
if nr == limit:
break
log.info('Found: %s', [result['titles'][0] + ' (' + str(result.get('year', 0)) + ')' for result in results]) return results
except SyntaxError as e:
self.setCache(cache_key, results) log.error('Failed to parse XML response: %s', e)
return results return False
except SyntaxError as e:
log.error('Failed to parse XML response: %s', e)
return False
return results return results
@ -69,101 +79,89 @@ class TheMovieDb(MovieProvider):
if not identifier: if not identifier:
return {} return {}
cache_key = 'tmdb.cache.%s%s' % (identifier, '.ex' if extended else '') result = self.parseMovie({
result = self.getCache(cache_key) 'id': identifier
}, extended = extended)
if not result:
try:
log.debug('Getting info: %s', cache_key)
# noinspection PyArgumentList
movie = tmdb3.Movie(identifier)
try: exists = movie.title is not None
except: exists = False
if exists:
result = self.parseMovie(movie, extended = extended)
self.setCache(cache_key, result)
else:
result = {}
except:
log.error('Failed getting info for %s: %s', (identifier, traceback.format_exc()))
return result return result
def parseMovie(self, movie, extended = True): def parseMovie(self, movie, extended = True):
cache_key = 'tmdb.cache.%s%s' % (movie.id, '.ex' if extended else '') # Do request, append other items
movie_data = self.getCache(cache_key) movie = self.request('movie/%s' % movie.get('id'), {
'append_to_response': 'alternative_titles' + (',images,casts' if extended else '')
})
# Images
poster = self.getImage(movie, type = 'poster', size = 'w154')
poster_original = self.getImage(movie, type = 'poster', size = 'original')
backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original')
extra_thumbs = self.getMultImages(movie, type = 'backdrops', size = 'original') if extended else []
images = {
'poster': [poster] if poster else [],
#'backdrop': [backdrop] if backdrop else [],
'poster_original': [poster_original] if poster_original else [],
'backdrop_original': [backdrop_original] if backdrop_original else [],
'actors': {},
'extra_thumbs': extra_thumbs
}
# Genres
try:
genres = [genre.get('name') for genre in movie.get('genres', [])]
except:
genres = []
if not movie_data: # 1900 is the same as None
year = str(movie.get('release_date') or '')[:4]
if not movie.get('release_date') or year == '1900' or year.lower() == 'none':
year = None
# Images # Gather actors data
poster = self.getImage(movie, type = 'poster', size = 'w154') actors = {}
poster_original = self.getImage(movie, type = 'poster', size = 'original') if extended:
backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original')
extra_thumbs = self.getMultImages(movie, type = 'backdrops', size = 'original', n = self.MAX_EXTRATHUMBS, skipfirst = True)
images = { # Full data
'poster': [poster] if poster else [], cast = movie.get('casts', {}).get('cast', [])
#'backdrop': [backdrop] if backdrop else [],
'poster_original': [poster_original] if poster_original else [],
'backdrop_original': [backdrop_original] if backdrop_original else [],
'actors': {},
'extra_thumbs': extra_thumbs
}
# Genres for cast_item in cast:
try: try:
genres = [genre.name for genre in movie.genres] actors[toUnicode(cast_item.get('name'))] = toUnicode(cast_item.get('character'))
except: images['actors'][toUnicode(cast_item.get('name'))] = self.getImage(cast_item, type = 'profile', size = 'original')
genres = [] except:
log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc()))
# 1900 is the same as None
year = str(movie.releasedate or '')[:4] movie_data = {
if not movie.releasedate or year == '1900' or year.lower() == 'none': 'type': 'movie',
year = None 'via_tmdb': True,
'tmdb_id': movie.get('id'),
# Gather actors data 'titles': [toUnicode(movie.get('title'))],
actors = {} 'original_title': movie.get('original_title'),
if extended: 'images': images,
for cast_item in movie.cast: 'imdb': movie.get('imdb_id'),
try: 'runtime': movie.get('runtime'),
actors[toUnicode(cast_item.name)] = toUnicode(cast_item.character) 'released': str(movie.get('release_date')),
images['actors'][toUnicode(cast_item.name)] = self.getImage(cast_item, type = 'profile', size = 'original') 'year': tryInt(year, None),
except: 'plot': movie.get('overview'),
log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc())) 'genres': genres,
'collection': getattr(movie.get('belongs_to_collection'), 'name', None),
movie_data = { 'actor_roles': actors
'type': 'movie', }
'via_tmdb': True,
'tmdb_id': movie.id, movie_data = dict((k, v) for k, v in movie_data.items() if v)
'titles': [toUnicode(movie.title)],
'original_title': movie.originaltitle, # Add alternative names
'images': images, if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']:
'imdb': movie.imdb, movie_data['titles'].append(movie_data['original_title'])
'runtime': movie.runtime,
'released': str(movie.releasedate), # Add alternative titles
'year': tryInt(year, None), alternate_titles = movie.get('alternative_titles', {}).get('titles', [])
'plot': movie.overview,
'genres': genres, for alt in alternate_titles:
'collection': getattr(movie.collection, 'name', None), alt_name = alt.get('title')
'actor_roles': actors if alt_name and alt_name not in movie_data['titles'] and alt_name.lower() != 'none' and alt_name is not None:
} movie_data['titles'].append(alt_name)
movie_data = dict((k, v) for k, v in movie_data.items() if v)
# Add alternative names
if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']:
movie_data['titles'].append(movie_data['original_title'])
if extended:
for alt in movie.alternate_titles:
alt_name = alt.title
if alt_name and alt_name not in movie_data['titles'] and alt_name.lower() != 'none' and alt_name is not None:
movie_data['titles'].append(alt_name)
# Cache movie parsed
self.setCache(cache_key, movie_data)
return movie_data return movie_data
@ -171,36 +169,37 @@ class TheMovieDb(MovieProvider):
image_url = '' image_url = ''
try: try:
image_url = getattr(movie, type).geturl(size = size) path = movie.get('%s_path' % type)
image_url = '%s%s%s' % (self.configuration['images']['secure_base_url'], size, path)
except: except:
log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie)))) log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie))))
return image_url return image_url
def getMultImages(self, movie, type = 'backdrops', size = 'original', n = -1, skipfirst = False): def getMultImages(self, movie, type = 'backdrops', size = 'original'):
"""
If n < 0, return all images. Otherwise return n images.
If n > len(getattr(movie, type)), then return all images.
If skipfirst is True, then it will skip getattr(movie, type)[0]. This
is because backdrops[0] is typically backdrop.
"""
image_urls = [] image_urls = []
try: try:
images = getattr(movie, type) for image in movie.get('images', {}).get(type, [])[1:5]:
if n < 0 or n > len(images): image_urls.append(self.getImage(image, 'file', size))
num_images = len(images)
else:
num_images = n
for i in range(int(skipfirst), num_images + int(skipfirst)):
image_urls.append(images[i].geturl(size = size))
except: except:
log.debug('Failed getting %i %s.%s for "%s"', (n, type, size, ss(str(movie)))) log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie))))
return image_urls return image_urls
def request(self, call = '', params = {}, return_key = None):
params = dict((k, v) for k, v in params.items() if v)
params = tryUrlencode(params)
url = 'http://api.themoviedb.org/3/%s?api_key=%s%s' % (call, self.conf('api_key'), '&%s' % params if params else '')
data = self.getJsonData(url)
if data and return_key and return_key in data:
data = data.get(return_key)
return data
def isDisabled(self): def isDisabled(self):
if self.conf('api_key') == '': if self.conf('api_key') == '':
log.error('No API key provided.') log.error('No API key provided.')

2
couchpotato/core/media/movie/providers/metadata/base.py

@ -28,7 +28,7 @@ class MovieMetaData(MetaDataBase):
# Update library to get latest info # Update library to get latest info
try: try:
group['media'] = fireEvent('movie.update_info', group['media'].get('_id'), identifier = getIdentifier(group['media']), extended = True, single = True) group['media'] = fireEvent('movie.update', group['media'].get('_id'), identifier = getIdentifier(group['media']), extended = True, single = True)
except: except:
log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc()) log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc())

30
couchpotato/core/media/movie/providers/nzb/nzbindex.py

@ -1,30 +0,0 @@
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.logger import CPLog
from couchpotato.core.event import fireEvent
from couchpotato.core.media._base.providers.nzb.nzbindex import Base
from couchpotato.core.media.movie.providers.base import MovieProvider
from couchpotato.environment import Env
log = CPLog(__name__)
autoload = 'NzbIndex'
class NzbIndex(MovieProvider, Base):
def buildUrl(self, media, quality):
title = fireEvent('library.query', media, include_year = False, single = True)
year = media['info']['year']
query = tryUrlencode({
'q': '"%s %s" | "%s (%s)"' % (title, year, title, year),
'age': Env.setting('retention', 'nzb'),
'sort': 'agedesc',
'minsize': quality.get('size_min'),
'maxsize': quality.get('size_max'),
'rating': 1,
'max': 250,
'more': 1,
'complete': 1,
})
return query

2
couchpotato/core/media/movie/providers/torrent/iptorrents.py

@ -13,7 +13,7 @@ class IPTorrents(MovieProvider, Base):
([87], ['3d']), ([87], ['3d']),
([48], ['720p', '1080p', 'bd50']), ([48], ['720p', '1080p', 'bd50']),
([72], ['cam', 'ts', 'tc', 'r5', 'scr']), ([72], ['cam', 'ts', 'tc', 'r5', 'scr']),
([7], ['dvdrip', 'brrip']), ([7,48], ['dvdrip', 'brrip']),
([6], ['dvdr']), ([6], ['dvdr']),
] ]

4
couchpotato/core/media/movie/providers/torrent/passthepopcorn.py

@ -13,7 +13,7 @@ class PassThePopcorn(MovieProvider, Base):
'bd50': {'media': 'Blu-ray', 'format': 'BD50'}, 'bd50': {'media': 'Blu-ray', 'format': 'BD50'},
'1080p': {'resolution': '1080p'}, '1080p': {'resolution': '1080p'},
'720p': {'resolution': '720p'}, '720p': {'resolution': '720p'},
'brrip': {'media': 'Blu-ray'}, 'brrip': {'resolution': 'anyhd'},
'dvdr': {'resolution': 'anysd'}, 'dvdr': {'resolution': 'anysd'},
'dvdrip': {'media': 'DVD'}, 'dvdrip': {'media': 'DVD'},
'scr': {'media': 'DVD-Screener'}, 'scr': {'media': 'DVD-Screener'},
@ -27,7 +27,7 @@ class PassThePopcorn(MovieProvider, Base):
'bd50': {'Codec': ['BD50']}, 'bd50': {'Codec': ['BD50']},
'1080p': {'Resolution': ['1080p']}, '1080p': {'Resolution': ['1080p']},
'720p': {'Resolution': ['720p']}, '720p': {'Resolution': ['720p']},
'brrip': {'Source': ['Blu-ray'], 'Quality': ['High Definition'], 'Container': ['!ISO']}, 'brrip': {'Quality': ['High Definition'], 'Container': ['!ISO']},
'dvdr': {'Codec': ['DVD5', 'DVD9']}, 'dvdr': {'Codec': ['DVD5', 'DVD9']},
'dvdrip': {'Source': ['DVD'], 'Codec': ['!DVD5', '!DVD9']}, 'dvdrip': {'Source': ['DVD'], 'Codec': ['!DVD5', '!DVD9']},
'scr': {'Source': ['DVD-Screener']}, 'scr': {'Source': ['DVD-Screener']},

2
couchpotato/core/media/movie/providers/torrent/torrentleech.py

@ -11,7 +11,7 @@ autoload = 'TorrentLeech'
class TorrentLeech(MovieProvider, Base): class TorrentLeech(MovieProvider, Base):
cat_ids = [ cat_ids = [
([13], ['720p', '1080p']), ([13], ['720p', '1080p', 'bd50']),
([8], ['cam']), ([8], ['cam']),
([9], ['ts', 'tc']), ([9], ['ts', 'tc']),
([10], ['r5', 'scr']), ([10], ['r5', 'scr']),

10
couchpotato/core/media/movie/providers/trailer/hdtrailers.py

@ -3,7 +3,7 @@ import re
from bs4 import SoupStrainer, BeautifulSoup from bs4 import SoupStrainer, BeautifulSoup
from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import mergeDicts, getTitle from couchpotato.core.helpers.variable import mergeDicts, getTitle, getIdentifier
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.media.movie.providers.trailer.base import TrailerProvider from couchpotato.core.media.movie.providers.trailer.base import TrailerProvider
from requests import HTTPError from requests import HTTPError
@ -29,7 +29,7 @@ class HDTrailers(TrailerProvider):
url = self.urls['api'] % self.movieUrlName(movie_name) url = self.urls['api'] % self.movieUrlName(movie_name)
try: try:
data = self.getCache('hdtrailers.%s' % group['identifier'], url, show_error = False) data = self.getCache('hdtrailers.%s' % getIdentifier(group), url, show_error = False)
except HTTPError: except HTTPError:
log.debug('No page found for: %s', movie_name) log.debug('No page found for: %s', movie_name)
data = None data = None
@ -59,7 +59,7 @@ class HDTrailers(TrailerProvider):
url = "%s?%s" % (self.urls['backup'], tryUrlencode({'s':movie_name})) url = "%s?%s" % (self.urls['backup'], tryUrlencode({'s':movie_name}))
try: try:
data = self.getCache('hdtrailers.alt.%s' % group['identifier'], url, show_error = False) data = self.getCache('hdtrailers.alt.%s' % getIdentifier(group), url, show_error = False)
except HTTPError: except HTTPError:
log.debug('No alternative page found for: %s', movie_name) log.debug('No alternative page found for: %s', movie_name)
data = None data = None
@ -68,7 +68,7 @@ class HDTrailers(TrailerProvider):
return results return results
try: try:
html = BeautifulSoup(data, 'html.parser', parse_only = self.only_tables_tags) html = BeautifulSoup(data, parse_only = self.only_tables_tags)
result_table = html.find_all('h2', text = re.compile(movie_name)) result_table = html.find_all('h2', text = re.compile(movie_name))
for h2 in result_table: for h2 in result_table:
@ -90,7 +90,7 @@ class HDTrailers(TrailerProvider):
results = {'480p':[], '720p':[], '1080p':[]} results = {'480p':[], '720p':[], '1080p':[]}
try: try:
html = BeautifulSoup(data, 'html.parser', parse_only = self.only_tables_tags) html = BeautifulSoup(data, parse_only = self.only_tables_tags)
result_table = html.find('table', attrs = {'class':'bottomTable'}) result_table = html.find('table', attrs = {'class':'bottomTable'})
for tr in result_table.find_all('tr'): for tr in result_table.find_all('tr'):

4
couchpotato/core/media/movie/providers/userscript/filmstarts.py

@ -25,6 +25,6 @@ class Filmstarts(UserscriptBase):
name = html.find("meta", {"property":"og:title"})['content'] name = html.find("meta", {"property":"og:title"})['content']
# Year of production is not available in the meta data, so get it from the table # Year of production is not available in the meta data, so get it from the table
year = table.find("tr", text="Produktionsjahr").parent.parent.parent.td.text year = table.find(text="Produktionsjahr").parent.parent.next_sibling.text
return self.search(name, year) return self.search(name, year)

39
couchpotato/core/media/movie/searcher.py

@ -74,7 +74,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
self.in_progress = True self.in_progress = True
fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started') fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started')
medias = [x['_id'] for x in fireEvent('media.with_status', 'active', with_doc = False, single = True)] medias = [x['_id'] for x in fireEvent('media.with_status', 'active', types = 'movie', with_doc = False, single = True)]
random.shuffle(medias) random.shuffle(medias)
total = len(medias) total = len(medias)
@ -89,12 +89,13 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
for media_id in medias: for media_id in medias:
media = fireEvent('media.get', media_id, single = True) media = fireEvent('media.get', media_id, single = True)
if not media: continue
try: try:
self.single(media, search_protocols, manual = manual) self.single(media, search_protocols, manual = manual)
except IndexError: except IndexError:
log.error('Forcing library update for %s, if you see this often, please report: %s', (getIdentifier(media), traceback.format_exc())) log.error('Forcing library update for %s, if you see this often, please report: %s', (getIdentifier(media), traceback.format_exc()))
fireEvent('movie.update_info', media_id) fireEvent('movie.update', media_id)
except: except:
log.error('Search failed for %s: %s', (getIdentifier(media), traceback.format_exc())) log.error('Search failed for %s: %s', (getIdentifier(media), traceback.format_exc()))
@ -140,17 +141,17 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
previous_releases = movie.get('releases', []) previous_releases = movie.get('releases', [])
too_early_to_search = [] too_early_to_search = []
outside_eta_results = 0 outside_eta_results = 0
alway_search = self.conf('always_search') always_search = self.conf('always_search')
ignore_eta = manual ignore_eta = manual
total_result_count = 0 total_result_count = 0
fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'_id': movie['_id']}, message = 'Searching for "%s"' % default_title) fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'_id': movie['_id']}, message = 'Searching for "%s"' % default_title)
# Ignore eta once every 7 days # Ignore eta once every 7 days
if not alway_search: if not always_search:
prop_name = 'last_ignored_eta.%s' % movie['_id'] prop_name = 'last_ignored_eta.%s' % movie['_id']
last_ignored_eta = float(Env.prop(prop_name, default = 0)) last_ignored_eta = float(Env.prop(prop_name, default = 0))
if last_ignored_eta > time.time() - 604800: if last_ignored_eta < time.time() - 604800:
ignore_eta = True ignore_eta = True
Env.prop(prop_name, value = time.time()) Env.prop(prop_name, value = time.time())
@ -165,11 +166,12 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
'quality': q_identifier, 'quality': q_identifier,
'finish': profile['finish'][index], 'finish': profile['finish'][index],
'wait_for': tryInt(profile['wait_for'][index]), 'wait_for': tryInt(profile['wait_for'][index]),
'3d': profile['3d'][index] if profile.get('3d') else False '3d': profile['3d'][index] if profile.get('3d') else False,
'minimum_score': profile.get('minimum_score', 1),
} }
could_not_be_released = not self.couldBeReleased(q_identifier in pre_releases, release_dates, movie['info']['year']) could_not_be_released = not self.couldBeReleased(q_identifier in pre_releases, release_dates, movie['info']['year'])
if not alway_search and could_not_be_released: if not always_search and could_not_be_released:
too_early_to_search.append(q_identifier) too_early_to_search.append(q_identifier)
# Skip release, if ETA isn't ignored # Skip release, if ETA isn't ignored
@ -195,7 +197,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
break break
quality = fireEvent('quality.single', identifier = q_identifier, single = True) quality = fireEvent('quality.single', identifier = q_identifier, single = True)
log.info('Search for %s in %s%s', (default_title, quality['label'], ' ignoring ETA' if alway_search or ignore_eta else '')) log.info('Search for %s in %s%s', (default_title, quality['label'], ' ignoring ETA' if always_search or ignore_eta else ''))
# Extend quality with profile customs # Extend quality with profile customs
quality['custom'] = quality_custom quality['custom'] = quality_custom
@ -222,7 +224,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
log.debug('Found %s releases for "%s", but ETA isn\'t correct yet.', (results_count, default_title)) log.debug('Found %s releases for "%s", but ETA isn\'t correct yet.', (results_count, default_title))
# Try find a valid result and download it # Try find a valid result and download it
if (force_download or not could_not_be_released or alway_search) and fireEvent('release.try_download_result', results, movie, quality_custom, single = True): if (force_download or not could_not_be_released or always_search) and fireEvent('release.try_download_result', results, movie, quality_custom, single = True):
ret = True ret = True
# Remove releases that aren't found anymore # Remove releases that aren't found anymore
@ -240,7 +242,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
break break
if total_result_count > 0: if total_result_count > 0:
fireEvent('media.tag', movie['_id'], 'recent', single = True) fireEvent('media.tag', movie['_id'], 'recent', update_edited = True, single = True)
if len(too_early_to_search) > 0: if len(too_early_to_search) > 0:
log.info2('Too early to search for %s, %s', (too_early_to_search, default_title)) log.info2('Too early to search for %s, %s', (too_early_to_search, default_title))
@ -277,7 +279,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
# Contains lower quality string # Contains lower quality string
contains_other = fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True) contains_other = fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True)
if contains_other != False: if contains_other and isinstance(contains_other, dict):
log.info2('Wrong: %s, looking for %s, found %s', (nzb['name'], quality['label'], [x for x in contains_other] if contains_other else 'no quality')) log.info2('Wrong: %s, looking for %s, found %s', (nzb['name'], quality['label'], [x for x in contains_other] if contains_other else 'no quality'))
return False return False
@ -381,16 +383,17 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
def tryNextRelease(self, media_id, manual = False, force_download = False): def tryNextRelease(self, media_id, manual = False, force_download = False):
try: try:
db = get_db()
rels = fireEvent('media.with_status', ['snatched', 'done'], single = True) rels = fireEvent('release.for_media', media_id, single = True)
for rel in rels: for rel in rels:
rel['status'] = 'ignored' if rel.get('status') in ['snatched', 'done']:
db.update(rel) fireEvent('release.update_status', rel.get('_id'), status = 'ignored')
movie_dict = fireEvent('media.get', media_id, single = True) media = fireEvent('media.get', media_id, single = True)
log.info('Trying next release for: %s', getTitle(movie_dict)) if media:
self.single(movie_dict, manual = manual, force_download = force_download) log.info('Trying next release for: %s', getTitle(media))
self.single(media, manual = manual, force_download = force_download)
return True return True

2
couchpotato/core/media/movie/suggestion/main.py

@ -27,7 +27,7 @@ class Suggestion(Plugin):
else: else:
if not movies or len(movies) == 0: if not movies or len(movies) == 0:
active_movies = fireEvent('media.with_status', ['active', 'done'], single = True) active_movies = fireEvent('media.with_status', ['active', 'done'], types = 'movie', single = True)
movies = [getIdentifier(x) for x in active_movies] movies = [getIdentifier(x) for x in active_movies]
if not ignored or len(ignored) == 0: if not ignored or len(ignored) == 0:

31
couchpotato/core/media/movie/suggestion/static/suggest.js

@ -2,6 +2,8 @@ var SuggestList = new Class({
Implements: [Options, Events], Implements: [Options, Events],
shown_once: false,
initialize: function(options){ initialize: function(options){
var self = this; var self = this;
self.setOptions(options); self.setOptions(options);
@ -44,12 +46,13 @@ var SuggestList = new Class({
} }
}); });
var cookie_menu_select = Cookie.read('suggestions_charts_menu_selected'); var cookie_menu_select = Cookie.read('suggestions_charts_menu_selected') || 'suggestions';
if( cookie_menu_select === 'suggestions' || cookie_menu_select === null ) self.el.show(); else self.el.hide(); if( cookie_menu_select === 'suggestions')
self.show();
else
self.hide();
self.api_request = Api.request('suggestion.view', { self.fireEvent('created');
'onComplete': self.fill.bind(self)
});
}, },
@ -145,6 +148,24 @@ var SuggestList = new Class({
}, },
show: function(){
var self = this;
self.el.show();
if(!self.shown_once){
self.api_request = Api.request('suggestion.view', {
'onComplete': self.fill.bind(self)
});
self.shown_once = true;
}
},
hide: function(){
this.el.hide();
},
toElement: function(){ toElement: function(){
return this.el; return this.el;
} }

25
couchpotato/core/notifications/core/main.py

@ -3,6 +3,7 @@ import threading
import time import time
import traceback import traceback
import uuid import uuid
from CodernityDB.database import RecordDeleted
from couchpotato import get_db from couchpotato import get_db
from couchpotato.api import addApiView, addNonBlockApiView from couchpotato.api import addApiView, addNonBlockApiView
@ -66,7 +67,9 @@ class CoreNotifier(Notification):
fireEvent('schedule.interval', 'core.clean_messages', self.cleanMessages, seconds = 15, single = True) fireEvent('schedule.interval', 'core.clean_messages', self.cleanMessages, seconds = 15, single = True)
addEvent('app.load', self.clean) addEvent('app.load', self.clean)
addEvent('app.load', self.checkMessages)
if not Env.get('dev'):
addEvent('app.load', self.checkMessages)
self.messages = [] self.messages = []
self.listeners = [] self.listeners = []
@ -153,9 +156,14 @@ class CoreNotifier(Notification):
n = { n = {
'_t': 'notification', '_t': 'notification',
'time': int(time.time()), 'time': int(time.time()),
'message': toUnicode(message), 'message': toUnicode(message)
'data': data
} }
if data.get('sticky'):
n['sticky'] = True
if data.get('important'):
n['important'] = True
db.insert(n) db.insert(n)
self.frontend(type = listener, data = n) self.frontend(type = listener, data = n)
@ -263,11 +271,16 @@ class CoreNotifier(Notification):
if init: if init:
db = get_db() db = get_db()
notifications = db.all('notification', with_doc = True) notifications = db.all('notification')
for n in notifications: for n in notifications:
if n['doc'].get('time') > (time.time() - 604800):
messages.append(n['doc']) try:
doc = db.get('id', n.get('_id'))
if doc.get('time') > (time.time() - 604800):
messages.append(doc)
except RecordDeleted:
pass
return { return {
'success': True, 'success': True,

4
couchpotato/core/notifications/core/static/notification.js

@ -50,7 +50,7 @@ var NotificationBase = new Class({
, 'top'); , 'top');
self.notifications.include(result); self.notifications.include(result);
if((result.data.important !== undefined || result.data.sticky !== undefined) && !result.read){ if((result.important !== undefined || result.sticky !== undefined) && !result.read){
var sticky = true; var sticky = true;
App.trigger('message', [result.message, sticky, result]) App.trigger('message', [result.message, sticky, result])
} }
@ -72,7 +72,7 @@ var NotificationBase = new Class({
if(!force_ids) { if(!force_ids) {
var rn = self.notifications.filter(function(n){ var rn = self.notifications.filter(function(n){
return !n.read && n.data.important === undefined return !n.read && n.important === undefined
}); });
var ids = []; var ids = [];

2
couchpotato/core/notifications/email_.py

@ -42,7 +42,7 @@ class Email(Notification):
# Open the SMTP connection, via SSL if requested # Open the SMTP connection, via SSL if requested
log.debug("Connecting to host %s on port %s" % (smtp_server, smtp_port)) log.debug("Connecting to host %s on port %s" % (smtp_server, smtp_port))
log.debug("SMTP over SSL %s", ("enabled" if ssl == 1 else "disabled")) log.debug("SMTP over SSL %s", ("enabled" if ssl == 1 else "disabled"))
mailserver = smtplib.SMTP_SSL(smtp_server) if ssl == 1 else smtplib.SMTP(smtp_server) mailserver = smtplib.SMTP_SSL(smtp_server, smtp_port) if ssl == 1 else smtplib.SMTP(smtp_server, smtp_port)
if starttls: if starttls:
log.debug("Using StartTLS to initiate the connection with the SMTP server") log.debug("Using StartTLS to initiate the connection with the SMTP server")

8
couchpotato/core/notifications/growl.py

@ -34,9 +34,9 @@ class Growl(Notification):
self.growl = notifier.GrowlNotifier( self.growl = notifier.GrowlNotifier(
applicationName = Env.get('appname'), applicationName = Env.get('appname'),
notifications = ["Updates"], notifications = ['Updates'],
defaultNotifications = ["Updates"], defaultNotifications = ['Updates'],
applicationIcon = '%s/static/images/couch.png' % fireEvent('app.api_url', single = True), applicationIcon = self.getNotificationImage('medium'),
hostname = hostname if hostname else 'localhost', hostname = hostname if hostname else 'localhost',
password = password if password else None, password = password if password else None,
port = port if port else 23053 port = port if port else 23053
@ -56,7 +56,7 @@ class Growl(Notification):
try: try:
self.growl.notify( self.growl.notify(
noteType = "Updates", noteType = 'Updates',
title = self.default_title, title = self.default_title,
description = message, description = message,
sticky = False, sticky = False,

68
couchpotato/core/notifications/notifymywp.py

@ -1,68 +0,0 @@
from couchpotato.core.helpers.variable import splitString
from couchpotato.core.logger import CPLog
from couchpotato.core.notifications.base import Notification
from pynmwp import PyNMWP
import six
log = CPLog(__name__)
autoload = 'NotifyMyWP'
class NotifyMyWP(Notification):
def notify(self, message = '', data = None, listener = None):
if not data: data = {}
keys = splitString(self.conf('api_key'))
p = PyNMWP(keys, self.conf('dev_key'))
response = p.push(application = self.default_title, event = message, description = message, priority = self.conf('priority'), batch_mode = len(keys) > 1)
for key in keys:
if not response[key]['Code'] == six.u('200'):
log.error('Could not send notification to NotifyMyWindowsPhone (%s). %s', (key, response[key]['message']))
return False
return response
config = [{
'name': 'notifymywp',
'groups': [
{
'tab': 'notifications',
'list': 'notification_providers',
'name': 'notifymywp',
'label': 'Windows Phone',
'options': [
{
'name': 'enabled',
'default': 0,
'type': 'enabler',
},
{
'name': 'api_key',
'description': 'Multiple keys seperated by a comma. Maximum of 5.'
},
{
'name': 'dev_key',
'advanced': True,
},
{
'name': 'priority',
'default': 0,
'type': 'dropdown',
'values': [('Very Low', -2), ('Moderate', -1), ('Normal', 0), ('High', 1), ('Emergency', 2)],
},
{
'name': 'on_snatch',
'default': 0,
'type': 'bool',
'advanced': True,
'description': 'Also send message when movie is snatched.',
},
],
}
],
}]

3
couchpotato/core/notifications/pushbullet.py

@ -84,7 +84,8 @@ config = [{
}, },
{ {
'name': 'api_key', 'name': 'api_key',
'label': 'User API Key' 'label': 'Access Token',
'description': 'Can be found on <a href="https://www.pushbullet.com/account" target="_blank">Account Settings</a>',
}, },
{ {
'name': 'devices', 'name': 'devices',

6
couchpotato/core/notifications/pushover.py

@ -1,7 +1,7 @@
from httplib import HTTPSConnection from httplib import HTTPSConnection
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.variable import getTitle from couchpotato.core.helpers.variable import getTitle, getIdentifier
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.notifications.base import Notification from couchpotato.core.notifications.base import Notification
@ -27,9 +27,9 @@ class Pushover(Notification):
'sound': self.conf('sound'), 'sound': self.conf('sound'),
} }
if data and data.get('identifier'): if data and getIdentifier(data):
api_data.update({ api_data.update({
'url': toUnicode('http://www.imdb.com/title/%s/' % data['identifier']), 'url': toUnicode('http://www.imdb.com/title/%s/' % getIdentifier(data)),
'url_title': toUnicode('%s on IMDb' % getTitle(data)), 'url_title': toUnicode('%s on IMDb' % getTitle(data)),
}) })

7
couchpotato/core/notifications/trakt.py

@ -1,4 +1,4 @@
from couchpotato.core.helpers.variable import getTitle from couchpotato.core.helpers.variable import getTitle, getIdentifier
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.notifications.base import Notification from couchpotato.core.notifications.base import Notification
@ -16,7 +16,8 @@ class Trakt(Notification):
'test': 'account/test/%s', 'test': 'account/test/%s',
} }
listen_to = ['movie.downloaded'] listen_to = ['movie.snatched']
enabled_option = 'notification_enabled'
def notify(self, message = '', data = None, listener = None): def notify(self, message = '', data = None, listener = None):
if not data: data = {} if not data: data = {}
@ -38,7 +39,7 @@ class Trakt(Notification):
'username': self.conf('automation_username'), 'username': self.conf('automation_username'),
'password': self.conf('automation_password'), 'password': self.conf('automation_password'),
'movies': [{ 'movies': [{
'imdb_id': data['identifier'], 'imdb_id': getIdentifier(data),
'title': getTitle(data), 'title': getTitle(data),
'year': data['info']['year'] 'year': data['info']['year']
}] if data else [] }] if data else []

8
couchpotato/core/notifications/xbmc.py

@ -7,8 +7,8 @@ import urllib
from couchpotato.core.helpers.variable import splitString, getTitle from couchpotato.core.helpers.variable import splitString, getTitle
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.notifications.base import Notification from couchpotato.core.notifications.base import Notification
import requests from requests.exceptions import ConnectionError, Timeout
from requests.packages.urllib3.exceptions import MaxRetryError, ConnectionError from requests.packages.urllib3.exceptions import MaxRetryError
log = CPLog(__name__) log = CPLog(__name__)
@ -172,7 +172,7 @@ class XBMC(Notification):
# manually fake expected response array # manually fake expected response array
return [{'result': 'Error'}] return [{'result': 'Error'}]
except (MaxRetryError, requests.exceptions.Timeout, ConnectionError): except (MaxRetryError, Timeout, ConnectionError):
log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off') log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off')
return [{'result': 'Error'}] return [{'result': 'Error'}]
except: except:
@ -208,7 +208,7 @@ class XBMC(Notification):
log.debug('Returned from request %s: %s', (host, response)) log.debug('Returned from request %s: %s', (host, response))
return response return response
except (MaxRetryError, requests.exceptions.Timeout): except (MaxRetryError, Timeout, ConnectionError):
log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off') log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off')
return [] return []
except: except:

3
couchpotato/core/plugins/automation.py

@ -46,7 +46,8 @@ class Automation(Plugin):
break break
movie_dict = fireEvent('media.get', movie_id, single = True) movie_dict = fireEvent('media.get', movie_id, single = True)
fireEvent('movie.searcher.single', movie_dict) if movie_dict:
fireEvent('movie.searcher.single', movie_dict)
return True return True

121
couchpotato/core/plugins/base.py

@ -1,3 +1,4 @@
import threading
from urllib import quote from urllib import quote
from urlparse import urlparse from urlparse import urlparse
import glob import glob
@ -10,7 +11,8 @@ import traceback
from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import ss, toSafeString, \ from couchpotato.core.helpers.encoding import ss, toSafeString, \
toUnicode, sp toUnicode, sp
from couchpotato.core.helpers.variable import getExt, md5, isLocalIP, scanForPassword, tryInt, getIdentifier from couchpotato.core.helpers.variable import getExt, md5, isLocalIP, scanForPassword, tryInt, getIdentifier, \
randomString
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.environment import Env from couchpotato.environment import Env
import requests import requests
@ -35,6 +37,8 @@ class Plugin(object):
_needs_shutdown = False _needs_shutdown = False
_running = None _running = None
_locks = {}
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0' user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0'
http_last_use = {} http_last_use = {}
http_time_between_calls = 0 http_time_between_calls = 0
@ -118,15 +122,31 @@ class Plugin(object):
if os.path.exists(path): if os.path.exists(path):
log.debug('%s already exists, overwriting file with new version', path) log.debug('%s already exists, overwriting file with new version', path)
try: write_type = 'w+' if not binary else 'w+b'
f = open(path, 'w+' if not binary else 'w+b')
f.write(content) # Stream file using response object
f.close() if isinstance(content, requests.models.Response):
os.chmod(path, Env.getPermission('file'))
except: # Write file to temp
log.error('Unable writing to file "%s": %s', (path, traceback.format_exc())) with open('%s.tmp' % path, write_type) as f:
if os.path.isfile(path): for chunk in content.iter_content(chunk_size = 1048576):
os.remove(path) if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
# Rename to destination
os.rename('%s.tmp' % path, path)
else:
try:
f = open(path, write_type)
f.write(content)
f.close()
os.chmod(path, Env.getPermission('file'))
except:
log.error('Unable writing to file "%s": %s', (path, traceback.format_exc()))
if os.path.isfile(path):
os.remove(path)
def makeDir(self, path): def makeDir(self, path):
path = sp(path) path = sp(path)
@ -143,21 +163,17 @@ class Plugin(object):
folder = sp(folder) folder = sp(folder)
for item in os.listdir(folder): for item in os.listdir(folder):
full_folder = os.path.join(folder, item) full_folder = sp(os.path.join(folder, item))
if not only_clean or (item in only_clean and os.path.isdir(full_folder)): if not only_clean or (item in only_clean and os.path.isdir(full_folder)):
for root, dirs, files in os.walk(full_folder): for subfolder, dirs, files in os.walk(full_folder, topdown = False):
for dir_name in dirs:
full_path = os.path.join(root, dir_name)
if len(os.listdir(full_path)) == 0: try:
try: os.rmdir(subfolder)
os.rmdir(full_path) except:
except: if show_error:
if show_error: log.info2('Couldn\'t remove directory %s: %s', (subfolder, traceback.format_exc()))
log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
try: try:
os.rmdir(folder) os.rmdir(folder)
@ -166,7 +182,7 @@ class Plugin(object):
log.error('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc())) log.error('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc()))
# http request # http request
def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True): def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True, stream = False):
url = quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]") url = quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]")
if not headers: headers = {} if not headers: headers = {}
@ -177,10 +193,10 @@ class Plugin(object):
host = '%s%s' % (parsed_url.hostname, (':' + str(parsed_url.port) if parsed_url.port else '')) host = '%s%s' % (parsed_url.hostname, (':' + str(parsed_url.port) if parsed_url.port else ''))
headers['Referer'] = headers.get('Referer', '%s://%s' % (parsed_url.scheme, host)) headers['Referer'] = headers.get('Referer', '%s://%s' % (parsed_url.scheme, host))
headers['Host'] = headers.get('Host', host) headers['Host'] = headers.get('Host', None)
headers['User-Agent'] = headers.get('User-Agent', self.user_agent) headers['User-Agent'] = headers.get('User-Agent', self.user_agent)
headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip') headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip')
headers['Connection'] = headers.get('Connection', 'keep-alive') headers['Connection'] = headers.get('Connection', 'close')
headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0') headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0')
r = Env.get('http_opener') r = Env.get('http_opener')
@ -198,6 +214,7 @@ class Plugin(object):
del self.http_failed_disabled[host] del self.http_failed_disabled[host]
self.wait(host) self.wait(host)
status_code = None
try: try:
kwargs = { kwargs = {
@ -206,14 +223,16 @@ class Plugin(object):
'timeout': timeout, 'timeout': timeout,
'files': files, 'files': files,
'verify': False, #verify_ssl, Disable for now as to many wrongly implemented certificates.. 'verify': False, #verify_ssl, Disable for now as to many wrongly implemented certificates..
'stream': stream,
} }
method = 'post' if len(data) > 0 or files else 'get' method = 'post' if len(data) > 0 or files else 'get'
log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance(data, dict) else 'with data')) log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance(data, dict) else 'with data'))
response = r.request(method, url, **kwargs) response = r.request(method, url, **kwargs)
status_code = response.status_code
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:
data = response.content data = response if stream else response.content
else: else:
response.raise_for_status() response.raise_for_status()
@ -224,6 +243,12 @@ class Plugin(object):
# Save failed requests by hosts # Save failed requests by hosts
try: try:
# To many requests
if status_code in [429]:
self.http_failed_request[host] = 1
self.http_failed_disabled[host] = time.time()
if not self.http_failed_request.get(host): if not self.http_failed_request.get(host):
self.http_failed_request[host] = 1 self.http_failed_request[host] = 1
else: else:
@ -254,8 +279,8 @@ class Plugin(object):
wait = (last_use - now) + self.http_time_between_calls wait = (last_use - now) + self.http_time_between_calls
if wait > 0: if wait > 0:
log.debug('Waiting for %s, %d seconds', (self.getName(), wait)) log.debug('Waiting for %s, %d seconds', (self.getName(), max(1, wait)))
time.sleep(wait) time.sleep(min(wait, 30))
def beforeCall(self, handler): def beforeCall(self, handler):
self.isRunning('%s.%s' % (self.getName(), handler.__name__)) self.isRunning('%s.%s' % (self.getName(), handler.__name__))
@ -322,9 +347,9 @@ class Plugin(object):
Env.get('cache').set(cache_key_md5, value, timeout) Env.get('cache').set(cache_key_md5, value, timeout)
return value return value
def createNzbName(self, data, media): def createNzbName(self, data, media, unique_tag = False):
release_name = data.get('name') release_name = data.get('name')
tag = self.cpTag(media) tag = self.cpTag(media, unique_tag = unique_tag)
# Check if password is filename # Check if password is filename
name_password = scanForPassword(data.get('name')) name_password = scanForPassword(data.get('name'))
@ -337,18 +362,26 @@ class Plugin(object):
max_length = 127 - len(tag) # Some filesystems don't support 128+ long filenames max_length = 127 - len(tag) # Some filesystems don't support 128+ long filenames
return '%s%s' % (toSafeString(toUnicode(release_name)[:max_length]), tag) return '%s%s' % (toSafeString(toUnicode(release_name)[:max_length]), tag)
def createFileName(self, data, filedata, media): def createFileName(self, data, filedata, media, unique_tag = False):
name = self.createNzbName(data, media) name = self.createNzbName(data, media, unique_tag = unique_tag)
if data.get('protocol') == 'nzb' and 'DOCTYPE nzb' not in filedata and '</nzb>' not in filedata: if data.get('protocol') == 'nzb' and 'DOCTYPE nzb' not in filedata and '</nzb>' not in filedata:
return '%s.%s' % (name, 'rar') return '%s.%s' % (name, 'rar')
return '%s.%s' % (name, data.get('protocol')) return '%s.%s' % (name, data.get('protocol'))
def cpTag(self, media): def cpTag(self, media, unique_tag = False):
if Env.setting('enabled', 'renamer'):
identifier = getIdentifier(media) tag = ''
return '.cp(' + identifier + ')' if identifier else '' if Env.setting('enabled', 'renamer') or unique_tag:
identifier = getIdentifier(media) or ''
unique_tag = ', ' + randomString() if unique_tag else ''
tag = '.cp('
tag += identifier
tag += ', ' if unique_tag and identifier else ''
tag += randomString() if unique_tag else ''
tag += ')'
return '' return tag if len(tag) > 7 else ''
def checkFilesChanged(self, files, unchanged_for = 60): def checkFilesChanged(self, files, unchanged_for = 60):
now = time.time() now = time.time()
@ -393,3 +426,19 @@ class Plugin(object):
def isEnabled(self): def isEnabled(self):
return self.conf(self.enabled_option) or self.conf(self.enabled_option) is None return self.conf(self.enabled_option) or self.conf(self.enabled_option) is None
def acquireLock(self, key):
lock = self._locks.get(key)
if not lock:
self._locks[key] = threading.RLock()
log.debug('Acquiring lock: %s', key)
self._locks.get(key).acquire()
def releaseLock(self, key):
lock = self._locks.get(key)
if lock:
log.debug('Releasing lock: %s', key)
self._locks.get(key).release()

29
couchpotato/core/plugins/browser.py

@ -1,12 +1,18 @@
import ctypes import ctypes
import os import os
import string import string
import traceback
import time
from couchpotato import CPLog
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.helpers.encoding import sp from couchpotato.core.event import addEvent
from couchpotato.core.helpers.encoding import sp, ss, toUnicode
from couchpotato.core.helpers.variable import getUserDir from couchpotato.core.helpers.variable import getUserDir
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
import six
log = CPLog(__name__)
if os.name == 'nt': if os.name == 'nt':
@ -53,9 +59,9 @@ class FileBrowser(Plugin):
dirs = [] dirs = []
path = sp(path) path = sp(path)
for f in os.listdir(path): for f in os.listdir(path):
p = os.path.join(path, f) p = sp(os.path.join(path, f))
if os.path.isdir(p) and ((self.is_hidden(p) and bool(int(show_hidden))) or not self.is_hidden(p)): if os.path.isdir(p) and ((self.is_hidden(p) and bool(int(show_hidden))) or not self.is_hidden(p)):
dirs.append(p + os.path.sep) dirs.append(toUnicode('%s%s' % (p, os.path.sep)))
return sorted(dirs) return sorted(dirs)
@ -66,8 +72,8 @@ class FileBrowser(Plugin):
driveletters = [] driveletters = []
for drive in string.ascii_uppercase: for drive in string.ascii_uppercase:
if win32file.GetDriveType(drive + ":") in [win32file.DRIVE_FIXED, win32file.DRIVE_REMOTE, win32file.DRIVE_RAMDISK, win32file.DRIVE_REMOVABLE]: if win32file.GetDriveType(drive + ':') in [win32file.DRIVE_FIXED, win32file.DRIVE_REMOTE, win32file.DRIVE_RAMDISK, win32file.DRIVE_REMOVABLE]:
driveletters.append(drive + ":\\") driveletters.append(drive + ':\\')
return driveletters return driveletters
@ -100,14 +106,19 @@ class FileBrowser(Plugin):
def is_hidden(self, filepath): def is_hidden(self, filepath):
name = os.path.basename(os.path.abspath(filepath)) name = ss(os.path.basename(os.path.abspath(filepath)))
return name.startswith('.') or self.has_hidden_attribute(filepath) return name.startswith('.') or self.has_hidden_attribute(filepath)
def has_hidden_attribute(self, filepath): def has_hidden_attribute(self, filepath):
result = False
try: try:
attrs = ctypes.windll.kernel32.GetFileAttributesW(six.text_type(filepath)) #@UndefinedVariable attrs = ctypes.windll.kernel32.GetFileAttributesW(sp(filepath)) #@UndefinedVariable
assert attrs != -1 assert attrs != -1
result = bool(attrs & 2) result = bool(attrs & 2)
except (AttributeError, AssertionError): except (AttributeError, AssertionError):
result = False pass
except:
log.error('Failed getting hidden attribute: %s', traceback.format_exc())
return result return result

2
couchpotato/core/plugins/category/main.py

@ -27,7 +27,7 @@ class CategoryPlugin(Plugin):
'desc': 'List all available categories', 'desc': 'List all available categories',
'return': {'type': 'object', 'example': """{ 'return': {'type': 'object', 'example': """{
'success': True, 'success': True,
'list': array, categories 'categories': array, categories
}"""} }"""}
}) })

24
couchpotato/core/plugins/dashboard.py

@ -1,6 +1,6 @@
from datetime import date
import random as rndm import random as rndm
import time import time
from CodernityDB.database import RecordDeleted
from couchpotato import get_db from couchpotato import get_db
from couchpotato.api import addApiView from couchpotato.api import addApiView
@ -48,7 +48,6 @@ class Dashboard(Plugin):
active_ids = [x['_id'] for x in fireEvent('media.with_status', 'active', with_doc = False, single = True)] active_ids = [x['_id'] for x in fireEvent('media.with_status', 'active', with_doc = False, single = True)]
medias = [] medias = []
now_year = date.today().year
if len(active_ids) > 0: if len(active_ids) > 0:
@ -60,9 +59,13 @@ class Dashboard(Plugin):
rndm.shuffle(active_ids) rndm.shuffle(active_ids)
for media_id in active_ids: for media_id in active_ids:
media = db.get('id', media_id) try:
media = db.get('id', media_id)
except RecordDeleted:
log.debug('Record already deleted: %s', media_id)
continue
pp = profile_pre.get(media['profile_id']) pp = profile_pre.get(media.get('profile_id'))
if not pp: continue if not pp: continue
eta = media['info'].get('release_date', {}) or {} eta = media['info'].get('release_date', {}) or {}
@ -70,22 +73,25 @@ class Dashboard(Plugin):
# Theater quality # Theater quality
if pp.get('theater') and fireEvent('movie.searcher.could_be_released', True, eta, media['info']['year'], single = True): if pp.get('theater') and fireEvent('movie.searcher.could_be_released', True, eta, media['info']['year'], single = True):
coming_soon = True coming_soon = 'theater'
elif pp.get('dvd') and fireEvent('movie.searcher.could_be_released', False, eta, media['info']['year'], single = True): elif pp.get('dvd') and fireEvent('movie.searcher.could_be_released', False, eta, media['info']['year'], single = True):
coming_soon = True coming_soon = 'dvd'
if coming_soon: if coming_soon:
# Don't list older movies # Don't list older movies
if ((not late and (media['info']['year'] >= now_year - 1) and (not eta.get('dvd') and not eta.get('theater') or eta.get('dvd') and eta.get('dvd') > (now - 2419200))) or eta_date = eta.get(coming_soon)
(late and (media['info']['year'] < now_year - 1 or (eta.get('dvd', 0) > 0 or eta.get('theater')) and eta.get('dvd') < (now - 2419200)))): eta_3month_passed = eta_date < (now - 7862400) # Release was more than 3 months ago
if (not late and not eta_3month_passed) or \
(late and eta_3month_passed):
add = True add = True
# Check if it doesn't have any releases # Check if it doesn't have any releases
if late: if late:
media['releases'] = fireEvent('release.for_media', media['_id'], single = True) media['releases'] = fireEvent('release.for_media', media['_id'], single = True)
for release in media.get('releases'): for release in media.get('releases'):
if release.get('status') in ['snatched', 'available', 'seeding', 'downloaded']: if release.get('status') in ['snatched', 'available', 'seeding', 'downloaded']:
add = False add = False

13
couchpotato/core/plugins/file.py

@ -4,7 +4,7 @@ import traceback
from couchpotato import get_db from couchpotato import get_db
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.encoding import toUnicode, ss, sp
from couchpotato.core.helpers.variable import md5, getExt, isSubFolder from couchpotato.core.helpers.variable import md5, getExt, isSubFolder
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
@ -59,13 +59,18 @@ class FileManager(Plugin):
log.error('Failed removing unused file: %s', traceback.format_exc()) log.error('Failed removing unused file: %s', traceback.format_exc())
def showCacheFile(self, route, **kwargs): def showCacheFile(self, route, **kwargs):
Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': Env.get('cache_dir')})]) Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': toUnicode(Env.get('cache_dir'))})])
def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None): def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None):
if not urlopen_kwargs: urlopen_kwargs = {} if not urlopen_kwargs: urlopen_kwargs = {}
# Return response object to stream download
urlopen_kwargs['stream'] = True
if not dest: # to Cache if not dest: # to Cache
dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) dest = os.path.join(Env.get('cache_dir'), ss('%s.%s' % (md5(url), getExt(url))))
dest = sp(dest)
if not overwrite and os.path.isfile(dest): if not overwrite and os.path.isfile(dest):
return dest return dest
@ -107,4 +112,4 @@ class FileManager(Plugin):
else: else:
log.info('Subfolder test succeeded') log.info('Subfolder test succeeded')
return failed == 0 return failed == 0

2
couchpotato/core/plugins/log/static/log.js

@ -241,7 +241,7 @@ Running on: ...\n\
'href': 'https://github.com/RuudBurger/CouchPotatoServer/blob/develop/contributing.md' 'href': 'https://github.com/RuudBurger/CouchPotatoServer/blob/develop/contributing.md'
}), }),
new Element('span', { new Element('span', {
'text': ' before posting, then copy the text below' 'html': ' before posting, then copy the text below and <strong>FILL IN</strong> the dots.'
}) })
), ),
textarea = new Element('textarea', { textarea = new Element('textarea', {

10
couchpotato/core/plugins/manage.py

@ -123,7 +123,7 @@ class Manage(Plugin):
fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder) fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder)
onFound = self.createAddToLibrary(folder, added_identifiers) onFound = self.createAddToLibrary(folder, added_identifiers)
fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True) fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, check_file_date = False, on_found = onFound, single = True)
# Break if CP wants to shut down # Break if CP wants to shut down
if self.shuttingDown(): if self.shuttingDown():
@ -165,7 +165,7 @@ class Manage(Plugin):
already_used = used_files.get(release_file) already_used = used_files.get(release_file)
if already_used: if already_used:
release_id = release['_id'] if already_used.get('last_edit', 0) < release.get('last_edit', 0) else already_used['_id'] release_id = release['_id'] if already_used.get('last_edit', 0) > release.get('last_edit', 0) else already_used['_id']
if release_id not in deleted_releases: if release_id not in deleted_releases:
fireEvent('release.delete', release_id, single = True) fireEvent('release.delete', release_id, single = True)
deleted_releases.append(release_id) deleted_releases.append(release_id)
@ -190,6 +190,7 @@ class Manage(Plugin):
delete_me = {} delete_me = {}
# noinspection PyTypeChecker
for folder in self.in_progress: for folder in self.in_progress:
if self.in_progress[folder]['to_go'] <= 0: if self.in_progress[folder]['to_go'] <= 0:
delete_me[folder] = True delete_me[folder] = True
@ -219,7 +220,7 @@ class Manage(Plugin):
# Add it to release and update the info # Add it to release and update the info
fireEvent('release.add', group = group, update_info = False) fireEvent('release.add', group = group, update_info = False)
fireEvent('movie.update_info', identifier = group['identifier'], on_complete = self.createAfterUpdate(folder, group['identifier'])) fireEvent('movie.update', identifier = group['identifier'], on_complete = self.createAfterUpdate(folder, group['identifier']))
return addToLibrary return addToLibrary
@ -233,7 +234,8 @@ class Manage(Plugin):
total = self.in_progress[folder]['total'] total = self.in_progress[folder]['total']
movie_dict = fireEvent('media.get', identifier, single = True) movie_dict = fireEvent('media.get', identifier, single = True)
fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict)) if movie_dict:
fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict))
return afterUpdate return afterUpdate

2
couchpotato/core/plugins/profile/main.py

@ -86,6 +86,7 @@ class ProfilePlugin(Plugin):
'label': toUnicode(kwargs.get('label')), 'label': toUnicode(kwargs.get('label')),
'order': tryInt(kwargs.get('order', 999)), 'order': tryInt(kwargs.get('order', 999)),
'core': kwargs.get('core', False), 'core': kwargs.get('core', False),
'minimum_score': tryInt(kwargs.get('minimum_score', 1)),
'qualities': [], 'qualities': [],
'wait_for': [], 'wait_for': [],
'stop_after': [], 'stop_after': [],
@ -217,6 +218,7 @@ class ProfilePlugin(Plugin):
'label': toUnicode(profile.get('label')), 'label': toUnicode(profile.get('label')),
'order': order, 'order': order,
'qualities': profile.get('qualities'), 'qualities': profile.get('qualities'),
'minimum_score': 1,
'finish': [], 'finish': [],
'wait_for': [], 'wait_for': [],
'stop_after': [], 'stop_after': [],

5
couchpotato/core/plugins/profile/static/profile.css

@ -51,6 +51,11 @@
margin: 0 5px !important; margin: 0 5px !important;
} }
.profile .wait_for .minimum_score_input {
width: 40px !important;
text-align: left;
}
.profile .types { .profile .types {
padding: 0; padding: 0;
margin: 0 20px 0 -4px; margin: 0 20px 0 -4px;

12
couchpotato/core/plugins/profile/static/profile.js

@ -53,12 +53,21 @@ var Profile = new Class({
}), }),
new Element('span', {'text':'day(s) for a better quality '}), new Element('span', {'text':'day(s) for a better quality '}),
new Element('span.advanced', {'text':'and keep searching'}), new Element('span.advanced', {'text':'and keep searching'}),
// "After a checked quality is found and downloaded, continue searching for even better quality releases for the entered number of days." // "After a checked quality is found and downloaded, continue searching for even better quality releases for the entered number of days."
new Element('input.inlay.xsmall.stop_after_input.advanced', { new Element('input.inlay.xsmall.stop_after_input.advanced', {
'type':'text', 'type':'text',
'value': data.stop_after && data.stop_after.length > 0 ? data.stop_after[0] : 0 'value': data.stop_after && data.stop_after.length > 0 ? data.stop_after[0] : 0
}), }),
new Element('span.advanced', {'text':'day(s) for a better (checked) quality.'}) new Element('span.advanced', {'text':'day(s) for a better (checked) quality.'}),
// Minimum score of
new Element('span.advanced', {'html':'<br/>Releases need a minimum score of'}),
new Element('input.advanced.inlay.xsmall.minimum_score_input', {
'size': 4,
'type':'text',
'value': data.minimum_score || 1
})
) )
); );
@ -126,6 +135,7 @@ var Profile = new Class({
'label' : self.el.getElement('.quality_label input').get('value'), 'label' : self.el.getElement('.quality_label input').get('value'),
'wait_for' : self.el.getElement('.wait_for_input').get('value'), 'wait_for' : self.el.getElement('.wait_for_input').get('value'),
'stop_after' : self.el.getElement('.stop_after_input').get('value'), 'stop_after' : self.el.getElement('.stop_after_input').get('value'),
'minimum_score' : self.el.getElement('.minimum_score_input').get('value'),
'types': [] 'types': []
}; };

123
couchpotato/core/plugins/quality/main.py

@ -1,3 +1,4 @@
from math import fabs, ceil
import traceback import traceback
import re import re
@ -6,7 +7,7 @@ from couchpotato import get_db
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import toUnicode, ss from couchpotato.core.helpers.encoding import toUnicode, ss
from couchpotato.core.helpers.variable import mergeDicts, getExt, tryInt, splitString from couchpotato.core.helpers.variable import mergeDicts, getExt, tryInt, splitString, tryFloat
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
from couchpotato.core.plugins.quality.index import QualityIndex from couchpotato.core.plugins.quality.index import QualityIndex
@ -22,17 +23,17 @@ class QualityPlugin(Plugin):
} }
qualities = [ qualities = [
{'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']}, {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'median_size': 40000, 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']},
{'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']}, {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'median_size': 10000, 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']},
{'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'median_size': 5500, 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']},
{'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip'), 'hdtv', 'hdrip'], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['webdl', ('web', 'dl')]},
{'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']},
{'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
{'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr', 'webrip', ('web', 'rip')], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': []},
{'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]}, {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p', '1080p'], 'ext':[]},
{'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':[]}, {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p', '1080p'], 'ext':[]},
{'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':[]}, {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p', '1080p'], 'ext':[]},
{'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]} {'identifier': 'cam', 'size': (600, 1000), 'median_size': 700, 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p', '1080p'], 'ext':[]}
] ]
pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr']
threed_tags = { threed_tags = {
@ -187,14 +188,15 @@ class QualityPlugin(Plugin):
return False return False
def guess(self, files, extra = None, size = None): def guess(self, files, extra = None, size = None, use_cache = True):
if not extra: extra = {} if not extra: extra = {}
# Create hash for cache # Create hash for cache
cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files]) cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files])
cached = self.getCache(cache_key) if use_cache:
if cached and len(extra) == 0: cached = self.getCache(cache_key)
return cached if cached and len(extra) == 0:
return cached
qualities = self.all() qualities = self.all()
@ -206,6 +208,10 @@ class QualityPlugin(Plugin):
'3d': {} '3d': {}
} }
# Use metadata titles as extra check
if extra and extra.get('titles'):
files.extend(extra.get('titles'))
for cur_file in files: for cur_file in files:
words = re.split('\W+', cur_file.lower()) words = re.split('\W+', cur_file.lower())
name_year = fireEvent('scanner.name_year', cur_file, file_name = cur_file, single = True) name_year = fireEvent('scanner.name_year', cur_file, file_name = cur_file, single = True)
@ -218,7 +224,7 @@ class QualityPlugin(Plugin):
contains_score = self.containsTagScore(quality, words, cur_file) contains_score = self.containsTagScore(quality, words, cur_file)
threedscore = self.contains3D(quality, threed_words, cur_file) if quality.get('allow_3d') else (0, None) threedscore = self.contains3D(quality, threed_words, cur_file) if quality.get('allow_3d') else (0, None)
self.calcScore(score, quality, contains_score, threedscore) self.calcScore(score, quality, contains_score, threedscore, penalty = contains_score)
size_scores = [] size_scores = []
for quality in qualities: for quality in qualities:
@ -230,11 +236,11 @@ class QualityPlugin(Plugin):
if size_score > 0: if size_score > 0:
size_scores.append(quality) size_scores.append(quality)
self.calcScore(score, quality, size_score + loose_score, penalty = False) self.calcScore(score, quality, size_score + loose_score)
# Add additional size score if only 1 size validated # Add additional size score if only 1 size validated
if len(size_scores) == 1: if len(size_scores) == 1:
self.calcScore(score, size_scores[0], 10, penalty = False) self.calcScore(score, size_scores[0], 8)
del size_scores del size_scores
# Return nothing if all scores are <= 0 # Return nothing if all scores are <= 0
@ -259,19 +265,21 @@ class QualityPlugin(Plugin):
def containsTagScore(self, quality, words, cur_file = ''): def containsTagScore(self, quality, words, cur_file = ''):
cur_file = ss(cur_file) cur_file = ss(cur_file)
score = 0 score = 0.0
extension = words[-1] extension = words[-1]
words = words[:-1] words = words[:-1]
points = { points = {
'identifier': 10, 'identifier': 20,
'label': 10, 'label': 20,
'alternative': 9, 'alternative': 20,
'tags': 9, 'tags': 11,
'ext': 3, 'ext': 5,
} }
scored_on = []
# Check alt and tags # Check alt and tags
for tag_type in ['identifier', 'alternative', 'tags', 'label']: for tag_type in ['identifier', 'alternative', 'tags', 'label']:
qualities = quality.get(tag_type, []) qualities = quality.get(tag_type, [])
@ -283,13 +291,12 @@ class QualityPlugin(Plugin):
log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
score += points.get(tag_type) score += points.get(tag_type)
if isinstance(alt, (str, unicode)) and ss(alt.lower()) in words: if isinstance(alt, (str, unicode)) and ss(alt.lower()) in words and ss(alt.lower()) not in scored_on:
log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
score += points.get(tag_type) / 2 score += points.get(tag_type)
if list(set(qualities) & set(words)): # Don't score twice on same tag
log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) scored_on.append(ss(alt).lower())
score += points.get(tag_type)
# Check extention # Check extention
for ext in quality.get('ext', []): for ext in quality.get('ext', []):
@ -325,7 +332,7 @@ class QualityPlugin(Plugin):
# Check width resolution, range 20 # Check width resolution, range 20
if quality.get('width') and (quality.get('width') - 20) <= extra.get('resolution_width', 0) <= (quality.get('width') + 20): if quality.get('width') and (quality.get('width') - 20) <= extra.get('resolution_width', 0) <= (quality.get('width') + 20):
log.debug('Found %s via resolution_width: %s == %s', (quality['identifier'], quality.get('width'), extra.get('resolution_width', 0))) log.debug('Found %s via resolution_width: %s == %s', (quality['identifier'], quality.get('width'), extra.get('resolution_width', 0)))
score += 5 score += 10
# Check height resolution, range 20 # Check height resolution, range 20
if quality.get('height') and (quality.get('height') - 20) <= extra.get('resolution_height', 0) <= (quality.get('height') + 20): if quality.get('height') and (quality.get('height') - 20) <= extra.get('resolution_height', 0) <= (quality.get('height') + 20):
@ -345,15 +352,28 @@ class QualityPlugin(Plugin):
if size: if size:
if tryInt(quality['size_min']) <= tryInt(size) <= tryInt(quality['size_max']): size = tryFloat(size)
log.debug('Found %s via release size: %s MB < %s MB < %s MB', (quality['identifier'], quality['size_min'], size, quality['size_max'])) size_min = tryFloat(quality['size_min'])
score += 5 size_max = tryFloat(quality['size_max'])
if size_min <= size <= size_max:
log.debug('Found %s via release size: %s MB < %s MB < %s MB', (quality['identifier'], size_min, size, size_max))
proc_range = size_max - size_min
size_diff = size - size_min
size_proc = (size_diff / proc_range)
median_diff = quality['median_size'] - size_min
median_proc = (median_diff / proc_range)
max_points = 8
score += ceil(max_points - (fabs(size_proc - median_proc) * max_points))
else: else:
score -= 5 score -= 5
return score return score
def calcScore(self, score, quality, add_score, threedscore = (0, None), penalty = True): def calcScore(self, score, quality, add_score, threedscore = (0, None), penalty = 0):
score[quality['identifier']]['score'] += add_score score[quality['identifier']]['score'] += add_score
@ -372,11 +392,11 @@ class QualityPlugin(Plugin):
if penalty and add_score != 0: if penalty and add_score != 0:
for allow in quality.get('allow', []): for allow in quality.get('allow', []):
score[allow]['score'] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5 score[allow]['score'] -= ((penalty * 2) if self.cached_order[allow] < self.cached_order[quality['identifier']] else penalty) * 2
# Give panelty for all lower qualities # Give panelty for all other qualities
for q in self.qualities[self.order.index(quality.get('identifier'))+1:]: for q in self.qualities:
if score.get(q.get('identifier')): if quality.get('identifier') != q.get('identifier') and score.get(q.get('identifier')):
score[q.get('identifier')]['score'] -= 1 score[q.get('identifier')]['score'] -= 1
def isFinish(self, quality, profile, release_age = 0): def isFinish(self, quality, profile, release_age = 0):
@ -444,21 +464,38 @@ class QualityPlugin(Plugin):
'Movie Monuments 2013 BrRip 1080p': {'size': 1800, 'quality': 'brrip'}, 'Movie Monuments 2013 BrRip 1080p': {'size': 1800, 'quality': 'brrip'},
'Movie Monuments 2013 BrRip 720p': {'size': 1300, 'quality': 'brrip'}, 'Movie Monuments 2013 BrRip 720p': {'size': 1300, 'quality': 'brrip'},
'The.Movie.2014.3D.1080p.BluRay.AVC.DTS-HD.MA.5.1-GroupName': {'size': 30000, 'quality': 'bd50', 'is_3d': True}, 'The.Movie.2014.3D.1080p.BluRay.AVC.DTS-HD.MA.5.1-GroupName': {'size': 30000, 'quality': 'bd50', 'is_3d': True},
'/home/namehou/Movie Monuments (2013)/Movie Monuments.mkv': {'size': 4500, 'quality': '1080p', 'is_3d': False}, '/home/namehou/Movie Monuments (2012)/Movie Monuments.mkv': {'size': 5500, 'quality': '720p', 'is_3d': False},
'/home/namehou/Movie Monuments (2013)/Movie Monuments Full-OU.mkv': {'size': 4500, 'quality': '1080p', 'is_3d': True}, '/home/namehou/Movie Monuments (2012)/Movie Monuments Full-OU.mkv': {'size': 5500, 'quality': '720p', 'is_3d': True},
'/home/namehou/Movie Monuments (2013)/Movie Monuments.mkv': {'size': 10000, 'quality': '1080p', 'is_3d': False},
'/home/namehou/Movie Monuments (2013)/Movie Monuments Full-OU.mkv': {'size': 10000, 'quality': '1080p', 'is_3d': True},
'/volume1/Public/3D/Moviename/Moviename (2009).3D.SBS.ts': {'size': 7500, 'quality': '1080p', 'is_3d': True}, '/volume1/Public/3D/Moviename/Moviename (2009).3D.SBS.ts': {'size': 7500, 'quality': '1080p', 'is_3d': True},
'/volume1/Public/Moviename/Moviename (2009).ts': {'size': 5500, 'quality': '1080p'}, '/volume1/Public/Moviename/Moviename (2009).ts': {'size': 7500, 'quality': '1080p'},
'/movies/BluRay HDDVD H.264 MKV 720p EngSub/QuiQui le fou (criterion collection #123, 1915)/QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'}, '/movies/BluRay HDDVD H.264 MKV 720p EngSub/QuiQui le fou (criterion collection #123, 1915)/QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'},
'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'}, 'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'},
'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) half-sbs 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p', 'is_3d': True}, 'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) half-sbs 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p', 'is_3d': True},
'Moviename 2014 720p HDCAM XviD DualAudio': {'size': 4000, 'quality': 'cam'}, 'Moviename 2014 720p HDCAM XviD DualAudio': {'size': 4000, 'quality': 'cam'},
'Moviename (2014) - 720p CAM x264': {'size': 2250, 'quality': 'cam'}, 'Moviename (2014) - 720p CAM x264': {'size': 2250, 'quality': 'cam'},
'Movie Name (2014).mp4': {'size': 750, 'quality': 'brrip'}, 'Movie Name (2014).mp4': {'size': 750, 'quality': 'brrip'},
'Moviename.2014.720p.R6.WEB-DL.x264.AC3-xyz': {'size': 750, 'quality': 'r5'},
'Movie name 2014 New Source 720p HDCAM x264 AC3 xyz': {'size': 750, 'quality': 'cam'},
'Movie.Name.2014.720p.HD.TS.AC3.x264': {'size': 750, 'quality': 'ts'},
'Movie.Name.2014.1080p.HDrip.x264.aac-ReleaseGroup': {'size': 7000, 'quality': 'brrip'},
'Movie.Name.2014.HDCam.Chinese.Subs-ReleaseGroup': {'size': 15000, 'quality': 'cam'},
'Movie Name 2014 HQ DVDRip X264 AC3 (bla)': {'size': 0, 'quality': 'dvdrip'},
'Movie Name1 (2012).mkv': {'size': 4500, 'quality': '720p'},
'Movie Name (2013).mkv': {'size': 8500, 'quality': '1080p'},
'Movie Name (2014).mkv': {'size': 4500, 'quality': '720p', 'extra': {'titles': ['Movie Name 2014 720p Bluray']}},
'Movie Name (2015).mkv': {'size': 500, 'quality': '1080p', 'extra': {'resolution_width': 1920}},
'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'},
'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'},
'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'},
'Movie Name.2014.720p.WEBRip.x264.AC3-ReleaseGroup': {'size': 3000, 'quality': 'scr'},
'Movie.Name.2014.1080p.HDCAM.-.ReleaseGroup': {'size': 5300, 'quality': 'cam'},
} }
correct = 0 correct = 0
for name in tests: for name in tests:
test_quality = self.guess(files = [name], extra = tests[name].get('extra', None), size = tests[name].get('size', None)) or {} test_quality = self.guess(files = [name], extra = tests[name].get('extra', None), size = tests[name].get('size', None), use_cache = False) or {}
success = test_quality.get('identifier') == tests[name]['quality'] and test_quality.get('is_3d') == tests[name].get('is_3d', False) success = test_quality.get('identifier') == tests[name]['quality'] and test_quality.get('is_3d') == tests[name].get('is_3d', False)
if not success: if not success:
log.error('%s failed check, thinks it\'s "%s" expecting "%s"', (name, log.error('%s failed check, thinks it\'s "%s" expecting "%s"', (name,

69
couchpotato/core/plugins/release/main.py

@ -8,7 +8,7 @@ from couchpotato import md5, get_db
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import toUnicode, sp from couchpotato.core.helpers.encoding import toUnicode, sp
from couchpotato.core.helpers.variable import getTitle from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
from .index import ReleaseIndex, ReleaseStatusIndex, ReleaseIDIndex, ReleaseDownloadIndex from .index import ReleaseIndex, ReleaseStatusIndex, ReleaseIDIndex, ReleaseDownloadIndex
@ -65,43 +65,58 @@ class Release(Plugin):
log.debug('Removing releases from dashboard') log.debug('Removing releases from dashboard')
now = time.time() now = time.time()
week = 262080 week = 604800
db = get_db() db = get_db()
# Get (and remove) parentless releases # Get (and remove) parentless releases
releases = db.all('release', with_doc = True) releases = db.all('release', with_doc = False)
media_exist = [] media_exist = []
reindex = 0
for release in releases: for release in releases:
if release.get('key') in media_exist: if release.get('key') in media_exist:
continue continue
try: try:
try:
doc = db.get('id', release.get('_id'))
except RecordDeleted:
reindex += 1
continue
db.get('id', release.get('key')) db.get('id', release.get('key'))
media_exist.append(release.get('key')) media_exist.append(release.get('key'))
try: try:
if release['doc'].get('status') == 'ignore': if doc.get('status') == 'ignore':
release['doc']['status'] = 'ignored' doc['status'] = 'ignored'
db.update(release['doc']) db.update(doc)
except: except:
log.error('Failed fixing mis-status tag: %s', traceback.format_exc()) log.error('Failed fixing mis-status tag: %s', traceback.format_exc())
except ValueError:
fireEvent('database.delete_corrupted', release.get('key'), traceback_error = traceback.format_exc(0))
reindex += 1
except RecordDeleted: except RecordDeleted:
db.delete(release['doc']) db.delete(doc)
log.debug('Deleted orphaned release: %s', release['doc']) log.debug('Deleted orphaned release: %s', doc)
reindex += 1
except: except:
log.debug('Failed cleaning up orphaned releases: %s', traceback.format_exc()) log.debug('Failed cleaning up orphaned releases: %s', traceback.format_exc())
if reindex > 0:
db.reindex()
del media_exist del media_exist
# get movies last_edit more than a week ago # get movies last_edit more than a week ago
medias = fireEvent('media.with_status', 'done', single = True) medias = fireEvent('media.with_status', ['done', 'active'], single = True)
for media in medias: for media in medias:
if media.get('last_edit', 0) > (now - week): if media.get('last_edit', 0) > (now - week):
continue continue
for rel in fireEvent('release.for_media', media['_id'], single = True): for rel in self.forMedia(media['_id']):
# Remove all available releases # Remove all available releases
if rel['status'] in ['available']: if rel['status'] in ['available']:
@ -111,7 +126,8 @@ class Release(Plugin):
elif rel['status'] in ['snatched', 'downloaded']: elif rel['status'] in ['snatched', 'downloaded']:
self.updateStatus(rel['_id'], status = 'ignored') self.updateStatus(rel['_id'], status = 'ignored')
fireEvent('media.untag', media.get('_id'), 'recent', single = True) if 'recent' in media.get('tags', []):
fireEvent('media.untag', media.get('_id'), 'recent', single = True)
def add(self, group, update_info = True, update_id = None): def add(self, group, update_info = True, update_id = None):
@ -171,7 +187,7 @@ class Release(Plugin):
release['files'] = dict((k, [toUnicode(x) for x in v]) for k, v in group['files'].items() if v) release['files'] = dict((k, [toUnicode(x) for x in v]) for k, v in group['files'].items() if v)
db.update(release) db.update(release)
fireEvent('media.restatus', media['_id'], single = True) fireEvent('media.restatus', media['_id'], allowed_restatus = ['done'], single = True)
return True return True
except: except:
@ -234,8 +250,9 @@ class Release(Plugin):
db = get_db() db = get_db()
try: try:
rel = db.get('id', id, with_doc = True) if id:
self.updateStatus(id, 'available' if rel['status'] in ['ignored', 'failed'] else 'ignored') rel = db.get('id', id, with_doc = True)
self.updateStatus(id, 'available' if rel['status'] in ['ignored', 'failed'] else 'ignored')
return { return {
'success': True 'success': True
@ -324,10 +341,10 @@ class Release(Plugin):
rls['download_info'] = download_result rls['download_info'] = download_result
db.update(rls) db.update(rls)
log_movie = '%s (%s) in %s' % (getTitle(media), media['info']['year'], rls['quality']) log_movie = '%s (%s) in %s' % (getTitle(media), media['info'].get('year'), rls['quality'])
snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie)
log.info(snatch_message) log.info(snatch_message)
fireEvent('%s.snatched' % data['type'], message = snatch_message, data = rls) fireEvent('%s.snatched' % data['type'], message = snatch_message, data = media)
# Mark release as snatched # Mark release as snatched
if renamer_enabled: if renamer_enabled:
@ -363,22 +380,28 @@ class Release(Plugin):
wait_for = False wait_for = False
let_through = False let_through = False
filtered_results = [] filtered_results = []
minimum_seeders = tryInt(Env.setting('minimum_seeders', section = 'torrent', default = 1))
# If a single release comes through the "wait for", let through all # Filter out ignored and other releases we don't want
for rel in results: for rel in results:
if rel['status'] in ['ignored', 'failed']: if rel['status'] in ['ignored', 'failed']:
log.info('Ignored: %s', rel['name']) log.info('Ignored: %s', rel['name'])
continue continue
if rel['score'] <= 0: if rel['score'] < quality_custom.get('minimum_score'):
log.info('Ignored, score "%s" to low: %s', (rel['score'], rel['name'])) log.info('Ignored, score "%s" to low, need at least "%s": %s', (rel['score'], quality_custom.get('minimum_score'), rel['name']))
continue continue
if rel['size'] <= 50: if rel['size'] <= 50:
log.info('Ignored, size "%sMB" to low: %s', (rel['size'], rel['name'])) log.info('Ignored, size "%sMB" to low: %s', (rel['size'], rel['name']))
continue continue
if 'seeders' in rel and rel.get('seeders') < minimum_seeders:
log.info('Ignored, not enough seeders, has %s needs %s: %s', (rel.get('seeders'), minimum_seeders, rel['name']))
continue
# If a single release comes through the "wait for", let through all
rel['wait_for'] = False rel['wait_for'] = False
if quality_custom.get('index') != 0 and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0): if quality_custom.get('index') != 0 and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0):
rel['wait_for'] = True rel['wait_for'] = True
@ -521,11 +544,15 @@ class Release(Plugin):
def forMedia(self, media_id): def forMedia(self, media_id):
db = get_db() db = get_db()
raw_releases = list(db.get_many('release', media_id, with_doc = True)) raw_releases = db.get_many('release', media_id)
releases = [] releases = []
for r in raw_releases: for r in raw_releases:
releases.append(r['doc']) try:
doc = db.get('id', r.get('_id'))
releases.append(doc)
except RecordDeleted:
pass
releases = sorted(releases, key = lambda k: k.get('info', {}).get('score', 0), reverse = True) releases = sorted(releases, key = lambda k: k.get('info', {}).get('score', 0), reverse = True)

157
couchpotato/core/plugins/renamer.py

@ -10,7 +10,8 @@ from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import toUnicode, ss, sp from couchpotato.core.helpers.encoding import toUnicode, ss, sp
from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, getIdentifier getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, \
getIdentifier, randomString, getFreeSpace, getSize
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env from couchpotato.environment import Env
@ -123,11 +124,6 @@ class Renamer(Plugin):
no_process = [to_folder] no_process = [to_folder]
cat_list = fireEvent('category.all', single = True) or [] cat_list = fireEvent('category.all', single = True) or []
no_process.extend([item['destination'] for item in cat_list]) no_process.extend([item['destination'] for item in cat_list])
try:
if Env.setting('library', section = 'manage').strip():
no_process.extend([sp(manage_folder) for manage_folder in splitString(Env.setting('library', section = 'manage'), '::')])
except:
pass
# Check to see if the no_process folders are inside the "from" folder. # Check to see if the no_process folders are inside the "from" folder.
if not os.path.isdir(base_folder) or not os.path.isdir(to_folder): if not os.path.isdir(base_folder) or not os.path.isdir(to_folder):
@ -202,14 +198,18 @@ class Renamer(Plugin):
db = get_db() db = get_db()
# Extend the download info with info stored in the downloaded release # Extend the download info with info stored in the downloaded release
keep_original = self.moveTypeIsLinked()
is_torrent = False
if release_download: if release_download:
release_download = self.extendReleaseDownload(release_download) release_download = self.extendReleaseDownload(release_download)
is_torrent = self.downloadIsTorrent(release_download)
keep_original = True if is_torrent and self.conf('file_action') not in ['move'] else keep_original
# Unpack any archives # Unpack any archives
extr_files = None extr_files = None
if self.conf('unrar'): if self.conf('unrar'):
folder, media_folder, files, extr_files = self.extractFiles(folder = folder, media_folder = media_folder, files = files, folder, media_folder, files, extr_files = self.extractFiles(folder = folder, media_folder = media_folder, files = files,
cleanup = self.conf('cleanup') and not self.downloadIsTorrent(release_download)) cleanup = self.conf('cleanup') and not keep_original)
groups = fireEvent('scanner.scan', folder = folder if folder else base_folder, groups = fireEvent('scanner.scan', folder = folder if folder else base_folder,
files = files, release_download = release_download, return_ignored = False, single = True) or [] files = files, release_download = release_download, return_ignored = False, single = True) or []
@ -220,6 +220,12 @@ class Renamer(Plugin):
nfo_name = self.conf('nfo_name') nfo_name = self.conf('nfo_name')
separator = self.conf('separator') separator = self.conf('separator')
cd_keys = ['<cd>','<cd_nr>', '<original>']
if not any(x in folder_name for x in cd_keys) and not any(x in file_name for x in cd_keys):
log.error('Missing `cd` or `cd_nr` in the renamer. This will cause multi-file releases of being renamed to the same file. '
'Please add it in the renamer settings. Force adding it for now.')
file_name = '%s %s' % ('<cd>', file_name)
# Tag release folder as failed_rename in case no groups were found. This prevents check_snatched from removing the release from the downloader. # Tag release folder as failed_rename in case no groups were found. This prevents check_snatched from removing the release from the downloader.
if not groups and self.statusInfoComplete(release_download): if not groups and self.statusInfoComplete(release_download):
self.tagRelease(release_download = release_download, tag = 'failed_rename') self.tagRelease(release_download = release_download, tag = 'failed_rename')
@ -248,7 +254,7 @@ class Renamer(Plugin):
'profile_id': None 'profile_id': None
}, search_after = False, status = 'done', single = True) }, search_after = False, status = 'done', single = True)
else: else:
group['media'] = fireEvent('movie.update_info', media_id = group['media'].get('_id'), single = True) group['media'] = fireEvent('movie.update', media_id = group['media'].get('_id'), single = True)
if not group['media'] or not group['media'].get('_id'): if not group['media'] or not group['media'].get('_id'):
log.error('Could not rename, no library item to work with: %s', group_identifier) log.error('Could not rename, no library item to work with: %s', group_identifier)
@ -267,13 +273,14 @@ class Renamer(Plugin):
category_label = category['label'] category_label = category['label']
if category['destination'] and len(category['destination']) > 0 and category['destination'] != 'None': if category['destination'] and len(category['destination']) > 0 and category['destination'] != 'None':
destination = category['destination'] destination = sp(category['destination'])
log.debug('Setting category destination for "%s": %s' % (media_title, destination)) log.debug('Setting category destination for "%s": %s' % (media_title, destination))
else: else:
log.debug('No category destination found for "%s"' % media_title) log.debug('No category destination found for "%s"' % media_title)
except: except:
log.error('Failed getting category label: %s', traceback.format_exc()) log.error('Failed getting category label: %s', traceback.format_exc())
# Find subtitle for renaming # Find subtitle for renaming
group['before_rename'] = [] group['before_rename'] = []
fireEvent('renamer.before', group) fireEvent('renamer.before', group)
@ -326,7 +333,7 @@ class Renamer(Plugin):
if file_type is 'nfo' and not self.conf('rename_nfo'): if file_type is 'nfo' and not self.conf('rename_nfo'):
log.debug('Skipping, renaming of %s disabled', file_type) log.debug('Skipping, renaming of %s disabled', file_type)
for current_file in group['files'][file_type]: for current_file in group['files'][file_type]:
if self.conf('cleanup') and (not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)): if self.conf('cleanup') and (not keep_original or self.fileIsAdded(current_file, group)):
remove_files.append(current_file) remove_files.append(current_file)
continue continue
@ -345,6 +352,9 @@ class Renamer(Plugin):
replacements['original'] = os.path.splitext(os.path.basename(current_file))[0] replacements['original'] = os.path.splitext(os.path.basename(current_file))[0]
replacements['original_folder'] = fireEvent('scanner.remove_cptag', group['dirname'], single = True) replacements['original_folder'] = fireEvent('scanner.remove_cptag', group['dirname'], single = True)
if not replacements['original_folder'] or len(replacements['original_folder']) == 0:
replacements['original_folder'] = replacements['original']
# Extension # Extension
replacements['ext'] = getExt(current_file) replacements['ext'] = getExt(current_file)
@ -363,10 +373,6 @@ class Renamer(Plugin):
elif file_type is 'nfo': elif file_type is 'nfo':
final_file_name = self.doReplace(nfo_name, replacements, remove_multiple = True) final_file_name = self.doReplace(nfo_name, replacements, remove_multiple = True)
# Seperator replace
if separator:
final_file_name = final_file_name.replace(' ', separator)
# Move DVD files (no structure renaming) # Move DVD files (no structure renaming)
if group['is_dvd'] and file_type is 'movie': if group['is_dvd'] and file_type is 'movie':
found = False found = False
@ -523,18 +529,26 @@ class Renamer(Plugin):
# Mark media for dashboard # Mark media for dashboard
if mark_as_recent: if mark_as_recent:
fireEvent('media.tag', group['media'].get('_id'), 'recent', single = True) fireEvent('media.tag', group['media'].get('_id'), 'recent', update_edited = True, single = True)
# Remove leftover files # Remove leftover files
if not remove_leftovers: # Don't remove anything if not remove_leftovers: # Don't remove anything
break continue
log.debug('Removing leftover files') log.debug('Removing leftover files')
for current_file in group['files']['leftover']: for current_file in group['files']['leftover']:
if self.conf('cleanup') and not self.conf('move_leftover') and \ if self.conf('cleanup') and not self.conf('move_leftover') and \
(not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)): (not keep_original or self.fileIsAdded(current_file, group)):
remove_files.append(current_file) remove_files.append(current_file)
if self.conf('check_space'):
total_space, available_space = getFreeSpace(destination)
renaming_size = getSize(rename_files.keys())
if renaming_size > available_space:
log.error('Not enough space left, need %s MB but only %s MB available', (renaming_size, available_space))
self.tagRelease(group = group, tag = 'not_enough_space')
continue
# Remove files # Remove files
delete_folders = [] delete_folders = []
for src in remove_files: for src in remove_files:
@ -550,9 +564,9 @@ class Renamer(Plugin):
os.remove(src) os.remove(src)
parent_dir = os.path.dirname(src) parent_dir = os.path.dirname(src)
if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and \ if parent_dir not in delete_folders and os.path.isdir(parent_dir) and \
not isSubFolder(destination, parent_dir) and not isSubFolder(media_folder, parent_dir) and \ not isSubFolder(destination, parent_dir) and not isSubFolder(media_folder, parent_dir) and \
not isSubFolder(parent_dir, base_folder): isSubFolder(parent_dir, base_folder):
delete_folders.append(parent_dir) delete_folders.append(parent_dir)
@ -561,6 +575,7 @@ class Renamer(Plugin):
self.tagRelease(group = group, tag = 'failed_remove') self.tagRelease(group = group, tag = 'failed_remove')
# Delete leftover folder from older releases # Delete leftover folder from older releases
delete_folders = sorted(delete_folders, key = len, reverse = True)
for delete_folder in delete_folders: for delete_folder in delete_folders:
try: try:
self.deleteEmptyFolder(delete_folder, show_error = False) self.deleteEmptyFolder(delete_folder, show_error = False)
@ -573,13 +588,16 @@ class Renamer(Plugin):
for src in rename_files: for src in rename_files:
if rename_files[src]: if rename_files[src]:
dst = rename_files[src] dst = rename_files[src]
log.info('Renaming "%s" to "%s"', (src, dst))
if dst in group['renamed_files']:
log.error('File "%s" already renamed once, adding random string at the end to prevent data loss', dst)
dst = '%s.random-%s' % (dst, randomString())
# Create dir # Create dir
self.makeDir(os.path.dirname(dst)) self.makeDir(os.path.dirname(dst))
try: try:
self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(release_download) or self.fileIsAdded(src, group)) self.moveFile(src, dst, use_default = not is_torrent or self.fileIsAdded(src, group))
group['renamed_files'].append(dst) group['renamed_files'].append(dst)
except: except:
log.error('Failed renaming the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) log.error('Failed renaming the file "%s" : %s', (os.path.basename(src), traceback.format_exc()))
@ -595,7 +613,7 @@ class Renamer(Plugin):
self.untagRelease(group = group, tag = 'failed_rename') self.untagRelease(group = group, tag = 'failed_rename')
# Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent
if self.movieInFromFolder(media_folder) and self.downloadIsTorrent(release_download): if self.movieInFromFolder(media_folder) and keep_original:
self.tagRelease(group = group, tag = 'renamed_already') self.tagRelease(group = group, tag = 'renamed_already')
# Remove matching releases # Remove matching releases
@ -606,7 +624,7 @@ class Renamer(Plugin):
except: except:
log.error('Failed removing %s: %s', (release, traceback.format_exc())) log.error('Failed removing %s: %s', (release, traceback.format_exc()))
if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(release_download): if group['dirname'] and group['parentdir'] and not keep_original:
if media_folder: if media_folder:
# Delete the movie folder # Delete the movie folder
group_folder = media_folder group_folder = media_folder
@ -615,8 +633,9 @@ class Renamer(Plugin):
group_folder = sp(os.path.join(base_folder, os.path.relpath(group['parentdir'], base_folder).split(os.path.sep)[0])) group_folder = sp(os.path.join(base_folder, os.path.relpath(group['parentdir'], base_folder).split(os.path.sep)[0]))
try: try:
log.info('Deleting folder: %s', group_folder) if self.conf('cleanup') or self.conf('move_leftover'):
self.deleteEmptyFolder(group_folder) log.info('Deleting folder: %s', group_folder)
self.deleteEmptyFolder(group_folder)
except: except:
log.error('Failed removing %s: %s', (group_folder, traceback.format_exc())) log.error('Failed removing %s: %s', (group_folder, traceback.format_exc()))
@ -768,33 +787,49 @@ Remove it if you want it to be renamed (again, or at least let it try again)
return False return False
def moveFile(self, old, dest, forcemove = False): def moveFile(self, old, dest, use_default = False):
dest = sp(dest) dest = sp(dest)
try: try:
if forcemove or self.conf('file_action') not in ['copy', 'link']:
if os.path.exists(dest):
raise Exception('Destination "%s" already exists' % dest)
move_type = self.conf('file_action')
if use_default:
move_type = self.conf('default_file_action')
if move_type not in ['copy', 'link']:
try: try:
log.info('Moving "%s" to "%s"', (old, dest))
shutil.move(old, dest) shutil.move(old, dest)
except: except:
if os.path.exists(dest): exists = os.path.exists(dest)
if exists and os.path.getsize(old) == os.path.getsize(dest):
log.error('Successfully moved file "%s", but something went wrong: %s', (dest, traceback.format_exc())) log.error('Successfully moved file "%s", but something went wrong: %s', (dest, traceback.format_exc()))
os.unlink(old) os.unlink(old)
else: else:
# remove faultly copied file
if exists:
os.unlink(dest)
raise raise
elif self.conf('file_action') == 'copy': elif move_type == 'copy':
log.info('Copying "%s" to "%s"', (old, dest))
shutil.copy(old, dest) shutil.copy(old, dest)
elif self.conf('file_action') == 'link': else:
log.info('Linking "%s" to "%s"', (old, dest))
# First try to hardlink # First try to hardlink
try: try:
log.debug('Hardlinking file "%s" to "%s"...', (old, dest)) log.debug('Hardlinking file "%s" to "%s"...', (old, dest))
link(old, dest) link(old, dest)
except: except:
# Try to simlink next # Try to simlink next
log.debug('Couldn\'t hardlink file "%s" to "%s". Simlinking instead. Error: %s.', (old, dest, traceback.format_exc())) log.debug('Couldn\'t hardlink file "%s" to "%s". Symlinking instead. Error: %s.', (old, dest, traceback.format_exc()))
shutil.copy(old, dest) shutil.copy(old, dest)
try: try:
symlink(dest, old + '.link') old_link = '%s.link' % sp(old)
symlink(dest, old_link)
os.unlink(old) os.unlink(old)
os.rename(old + '.link', old) os.rename(old_link, old)
except: except:
log.error('Couldn\'t symlink file "%s" to "%s". Copied instead. Error: %s. ', (old, dest, traceback.format_exc())) log.error('Couldn\'t symlink file "%s" to "%s". Copied instead. Error: %s. ', (old, dest, traceback.format_exc()))
@ -803,7 +838,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
if os.name == 'nt' and self.conf('ntfs_permission'): if os.name == 'nt' and self.conf('ntfs_permission'):
os.popen('icacls "' + dest + '"* /reset /T') os.popen('icacls "' + dest + '"* /reset /T')
except: except:
log.error('Failed setting permissions for file: %s, %s', (dest, traceback.format_exc(1))) log.debug('Failed setting permissions for file: %s, %s', (dest, traceback.format_exc(1)))
except: except:
log.error('Couldn\'t move file "%s" to "%s": %s', (old, dest, traceback.format_exc())) log.error('Couldn\'t move file "%s" to "%s": %s', (old, dest, traceback.format_exc()))
raise raise
@ -837,7 +872,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
replaced = re.sub(r"[\x00:\*\?\"<>\|]", '', replaced) replaced = re.sub(r"[\x00:\*\?\"<>\|]", '', replaced)
sep = self.conf('foldersep') if folder else self.conf('separator') sep = self.conf('foldersep') if folder else self.conf('separator')
return replaced.replace(' ', ' ' if not sep else sep) return ss(replaced.replace(' ', ' ' if not sep else sep))
def replaceDoubles(self, string): def replaceDoubles(self, string):
@ -850,6 +885,8 @@ Remove it if you want it to be renamed (again, or at least let it try again)
reg, replace_with = r reg, replace_with = r
string = re.sub(reg, replace_with, string) string = re.sub(reg, replace_with, string)
string = string.rstrip(',_-/\\ ')
return string return string
def checkSnatched(self, fire_scan = True): def checkSnatched(self, fire_scan = True):
@ -1089,6 +1126,9 @@ Remove it if you want it to be renamed (again, or at least let it try again)
return False return False
return src in group['before_rename'] return src in group['before_rename']
def moveTypeIsLinked(self):
return self.conf('default_file_action') in ['copy', 'link']
def statusInfoComplete(self, release_download): def statusInfoComplete(self, release_download):
return release_download.get('id') and release_download.get('downloader') and release_download.get('folder') return release_download.get('id') and release_download.get('downloader') and release_download.get('folder')
@ -1140,14 +1180,20 @@ Remove it if you want it to be renamed (again, or at least let it try again)
log.info('Archive %s found. Extracting...', os.path.basename(archive['file'])) log.info('Archive %s found. Extracting...', os.path.basename(archive['file']))
try: try:
rar_handle = RarFile(archive['file']) rar_handle = RarFile(archive['file'], custom_path = self.conf('unrar_path'))
extr_path = os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder)) extr_path = os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder))
self.makeDir(extr_path) self.makeDir(extr_path)
for packedinfo in rar_handle.infolist(): for packedinfo in rar_handle.infolist():
if not packedinfo.isdir and not os.path.isfile(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))): extr_file_path = sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))
if not packedinfo.isdir and not os.path.isfile(extr_file_path):
log.debug('Extracting %s...', packedinfo.filename) log.debug('Extracting %s...', packedinfo.filename)
rar_handle.extract(condition = [packedinfo.index], path = extr_path, withSubpath = False, overwrite = False) rar_handle.extract(condition = [packedinfo.index], path = extr_path, withSubpath = False, overwrite = False)
extr_files.append(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))) if self.conf('unrar_modify_date'):
try:
os.utime(extr_file_path, (os.path.getatime(archive['file']), os.path.getmtime(archive['file'])))
except:
log.error('Rar modify date enabled, but failed: %s', traceback.format_exc())
extr_files.append(extr_file_path)
del rar_handle del rar_handle
except Exception as e: except Exception as e:
log.error('Failed to extract %s: %s %s', (archive['file'], e, traceback.format_exc())) log.error('Failed to extract %s: %s %s', (archive['file'], e, traceback.format_exc()))
@ -1174,7 +1220,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
except Exception as e: except Exception as e:
log.error('Failed moving left over file %s to %s: %s %s', (leftoverfile, move_to, e, traceback.format_exc())) log.error('Failed moving left over file %s to %s: %s %s', (leftoverfile, move_to, e, traceback.format_exc()))
# As we probably tried to overwrite the nfo file, check if it exists and then remove the original # As we probably tried to overwrite the nfo file, check if it exists and then remove the original
if os.path.isfile(move_to): if os.path.isfile(move_to) and os.path.getsize(leftoverfile) == os.path.getsize(move_to):
if cleanup: if cleanup:
log.info('Deleting left over file %s instead...', leftoverfile) log.info('Deleting left over file %s instead...', leftoverfile)
os.unlink(leftoverfile) os.unlink(leftoverfile)
@ -1283,6 +1329,18 @@ config = [{
'default': False, 'default': False,
}, },
{ {
'advanced': True,
'name': 'unrar_path',
'description': 'Custom path to unrar bin',
},
{
'advanced': True,
'name': 'unrar_modify_date',
'type': 'bool',
'description': ('Set modify date of unrar-ed files to the rar-file\'s date.', 'This will allow XBMC to recognize extracted files as recently added even if the movie was released some time ago.'),
'default': False,
},
{
'name': 'cleanup', 'name': 'cleanup',
'type': 'bool', 'type': 'bool',
'description': 'Cleanup leftover files after successful rename.', 'description': 'Cleanup leftover files after successful rename.',
@ -1333,13 +1391,30 @@ config = [{
'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'), 'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'),
}, },
{ {
'name': 'check_space',
'label': 'Check space',
'default': True,
'type': 'bool',
'description': ('Check if there\'s enough available space to rename the files', 'Disable when the filesystem doesn\'t return the proper value'),
'advanced': True,
},
{
'name': 'default_file_action',
'label': 'Default File Action',
'default': 'move',
'type': 'dropdown',
'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')],
'description': ('<strong>Link</strong>, <strong>Copy</strong> or <strong>Move</strong> after download completed.',
'Link first tries <a href="http://en.wikipedia.org/wiki/Hard_link">hard link</a>, then <a href="http://en.wikipedia.org/wiki/Sym_link">sym link</a> and falls back to Copy.'),
'advanced': True,
},
{
'name': 'file_action', 'name': 'file_action',
'label': 'Torrent File Action', 'label': 'Torrent File Action',
'default': 'link', 'default': 'link',
'type': 'dropdown', 'type': 'dropdown',
'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')], 'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')],
'description': ('<strong>Link</strong>, <strong>Copy</strong> or <strong>Move</strong> after download completed.', 'description': 'See above. It is prefered to use link when downloading torrents as it will save you space, while still beeing able to seed.',
'Link first tries <a href="http://en.wikipedia.org/wiki/Hard_link">hard link</a>, then <a href="http://en.wikipedia.org/wiki/Sym_link">sym link</a> and falls back to Copy. It is perfered to use link when downloading torrents as it will save you space, while still beeing able to seed.'),
'advanced': True, 'advanced': True,
}, },
{ {

34
couchpotato/core/plugins/scanner.py

@ -11,7 +11,6 @@ from couchpotato.core.helpers.variable import getExt, getImdb, tryInt, \
splitString, getIdentifier splitString, getIdentifier
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
from enzyme.exceptions import NoParserError, ParseError
from guessit import guess_movie_info from guessit import guess_movie_info
from subliminal.videos import Video from subliminal.videos import Video
import enzyme import enzyme
@ -121,7 +120,7 @@ class Scanner(Plugin):
'()([ab])(\.....?)$' #*a.mkv '()([ab])(\.....?)$' #*a.mkv
] ]
cp_imdb = '(.cp.(?P<id>tt[0-9{7}]+).)' cp_imdb = '\.cp\((?P<id>tt[0-9]+),?\s?(?P<random>[A-Za-z0-9]+)?\)'
def __init__(self): def __init__(self):
@ -132,7 +131,7 @@ class Scanner(Plugin):
addEvent('scanner.name_year', self.getReleaseNameYear) addEvent('scanner.name_year', self.getReleaseNameYear)
addEvent('scanner.partnumber', self.getPartNumber) addEvent('scanner.partnumber', self.getPartNumber)
def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, on_found = None): def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, check_file_date = True, on_found = None):
folder = sp(folder) folder = sp(folder)
@ -146,7 +145,6 @@ class Scanner(Plugin):
# Scan all files of the folder if no files are set # Scan all files of the folder if no files are set
if not files: if not files:
check_file_date = True
try: try:
files = [] files = []
for root, dirs, walk_files in os.walk(folder, followlinks=True): for root, dirs, walk_files in os.walk(folder, followlinks=True):
@ -457,6 +455,7 @@ class Scanner(Plugin):
meta = self.getMeta(cur_file) meta = self.getMeta(cur_file)
try: try:
data['titles'] = meta.get('titles', [])
data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video'])) data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video']))
data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio'])) data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio']))
data['audio_channels'] = meta.get('audio_channels', 2.0) data['audio_channels'] = meta.get('audio_channels', 2.0)
@ -492,7 +491,7 @@ class Scanner(Plugin):
data['quality_type'] = 'HD' if data.get('resolution_width', 0) >= 1280 or data['quality'].get('hd') else 'SD' data['quality_type'] = 'HD' if data.get('resolution_width', 0) >= 1280 or data['quality'].get('hd') else 'SD'
filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', files[0]) filename = re.sub(self.cp_imdb, '', files[0])
data['group'] = self.getGroup(filename[len(folder):]) data['group'] = self.getGroup(filename[len(folder):])
data['source'] = self.getSourceMedia(filename) data['source'] = self.getSourceMedia(filename)
if data['quality'].get('is_3d', 0): if data['quality'].get('is_3d', 0):
@ -527,16 +526,33 @@ class Scanner(Plugin):
try: ac = self.audio_codec_map.get(p.audio[0].codec) try: ac = self.audio_codec_map.get(p.audio[0].codec)
except: pass except: pass
# Find title in video headers
titles = []
try:
if p.title and self.findYear(p.title):
titles.append(ss(p.title))
except:
log.error('Failed getting title from meta: %s', traceback.format_exc())
for video in p.video:
try:
if video.title and self.findYear(video.title):
titles.append(ss(video.title))
except:
log.error('Failed getting title from meta: %s', traceback.format_exc())
return { return {
'titles': list(set(titles)),
'video': vc, 'video': vc,
'audio': ac, 'audio': ac,
'resolution_width': tryInt(p.video[0].width), 'resolution_width': tryInt(p.video[0].width),
'resolution_height': tryInt(p.video[0].height), 'resolution_height': tryInt(p.video[0].height),
'audio_channels': p.audio[0].channels, 'audio_channels': p.audio[0].channels,
} }
except ParseError: except enzyme.exceptions.ParseError:
log.debug('Failed to parse meta for %s', filename) log.debug('Failed to parse meta for %s', filename)
except NoParserError: except enzyme.exceptions.NoParserError:
log.debug('No parser found for %s', filename) log.debug('No parser found for %s', filename)
except: except:
log.debug('Failed parsing %s', filename) log.debug('Failed parsing %s', filename)
@ -553,7 +569,7 @@ class Scanner(Plugin):
scan_result = [] scan_result = []
for p in paths: for p in paths:
if not group['is_dvd']: if not group['is_dvd']:
video = Video.from_path(toUnicode(p)) video = Video.from_path(sp(p))
video_result = [(video, video.scan())] video_result = [(video, video.scan())]
scan_result.extend(video_result) scan_result.extend(video_result)
@ -677,7 +693,7 @@ class Scanner(Plugin):
def removeCPTag(self, name): def removeCPTag(self, name):
try: try:
return re.sub(self.cp_imdb, '', name) return re.sub(self.cp_imdb, '', name).strip()
except: except:
pass pass
return name return name

101
couchpotato/core/plugins/score/scores.py

@ -33,33 +33,43 @@ name_scores = [
def nameScore(name, year, preferred_words): def nameScore(name, year, preferred_words):
""" Calculate score for words in the NZB name """ """ Calculate score for words in the NZB name """
score = 0 try:
name = name.lower() score = 0
name = name.lower()
# give points for the cool stuff # give points for the cool stuff
for value in name_scores: for value in name_scores:
v = value.split(':') v = value.split(':')
add = int(v.pop()) add = int(v.pop())
if v.pop() in name: if v.pop() in name:
score += add score += add
# points if the year is correct # points if the year is correct
if str(year) in name: if str(year) in name:
score += 5 score += 5
# Contains preferred word # Contains preferred word
nzb_words = re.split('\W+', simplifyString(name)) nzb_words = re.split('\W+', simplifyString(name))
score += 100 * len(list(set(nzb_words) & set(preferred_words))) score += 100 * len(list(set(nzb_words) & set(preferred_words)))
return score return score
except:
log.error('Failed doing nameScore: %s', traceback.format_exc())
return 0
def nameRatioScore(nzb_name, movie_name): def nameRatioScore(nzb_name, movie_name):
nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True)) try:
movie_words = re.split('\W+', simplifyString(movie_name)) nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True))
movie_words = re.split('\W+', simplifyString(movie_name))
left_over = set(nzb_words) - set(movie_words) left_over = set(nzb_words) - set(movie_words)
return 10 - len(left_over) return 10 - len(left_over)
except:
log.error('Failed doing nameRatioScore: %s', traceback.format_exc())
return 0
def namePositionScore(nzb_name, movie_name): def namePositionScore(nzb_name, movie_name):
@ -134,38 +144,53 @@ def providerScore(provider):
def duplicateScore(nzb_name, movie_name): def duplicateScore(nzb_name, movie_name):
nzb_words = re.split('\W+', simplifyString(nzb_name)) try:
movie_words = re.split('\W+', simplifyString(movie_name)) nzb_words = re.split('\W+', simplifyString(nzb_name))
movie_words = re.split('\W+', simplifyString(movie_name))
# minus for duplicates # minus for duplicates
duplicates = [x for i, x in enumerate(nzb_words) if nzb_words[i:].count(x) > 1] duplicates = [x for i, x in enumerate(nzb_words) if nzb_words[i:].count(x) > 1]
return len(list(set(duplicates) - set(movie_words))) * -4 return len(list(set(duplicates) - set(movie_words))) * -4
except:
log.error('Failed doing duplicateScore: %s', traceback.format_exc())
return 0
def partialIgnoredScore(nzb_name, movie_name, ignored_words): def partialIgnoredScore(nzb_name, movie_name, ignored_words):
nzb_name = nzb_name.lower() try:
movie_name = movie_name.lower() nzb_name = nzb_name.lower()
movie_name = movie_name.lower()
score = 0 score = 0
for ignored_word in ignored_words: for ignored_word in ignored_words:
if ignored_word in nzb_name and ignored_word not in movie_name: if ignored_word in nzb_name and ignored_word not in movie_name:
score -= 5 score -= 5
return score return score
except:
log.error('Failed doing partialIgnoredScore: %s', traceback.format_exc())
return 0
def halfMultipartScore(nzb_name): def halfMultipartScore(nzb_name):
wrong_found = 0 try:
for nr in [1, 2, 3, 4, 5, 'i', 'ii', 'iii', 'iv', 'v', 'a', 'b', 'c', 'd', 'e']: wrong_found = 0
for wrong in ['cd', 'part', 'dis', 'disc', 'dvd']: for nr in [1, 2, 3, 4, 5, 'i', 'ii', 'iii', 'iv', 'v', 'a', 'b', 'c', 'd', 'e']:
if '%s%s' % (wrong, nr) in nzb_name.lower(): for wrong in ['cd', 'part', 'dis', 'disc', 'dvd']:
wrong_found += 1 if '%s%s' % (wrong, nr) in nzb_name.lower():
wrong_found += 1
if wrong_found == 1:
return -30
if wrong_found == 1: return 0
return -30 except:
log.error('Failed doing halfMultipartScore: %s', traceback.format_exc())
return 0 return 0

32
couchpotato/runner.py

@ -9,6 +9,7 @@ import traceback
import warnings import warnings
import re import re
import tarfile import tarfile
import shutil
from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase
from argparse import ArgumentParser from argparse import ArgumentParser
@ -19,6 +20,7 @@ from couchpotato.core.event import fireEventAsync, fireEvent
from couchpotato.core.helpers.encoding import sp from couchpotato.core.helpers.encoding import sp
from couchpotato.core.helpers.variable import getDataDir, tryInt, getFreeSpace from couchpotato.core.helpers.variable import getDataDir, tryInt, getFreeSpace
import requests import requests
from requests.packages.urllib3 import disable_warnings
from tornado.httpserver import HTTPServer from tornado.httpserver import HTTPServer
from tornado.web import Application, StaticFileHandler, RedirectHandler from tornado.web import Application, StaticFileHandler, RedirectHandler
@ -107,14 +109,20 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
if not os.path.isdir(backup_path): os.makedirs(backup_path) if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path): for root, dirs, files in os.walk(backup_path):
for backup_file in sorted(files): # Only consider files being a direct child of the backup_path
ints = re.findall('\d+', backup_file) if root == backup_path:
for backup_file in sorted(files):
# Delete non zip files ints = re.findall('\d+', backup_file)
if len(ints) != 1:
os.remove(os.path.join(backup_path, backup_file)) # Delete non zip files
else: if len(ints) != 1:
existing_backups.append((int(ints[0]), backup_file)) try: os.remove(os.path.join(root, backup_file))
except: pass
else:
existing_backups.append((int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5 # Remove all but the last 5
for eb in existing_backups[:-backup_count]: for eb in existing_backups[:-backup_count]:
@ -144,12 +152,15 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
if not os.path.exists(python_cache): if not os.path.exists(python_cache):
os.mkdir(python_cache) os.mkdir(python_cache)
session = requests.Session()
session.max_redirects = 5
# Register environment settings # Register environment settings
Env.set('app_dir', sp(base_path)) Env.set('app_dir', sp(base_path))
Env.set('data_dir', sp(data_dir)) Env.set('data_dir', sp(data_dir))
Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db', db) Env.set('db', db)
Env.set('http_opener', requests.Session()) Env.set('http_opener', session)
Env.set('cache_dir', cache_dir) Env.set('cache_dir', cache_dir)
Env.set('cache', FileSystemCache(python_cache)) Env.set('cache', FileSystemCache(python_cache))
Env.set('console_log', options.console_log) Env.set('console_log', options.console_log)
@ -174,6 +185,9 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
for logger_name in ['gntp']: for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING) logging.getLogger(logger_name).setLevel(logging.WARNING)
# Disable SSL warning
disable_warnings()
# Use reloader # Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon reloader = debug is True and development and not Env.get('desktop') and not options.daemon

15
couchpotato/static/scripts/couchpotato.js

@ -54,16 +54,22 @@
}, },
pushState: function(e){ pushState: function(e){
if((!e.meta && Browser.platform.mac) || (!e.control && !Browser.platform.mac)){ var self = this;
if((!e.meta && self.isMac()) || (!e.control && !self.isMac())){
(e).preventDefault(); (e).preventDefault();
var url = e.target.get('href'); var url = e.target.get('href');
if(History.getPath() != url)
// Middle click
if(e.event && e.event.button == 1)
window.open(url);
else if(History.getPath() != url)
History.push(url); History.push(url);
} }
}, },
isMac: function(){ isMac: function(){
return Browser.platform.mac return Browser.platform == 'mac'
}, },
createLayout: function(){ createLayout: function(){
@ -325,11 +331,12 @@
}, },
openDerefered: function(e, el){ openDerefered: function(e, el){
var self = this;
(e).stop(); (e).stop();
var url = 'http://www.dereferer.org/?' + el.get('href'); var url = 'http://www.dereferer.org/?' + el.get('href');
if(el.get('target') == '_blank' || (e.meta && Browser.platform.mac) || (e.control && !Browser.platform.mac)) if(el.get('target') == '_blank' || (e.meta && self.isMac()) || (e.control && !self.isMac()))
window.open(url); window.open(url);
else else
window.location = url; window.location = url;

84
couchpotato/static/scripts/page/home.js

@ -146,13 +146,13 @@ Page.Home = new Class({
var self = this; var self = this;
// Suggest // Suggest
self.suggestion_list = new SuggestList({ self.suggestions_list = new SuggestList({
'onLoaded': function(){ 'onCreated': function(){
self.chain.callChain(); self.chain.callChain();
} }
}); });
$(self.suggestion_list).inject(self.el); $(self.suggestions_list).inject(self.el);
}, },
@ -160,46 +160,38 @@ Page.Home = new Class({
var self = this; var self = this;
// Charts // Charts
self.charts = new Charts({ self.charts_list = new Charts({
'onCreated': function(){ 'onCreated': function(){
self.chain.callChain(); self.chain.callChain();
} }
}); });
$(self.charts).inject(self.el); $(self.charts_list).inject(self.el);
}, },
createSuggestionsChartsMenu: function(){ createSuggestionsChartsMenu: function(){
var self = this; var self = this,
suggestion_tab, charts_tab;
self.el_toggle_menu = new Element('div.toggle_menu', {
'events': {
'click:relay(a)': function(e, el) {
e.preventDefault();
self.toggleSuggestionsCharts(el.get('data-container'), el);
}
}
}).adopt(
suggestion_tab = new Element('a.toggle_suggestions', {
'data-container': 'suggestions'
}).grab(new Element('h2', {'text': 'Suggestions'})),
charts_tab = new Element('a.toggle_charts', {
'data-container': 'charts'
}).grab( new Element('h2', {'text': 'Charts'}))
);
self.el_toggle_menu_suggestions = new Element('a.toggle_suggestions.active', { var menu_selected = Cookie.read('suggestions_charts_menu_selected') || 'suggestions';
'href': '#', self.toggleSuggestionsCharts(menu_selected, menu_selected == 'suggestions' ? suggestion_tab : charts_tab);
'events': { 'click': function(e) {
e.preventDefault();
self.toggleSuggestionsCharts('suggestions');
}
}
}).grab( new Element('h2', {'text': 'Suggestions'}));
self.el_toggle_menu_charts = new Element('a.toggle_charts', {
'href': '#',
'events': { 'click': function(e) {
e.preventDefault();
self.toggleSuggestionsCharts('charts');
}
}
}).grab( new Element('h2', {'text': 'Charts'}));
self.el_toggle_menu = new Element('div.toggle_menu').grab(
self.el_toggle_menu_suggestions
).grab(
self.el_toggle_menu_charts
);
var menu_selected = Cookie.read('suggestions_charts_menu_selected');
if( menu_selected === null ) menu_selected = 'suggestions';
self.toggleSuggestionsCharts( menu_selected );
self.el_toggle_menu.inject(self.el); self.el_toggle_menu.inject(self.el);
@ -207,23 +199,19 @@ Page.Home = new Class({
}, },
toggleSuggestionsCharts: function(menu_id){ toggleSuggestionsCharts: function(menu_id, el){
var self = this; var self = this;
switch(menu_id) { // Toggle ta
case 'suggestions': self.el_toggle_menu.getElements('.active').removeClass('active');
if($(self.suggestion_list)) $(self.suggestion_list).show(); if(el) el.addClass('active');
self.el_toggle_menu_suggestions.addClass('active');
if($(self.charts)) $(self.charts).hide(); // Hide both
self.el_toggle_menu_charts.removeClass('active'); if(self.suggestions_list) self.suggestions_list.hide();
break; if(self.charts_list) self.charts_list.hide();
case 'charts':
if($(self.charts)) $(self.charts).show(); var toggle_to = self[menu_id + '_list'];
self.el_toggle_menu_charts.addClass('active'); if(toggle_to) toggle_to.show();
if($(self.suggestion_list)) $(self.suggestion_list).hide();
self.el_toggle_menu_suggestions.removeClass('active');
break;
}
Cookie.write('suggestions_charts_menu_selected', menu_id, {'duration': 365}); Cookie.write('suggestions_charts_menu_selected', menu_id, {'duration': 365});
}, },

136
couchpotato/static/scripts/page/settings.js

@ -560,11 +560,19 @@ Option.Password = new Class({
create: function(){ create: function(){
var self = this; var self = this;
self.parent(); self.el.adopt(
self.input.set('type', 'password'); self.createLabel(),
self.input = new Element('input.inlay', {
'type': 'text',
'name': self.postName(),
'value': self.getSettingValue() ? '********' : '',
'placeholder': self.getPlaceholder()
})
);
self.input.addEvent('focus', function(){ self.input.addEvent('focus', function(){
self.input.set('value', '') self.input.set('value', '');
self.input.set('type', 'password');
}) })
} }
@ -634,6 +642,7 @@ Option.Directory = new Class({
browser: null, browser: null,
save_on_change: false, save_on_change: false,
use_cache: false, use_cache: false,
current_dir: '',
create: function(){ create: function(){
var self = this; var self = this;
@ -645,8 +654,17 @@ Option.Directory = new Class({
'click': self.showBrowser.bind(self) 'click': self.showBrowser.bind(self)
} }
}).adopt( }).adopt(
self.input = new Element('span', { self.input = new Element('input', {
'text': self.getSettingValue() 'value': self.getSettingValue(),
'events': {
'change': self.filterDirectory.bind(self),
'keydown': function(e){
if(e.key == 'enter' || e.key == 'tab')
(e).stop();
},
'keyup': self.filterDirectory.bind(self),
'paste': self.filterDirectory.bind(self)
}
}) })
) )
); );
@ -654,10 +672,55 @@ Option.Directory = new Class({
self.cached = {}; self.cached = {};
}, },
filterDirectory: function(e){
var self = this,
value = self.getValue(),
path_sep = Api.getOption('path_sep'),
active_selector = 'li:not(.blur):not(.empty)';
if(e.key == 'enter' || e.key == 'tab'){
(e).stop();
var first = self.dir_list.getElement(active_selector);
if(first){
self.selectDirectory(first.get('data-value'));
}
}
else {
// New folder
if(value.substr(-1) == path_sep){
if(self.current_dir != value)
self.selectDirectory(value)
}
else {
var pd = self.getParentDir(value);
if(self.current_dir != pd)
self.getDirs(pd);
var folder_filter = value.split(path_sep).getLast()
self.dir_list.getElements('li').each(function(li){
var valid = li.get('text').substr(0, folder_filter.length).toLowerCase() != folder_filter.toLowerCase()
li[valid ? 'addClass' : 'removeClass']('blur')
});
var first = self.dir_list.getElement(active_selector);
if(first){
if(!self.dir_list_scroll)
self.dir_list_scroll = new Fx.Scroll(self.dir_list, {
'transition': 'quint:in:out'
});
self.dir_list_scroll.toElement(first);
}
}
}
},
selectDirectory: function(dir){ selectDirectory: function(dir){
var self = this; var self = this;
self.input.set('text', dir); self.input.set('value', dir);
self.getDirs() self.getDirs()
}, },
@ -668,9 +731,28 @@ Option.Directory = new Class({
self.selectDirectory(self.getParentDir()) self.selectDirectory(self.getParentDir())
}, },
caretAtEnd: function(){
var self = this;
self.input.focus();
if (typeof self.input.selectionStart == "number") {
self.input.selectionStart = self.input.selectionEnd = self.input.get('value').length;
} else if (typeof el.createTextRange != "undefined") {
self.input.focus();
var range = self.input.createTextRange();
range.collapse(false);
range.select();
}
},
showBrowser: function(){ showBrowser: function(){
var self = this; var self = this;
// Move caret to back of the input
if(!self.browser || self.browser && !self.browser.isVisible())
self.caretAtEnd()
if(!self.browser){ if(!self.browser){
self.browser = new Element('div.directory_list').adopt( self.browser = new Element('div.directory_list').adopt(
new Element('div.pointer'), new Element('div.pointer'),
@ -686,7 +768,9 @@ Option.Directory = new Class({
}).adopt( }).adopt(
self.show_hidden = new Element('input[type=checkbox].inlay', { self.show_hidden = new Element('input[type=checkbox].inlay', {
'events': { 'events': {
'change': self.getDirs.bind(self) 'change': function(){
self.getDirs()
}
} }
}) })
) )
@ -707,7 +791,7 @@ Option.Directory = new Class({
'text': 'Clear', 'text': 'Clear',
'events': { 'events': {
'click': function(e){ 'click': function(e){
self.input.set('text', ''); self.input.set('value', '');
self.hideBrowser(e, true); self.hideBrowser(e, true);
} }
} }
@ -735,7 +819,7 @@ Option.Directory = new Class({
new Form.Check(self.show_hidden); new Form.Check(self.show_hidden);
} }
self.initial_directory = self.input.get('text'); self.initial_directory = self.input.get('value');
self.getDirs(); self.getDirs();
self.browser.show(); self.browser.show();
@ -749,7 +833,7 @@ Option.Directory = new Class({
if(save) if(save)
self.save(); self.save();
else else
self.input.set('text', self.initial_directory); self.input.set('value', self.initial_directory);
self.browser.hide(); self.browser.hide();
self.el.removeEvents('outerClick') self.el.removeEvents('outerClick')
@ -757,21 +841,21 @@ Option.Directory = new Class({
}, },
fillBrowser: function(json){ fillBrowser: function(json){
var self = this; var self = this,
v = self.getValue();
self.data = json; self.data = json;
var v = self.getValue(); var previous_dir = json.parent;
var previous_dir = self.getParentDir();
if(v == '') if(v == '')
self.input.set('text', json.home); self.input.set('value', json.home);
if(previous_dir != v && previous_dir.length >= 1 && !json.is_root){ if(previous_dir.length >= 1 && !json.is_root){
var prev_dirname = self.getCurrentDirname(previous_dir); var prev_dirname = self.getCurrentDirname(previous_dir);
if(previous_dir == json.home) if(previous_dir == json.home)
prev_dirname = 'Home'; prev_dirname = 'Home Folder';
else if(previous_dir == '/' && json.platform == 'nt') else if(previous_dir == '/' && json.platform == 'nt')
prev_dirname = 'Computer'; prev_dirname = 'Computer';
@ -801,12 +885,13 @@ Option.Directory = new Class({
new Element('li.empty', { new Element('li.empty', {
'text': 'Selected folder is empty' 'text': 'Selected folder is empty'
}).inject(self.dir_list) }).inject(self.dir_list)
},
getDirs: function(){ self.caretAtEnd();
var self = this; },
var c = self.getValue(); getDirs: function(dir){
var self = this,
c = dir || self.getValue();
if(self.cached[c] && self.use_cache){ if(self.cached[c] && self.use_cache){
self.fillBrowser() self.fillBrowser()
@ -817,7 +902,10 @@ Option.Directory = new Class({
'path': c, 'path': c,
'show_hidden': +self.show_hidden.checked 'show_hidden': +self.show_hidden.checked
}, },
'onComplete': self.fillBrowser.bind(self) 'onComplete': function(json){
self.current_dir = c;
self.fillBrowser(json);
}
}) })
} }
}, },
@ -831,8 +919,8 @@ Option.Directory = new Class({
var v = dir || self.getValue(); var v = dir || self.getValue();
var sep = Api.getOption('path_sep'); var sep = Api.getOption('path_sep');
var dirs = v.split(sep); var dirs = v.split(sep);
if(dirs.pop() == '') if(dirs.pop() == '')
dirs.pop(); dirs.pop();
return dirs.join(sep) + sep return dirs.join(sep) + sep
}, },
@ -845,7 +933,7 @@ Option.Directory = new Class({
getValue: function(){ getValue: function(){
var self = this; var self = this;
return self.input.get('text'); return self.input.get('value');
} }
}); });

13
couchpotato/static/style/settings.css

@ -302,15 +302,19 @@
font-family: 'Elusive-Icons'; font-family: 'Elusive-Icons';
color: #f5e39c; color: #f5e39c;
} }
.page form .directory > span { .page form .directory > input {
height: 25px; height: 25px;
display: inline-block; display: inline-block;
float: right; float: right;
text-align: right; text-align: right;
white-space: nowrap; white-space: nowrap;
cursor: pointer; cursor: pointer;
background: none;
border: 0;
color: #FFF;
width: 100%;
} }
.page form .directory span:empty:before { .page form .directory input:empty:before {
content: 'No folder selected'; content: 'No folder selected';
font-style: italic; font-style: italic;
opacity: .3; opacity: .3;
@ -353,6 +357,11 @@
white-space: nowrap; white-space: nowrap;
text-overflow: ellipsis; text-overflow: ellipsis;
} }
.page .directory_list li.blur {
opacity: .3;
}
.page .directory_list li:last-child { .page .directory_list li:last-child {
border-bottom: 1px solid rgba(255,255,255,0.1); border-bottom: 1px solid rgba(255,255,255,0.1);
} }

2
couchpotato/templates/index.html

@ -5,7 +5,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"/> <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"/>
<meta name="apple-mobile-web-app-capable" content="yes"> <meta name="apple-mobile-web-app-capable" content="yes">
<meta name="mobile-web-app-capable" content="yes"> <meta name="mobile-web-app-capable" content="yes">
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
{% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %} {% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %}
<link rel="stylesheet" href="{{ Env.get('web_base') }}{{ url }}" type="text/css">{% end %} <link rel="stylesheet" href="{{ Env.get('web_base') }}{{ url }}" type="text/css">{% end %}

12
couchpotato/templates/login.html

@ -4,22 +4,24 @@
<head> <head>
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"/> <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"/>
<meta name="apple-mobile-web-app-capable" content="yes"> <meta name="apple-mobile-web-app-capable" content="yes">
<meta name="mobile-web-app-capable" content="yes">
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
{% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %} {% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %}
<link rel="stylesheet" href="{{ Env.get('web_base') }}{{ url }}" type="text/css">{% end %} <link rel="stylesheet" href="{{ Env.get('web_base') }}{{ url }}" type="text/css">{% end %}
{% for url in fireEvent('clientscript.get_scripts', as_html = True, location = 'front', single = True) %} {% for url in fireEvent('clientscript.get_scripts', as_html = True, location = 'front', single = True) %}
<script type="text/javascript" src="{{ Env.get('web_base') }}{{ url }}"></script>{% end %} <script type="text/javascript" src="{{ Env.get('web_base') }}{{ url }}"></script>{% end %}
<link href="{{ Env.get('static_path') }}images/favicon.ico" rel="icon" type="image/x-icon" /> <link href="{{ Env.get('static_path') }}images/favicon.ico" rel="icon" type="image/x-icon" />
<link rel="apple-touch-icon" href="{{ Env.get('static_path') }}images/homescreen.png" /> <link rel="apple-touch-icon" href="{{ Env.get('static_path') }}images/homescreen.png" />
<script type="text/javascript"> <script type="text/javascript">
window.addEvent('domready', function(){ window.addEvent('domready', function(){
new Form.Check($('remember_me')); new Form.Check($('remember_me'));
}); });
</script> </script>
<title>CouchPotato</title> <title>CouchPotato</title>
@ -35,4 +37,4 @@
</div> </div>
</form> </form>
</body> </body>
</html> </html>

6
init/ubuntu

@ -46,7 +46,7 @@ DESC=CouchPotato
# Run CP as username # Run CP as username
RUN_AS=${CP_USER-couchpotato} RUN_AS=${CP_USER-couchpotato}
# Path to app # Path to app
# CP_HOME=path_to_app_CouchPotato.py # CP_HOME=path_to_app_CouchPotato.py
APP_PATH=${CP_HOME-/opt/couchpotato/} APP_PATH=${CP_HOME-/opt/couchpotato/}
@ -100,12 +100,12 @@ case "$1" in
;; ;;
stop) stop)
echo "Stopping $DESC" echo "Stopping $DESC"
start-stop-daemon --stop --pidfile $PID_FILE --retry 15 start-stop-daemon --stop --pidfile $PID_FILE --retry 15 --oknodo
;; ;;
restart|force-reload) restart|force-reload)
echo "Restarting $DESC" echo "Restarting $DESC"
start-stop-daemon --stop --pidfile $PID_FILE --retry 15 start-stop-daemon --stop --pidfile $PID_FILE --retry 15 --oknodo
start-stop-daemon -d $APP_PATH -c $RUN_AS $EXTRA_SSD_OPTS --start --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS start-stop-daemon -d $APP_PATH -c $RUN_AS $EXTRA_SSD_OPTS --start --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS
;; ;;

6
libs/axl/axel.py

@ -235,12 +235,12 @@ class Event(object):
self.error_handler(sys.exc_info()) self.error_handler(sys.exc_info())
finally: finally:
if not self.asynchronous:
self.queue.task_done()
if order_lock: if order_lock:
order_lock.release() order_lock.release()
if not self.asynchronous:
self.queue.task_done()
if self.queue.empty(): if self.queue.empty():
raise Empty raise Empty

14
libs/chardet/__init__.py

@ -3,22 +3,28 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
__version__ = "1.0.1" __version__ = "2.2.1"
from sys import version_info
def detect(aBuf): def detect(aBuf):
import universaldetector if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or
(version_info >= (3, 0) and not isinstance(aBuf, bytes))):
raise ValueError('Expected a bytes object, not a unicode object')
from . import universaldetector
u = universaldetector.UniversalDetector() u = universaldetector.UniversalDetector()
u.reset() u.reset()
u.feed(aBuf) u.feed(aBuf)

20
libs/chardet/big5freq.py

@ -1,11 +1,11 @@
######################## BEGIN LICENSE BLOCK ######################## ######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code. # The Original Code is Mozilla Communicator client code.
# #
# The Initial Developer of the Original Code is # The Initial Developer of the Original Code is
# Netscape Communications Corporation. # Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998 # Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved. # the Initial Developer. All Rights Reserved.
# #
# Contributor(s): # Contributor(s):
# Mark Pilgrim - port to Python # Mark Pilgrim - port to Python
# #
@ -13,12 +13,12 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
@ -26,18 +26,18 @@
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
# Big5 frequency table # Big5 frequency table
# by Taiwan's Mandarin Promotion Council # by Taiwan's Mandarin Promotion Council
# <http://www.edu.tw:81/mandr/> # <http://www.edu.tw:81/mandr/>
# #
# 128 --> 0.42261 # 128 --> 0.42261
# 256 --> 0.57851 # 256 --> 0.57851
# 512 --> 0.74851 # 512 --> 0.74851
# 1024 --> 0.89384 # 1024 --> 0.89384
# 2048 --> 0.97583 # 2048 --> 0.97583
# #
# Ideal Distribution Ratio = 0.74851/(1-0.74851) =2.98 # Ideal Distribution Ratio = 0.74851/(1-0.74851) =2.98
# Random Distribution Ration = 512/(5401-512)=0.105 # Random Distribution Ration = 512/(5401-512)=0.105
# #
# Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR # Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR
BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75 BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75
@ -45,7 +45,7 @@ BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75
#Char to FreqOrder table #Char to FreqOrder table
BIG5_TABLE_SIZE = 5376 BIG5_TABLE_SIZE = 5376
Big5CharToFreqOrder = ( \ Big5CharToFreqOrder = (
1,1801,1506, 255,1431, 198, 9, 82, 6,5008, 177, 202,3681,1256,2821, 110, # 16 1,1801,1506, 255,1431, 198, 9, 82, 6,5008, 177, 202,3681,1256,2821, 110, # 16
3814, 33,3274, 261, 76, 44,2114, 16,2946,2187,1176, 659,3971, 26,3451,2653, # 32 3814, 33,3274, 261, 76, 44,2114, 16,2946,2187,1176, 659,3971, 26,3451,2653, # 32
1198,3972,3350,4202, 410,2215, 302, 590, 361,1964, 8, 204, 58,4510,5009,1932, # 48 1198,3972,3350,4202, 410,2215, 302, 590, 361,1964, 8, 204, 58,4510,5009,1932, # 48
@ -921,3 +921,5 @@ Big5CharToFreqOrder = ( \
13936,13937,13938,13939,13940,13941,13942,13943,13944,13945,13946,13947,13948,13949,13950,13951, #13952 13936,13937,13938,13939,13940,13941,13942,13943,13944,13945,13946,13947,13948,13949,13950,13951, #13952
13952,13953,13954,13955,13956,13957,13958,13959,13960,13961,13962,13963,13964,13965,13966,13967, #13968 13952,13953,13954,13955,13956,13957,13958,13959,13960,13961,13962,13963,13964,13965,13966,13967, #13968
13968,13969,13970,13971,13972) #13973 13968,13969,13970,13971,13972) #13973
# flake8: noqa

17
libs/chardet/big5prober.py

@ -1,11 +1,11 @@
######################## BEGIN LICENSE BLOCK ######################## ######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code. # The Original Code is Mozilla Communicator client code.
# #
# The Initial Developer of the Original Code is # The Initial Developer of the Original Code is
# Netscape Communications Corporation. # Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998 # Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved. # the Initial Developer. All Rights Reserved.
# #
# Contributor(s): # Contributor(s):
# Mark Pilgrim - port to Python # Mark Pilgrim - port to Python
# #
@ -13,22 +13,23 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from mbcharsetprober import MultiByteCharSetProber from .mbcharsetprober import MultiByteCharSetProber
from codingstatemachine import CodingStateMachine from .codingstatemachine import CodingStateMachine
from chardistribution import Big5DistributionAnalysis from .chardistribution import Big5DistributionAnalysis
from mbcssm import Big5SMModel from .mbcssm import Big5SMModel
class Big5Prober(MultiByteCharSetProber): class Big5Prober(MultiByteCharSetProber):
def __init__(self): def __init__(self):

46
libs/chardet/chardetect.py

@ -0,0 +1,46 @@
#!/usr/bin/env python
"""
Script which takes one or more file paths and reports on their detected
encodings
Example::
% chardetect somefile someotherfile
somefile: windows-1252 with confidence 0.5
someotherfile: ascii with confidence 1.0
If no paths are provided, it takes its input from stdin.
"""
from io import open
from sys import argv, stdin
from chardet.universaldetector import UniversalDetector
def description_of(file, name='stdin'):
"""Return a string describing the probable encoding of a file."""
u = UniversalDetector()
for line in file:
u.feed(line)
u.close()
result = u.result
if result['encoding']:
return '%s: %s with confidence %s' % (name,
result['encoding'],
result['confidence'])
else:
return '%s: no result' % name
def main():
if len(argv) <= 1:
print(description_of(stdin))
else:
for path in argv[1:]:
with open(path, 'rb') as f:
print(description_of(f, path))
if __name__ == '__main__':
main()

159
libs/chardet/chardistribution.py

@ -1,11 +1,11 @@
######################## BEGIN LICENSE BLOCK ######################## ######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code. # The Original Code is Mozilla Communicator client code.
# #
# The Initial Developer of the Original Code is # The Initial Developer of the Original Code is
# Netscape Communications Corporation. # Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998 # Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved. # the Initial Developer. All Rights Reserved.
# #
# Contributor(s): # Contributor(s):
# Mark Pilgrim - port to Python # Mark Pilgrim - port to Python
# #
@ -13,47 +13,63 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
import constants from .euctwfreq import (EUCTWCharToFreqOrder, EUCTW_TABLE_SIZE,
from euctwfreq import EUCTWCharToFreqOrder, EUCTW_TABLE_SIZE, EUCTW_TYPICAL_DISTRIBUTION_RATIO EUCTW_TYPICAL_DISTRIBUTION_RATIO)
from euckrfreq import EUCKRCharToFreqOrder, EUCKR_TABLE_SIZE, EUCKR_TYPICAL_DISTRIBUTION_RATIO from .euckrfreq import (EUCKRCharToFreqOrder, EUCKR_TABLE_SIZE,
from gb2312freq import GB2312CharToFreqOrder, GB2312_TABLE_SIZE, GB2312_TYPICAL_DISTRIBUTION_RATIO EUCKR_TYPICAL_DISTRIBUTION_RATIO)
from big5freq import Big5CharToFreqOrder, BIG5_TABLE_SIZE, BIG5_TYPICAL_DISTRIBUTION_RATIO from .gb2312freq import (GB2312CharToFreqOrder, GB2312_TABLE_SIZE,
from jisfreq import JISCharToFreqOrder, JIS_TABLE_SIZE, JIS_TYPICAL_DISTRIBUTION_RATIO GB2312_TYPICAL_DISTRIBUTION_RATIO)
from .big5freq import (Big5CharToFreqOrder, BIG5_TABLE_SIZE,
BIG5_TYPICAL_DISTRIBUTION_RATIO)
from .jisfreq import (JISCharToFreqOrder, JIS_TABLE_SIZE,
JIS_TYPICAL_DISTRIBUTION_RATIO)
from .compat import wrap_ord
ENOUGH_DATA_THRESHOLD = 1024 ENOUGH_DATA_THRESHOLD = 1024
SURE_YES = 0.99 SURE_YES = 0.99
SURE_NO = 0.01 SURE_NO = 0.01
MINIMUM_DATA_THRESHOLD = 3
class CharDistributionAnalysis: class CharDistributionAnalysis:
def __init__(self): def __init__(self):
self._mCharToFreqOrder = None # Mapping table to get frequency order from char order (get from GetOrder()) # Mapping table to get frequency order from char order (get from
self._mTableSize = None # Size of above table # GetOrder())
self._mTypicalDistributionRatio = None # This is a constant value which varies from language to language, used in calculating confidence. See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail. self._mCharToFreqOrder = None
self._mTableSize = None # Size of above table
# This is a constant value which varies from language to language,
# used in calculating confidence. See
# http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html
# for further detail.
self._mTypicalDistributionRatio = None
self.reset() self.reset()
def reset(self): def reset(self):
"""reset analyser, clear any state""" """reset analyser, clear any state"""
self._mDone = constants.False # If this flag is set to constants.True, detection is done and conclusion has been made # If this flag is set to True, detection is done and conclusion has
self._mTotalChars = 0 # Total characters encountered # been made
self._mFreqChars = 0 # The number of characters whose frequency order is less than 512 self._mDone = False
self._mTotalChars = 0 # Total characters encountered
def feed(self, aStr, aCharLen): # The number of characters whose frequency order is less than 512
self._mFreqChars = 0
def feed(self, aBuf, aCharLen):
"""feed a character with known length""" """feed a character with known length"""
if aCharLen == 2: if aCharLen == 2:
# we only care about 2-bytes character in our distribution analysis # we only care about 2-bytes character in our distribution analysis
order = self.get_order(aStr) order = self.get_order(aBuf)
else: else:
order = -1 order = -1
if order >= 0: if order >= 0:
@ -65,12 +81,14 @@ class CharDistributionAnalysis:
def get_confidence(self): def get_confidence(self):
"""return confidence based on existing data""" """return confidence based on existing data"""
# if we didn't receive any character in our consideration range, return negative answer # if we didn't receive any character in our consideration range,
if self._mTotalChars <= 0: # return negative answer
if self._mTotalChars <= 0 or self._mFreqChars <= MINIMUM_DATA_THRESHOLD:
return SURE_NO return SURE_NO
if self._mTotalChars != self._mFreqChars: if self._mTotalChars != self._mFreqChars:
r = self._mFreqChars / ((self._mTotalChars - self._mFreqChars) * self._mTypicalDistributionRatio) r = (self._mFreqChars / ((self._mTotalChars - self._mFreqChars)
* self._mTypicalDistributionRatio))
if r < SURE_YES: if r < SURE_YES:
return r return r
@ -78,16 +96,18 @@ class CharDistributionAnalysis:
return SURE_YES return SURE_YES
def got_enough_data(self): def got_enough_data(self):
# It is not necessary to receive all data to draw conclusion. For charset detection, # It is not necessary to receive all data to draw conclusion.
# certain amount of data is enough # For charset detection, certain amount of data is enough
return self._mTotalChars > ENOUGH_DATA_THRESHOLD return self._mTotalChars > ENOUGH_DATA_THRESHOLD
def get_order(self, aStr): def get_order(self, aBuf):
# We do not handle characters based on the original encoding string, but # We do not handle characters based on the original encoding string,
# convert this encoding string to a number, here called order. # but convert this encoding string to a number, here called order.
# This allows multiple encodings of a language to share one frequency table. # This allows multiple encodings of a language to share one frequency
# table.
return -1 return -1
class EUCTWDistributionAnalysis(CharDistributionAnalysis): class EUCTWDistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self):
CharDistributionAnalysis.__init__(self) CharDistributionAnalysis.__init__(self)
@ -95,16 +115,18 @@ class EUCTWDistributionAnalysis(CharDistributionAnalysis):
self._mTableSize = EUCTW_TABLE_SIZE self._mTableSize = EUCTW_TABLE_SIZE
self._mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO self._mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, aStr): def get_order(self, aBuf):
# for euc-TW encoding, we are interested # for euc-TW encoding, we are interested
# first byte range: 0xc4 -- 0xfe # first byte range: 0xc4 -- 0xfe
# second byte range: 0xa1 -- 0xfe # second byte range: 0xa1 -- 0xfe
# no validation needed here. State machine has done that # no validation needed here. State machine has done that
if aStr[0] >= '\xC4': first_char = wrap_ord(aBuf[0])
return 94 * (ord(aStr[0]) - 0xC4) + ord(aStr[1]) - 0xA1 if first_char >= 0xC4:
return 94 * (first_char - 0xC4) + wrap_ord(aBuf[1]) - 0xA1
else: else:
return -1 return -1
class EUCKRDistributionAnalysis(CharDistributionAnalysis): class EUCKRDistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self):
CharDistributionAnalysis.__init__(self) CharDistributionAnalysis.__init__(self)
@ -112,15 +134,17 @@ class EUCKRDistributionAnalysis(CharDistributionAnalysis):
self._mTableSize = EUCKR_TABLE_SIZE self._mTableSize = EUCKR_TABLE_SIZE
self._mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO self._mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, aStr): def get_order(self, aBuf):
# for euc-KR encoding, we are interested # for euc-KR encoding, we are interested
# first byte range: 0xb0 -- 0xfe # first byte range: 0xb0 -- 0xfe
# second byte range: 0xa1 -- 0xfe # second byte range: 0xa1 -- 0xfe
# no validation needed here. State machine has done that # no validation needed here. State machine has done that
if aStr[0] >= '\xB0': first_char = wrap_ord(aBuf[0])
return 94 * (ord(aStr[0]) - 0xB0) + ord(aStr[1]) - 0xA1 if first_char >= 0xB0:
return 94 * (first_char - 0xB0) + wrap_ord(aBuf[1]) - 0xA1
else: else:
return -1; return -1
class GB2312DistributionAnalysis(CharDistributionAnalysis): class GB2312DistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self):
@ -129,15 +153,17 @@ class GB2312DistributionAnalysis(CharDistributionAnalysis):
self._mTableSize = GB2312_TABLE_SIZE self._mTableSize = GB2312_TABLE_SIZE
self._mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO self._mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, aStr): def get_order(self, aBuf):
# for GB2312 encoding, we are interested # for GB2312 encoding, we are interested
# first byte range: 0xb0 -- 0xfe # first byte range: 0xb0 -- 0xfe
# second byte range: 0xa1 -- 0xfe # second byte range: 0xa1 -- 0xfe
# no validation needed here. State machine has done that # no validation needed here. State machine has done that
if (aStr[0] >= '\xB0') and (aStr[1] >= '\xA1'): first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1])
return 94 * (ord(aStr[0]) - 0xB0) + ord(aStr[1]) - 0xA1 if (first_char >= 0xB0) and (second_char >= 0xA1):
return 94 * (first_char - 0xB0) + second_char - 0xA1
else: else:
return -1; return -1
class Big5DistributionAnalysis(CharDistributionAnalysis): class Big5DistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self):
@ -146,19 +172,21 @@ class Big5DistributionAnalysis(CharDistributionAnalysis):
self._mTableSize = BIG5_TABLE_SIZE self._mTableSize = BIG5_TABLE_SIZE
self._mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO self._mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, aStr): def get_order(self, aBuf):
# for big5 encoding, we are interested # for big5 encoding, we are interested
# first byte range: 0xa4 -- 0xfe # first byte range: 0xa4 -- 0xfe
# second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe # second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
# no validation needed here. State machine has done that # no validation needed here. State machine has done that
if aStr[0] >= '\xA4': first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1])
if aStr[1] >= '\xA1': if first_char >= 0xA4:
return 157 * (ord(aStr[0]) - 0xA4) + ord(aStr[1]) - 0xA1 + 63 if second_char >= 0xA1:
return 157 * (first_char - 0xA4) + second_char - 0xA1 + 63
else: else:
return 157 * (ord(aStr[0]) - 0xA4) + ord(aStr[1]) - 0x40 return 157 * (first_char - 0xA4) + second_char - 0x40
else: else:
return -1 return -1
class SJISDistributionAnalysis(CharDistributionAnalysis): class SJISDistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self):
CharDistributionAnalysis.__init__(self) CharDistributionAnalysis.__init__(self)
@ -166,22 +194,24 @@ class SJISDistributionAnalysis(CharDistributionAnalysis):
self._mTableSize = JIS_TABLE_SIZE self._mTableSize = JIS_TABLE_SIZE
self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, aStr): def get_order(self, aBuf):
# for sjis encoding, we are interested # for sjis encoding, we are interested
# first byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe # first byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
# second byte range: 0x40 -- 0x7e, 0x81 -- oxfe # second byte range: 0x40 -- 0x7e, 0x81 -- oxfe
# no validation needed here. State machine has done that # no validation needed here. State machine has done that
if (aStr[0] >= '\x81') and (aStr[0] <= '\x9F'): first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1])
order = 188 * (ord(aStr[0]) - 0x81) if (first_char >= 0x81) and (first_char <= 0x9F):
elif (aStr[0] >= '\xE0') and (aStr[0] <= '\xEF'): order = 188 * (first_char - 0x81)
order = 188 * (ord(aStr[0]) - 0xE0 + 31) elif (first_char >= 0xE0) and (first_char <= 0xEF):
order = 188 * (first_char - 0xE0 + 31)
else: else:
return -1; return -1
order = order + ord(aStr[1]) - 0x40 order = order + second_char - 0x40
if aStr[1] > '\x7F': if second_char > 0x7F:
order =- 1 order = -1
return order return order
class EUCJPDistributionAnalysis(CharDistributionAnalysis): class EUCJPDistributionAnalysis(CharDistributionAnalysis):
def __init__(self): def __init__(self):
CharDistributionAnalysis.__init__(self) CharDistributionAnalysis.__init__(self)
@ -189,12 +219,13 @@ class EUCJPDistributionAnalysis(CharDistributionAnalysis):
self._mTableSize = JIS_TABLE_SIZE self._mTableSize = JIS_TABLE_SIZE
self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
def get_order(self, aStr): def get_order(self, aBuf):
# for euc-JP encoding, we are interested # for euc-JP encoding, we are interested
# first byte range: 0xa0 -- 0xfe # first byte range: 0xa0 -- 0xfe
# second byte range: 0xa1 -- 0xfe # second byte range: 0xa1 -- 0xfe
# no validation needed here. State machine has done that # no validation needed here. State machine has done that
if aStr[0] >= '\xA0': char = wrap_ord(aBuf[0])
return 94 * (ord(aStr[0]) - 0xA1) + ord(aStr[1]) - 0xa1 if char >= 0xA0:
return 94 * (char - 0xA1) + wrap_ord(aBuf[1]) - 0xa1
else: else:
return -1 return -1

36
libs/chardet/charsetgroupprober.py

@ -25,8 +25,10 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
import constants, sys from . import constants
from charsetprober import CharSetProber import sys
from .charsetprober import CharSetProber
class CharSetGroupProber(CharSetProber): class CharSetGroupProber(CharSetProber):
def __init__(self): def __init__(self):
@ -34,35 +36,39 @@ class CharSetGroupProber(CharSetProber):
self._mActiveNum = 0 self._mActiveNum = 0
self._mProbers = [] self._mProbers = []
self._mBestGuessProber = None self._mBestGuessProber = None
def reset(self): def reset(self):
CharSetProber.reset(self) CharSetProber.reset(self)
self._mActiveNum = 0 self._mActiveNum = 0
for prober in self._mProbers: for prober in self._mProbers:
if prober: if prober:
prober.reset() prober.reset()
prober.active = constants.True prober.active = True
self._mActiveNum += 1 self._mActiveNum += 1
self._mBestGuessProber = None self._mBestGuessProber = None
def get_charset_name(self): def get_charset_name(self):
if not self._mBestGuessProber: if not self._mBestGuessProber:
self.get_confidence() self.get_confidence()
if not self._mBestGuessProber: return None if not self._mBestGuessProber:
return None
# self._mBestGuessProber = self._mProbers[0] # self._mBestGuessProber = self._mProbers[0]
return self._mBestGuessProber.get_charset_name() return self._mBestGuessProber.get_charset_name()
def feed(self, aBuf): def feed(self, aBuf):
for prober in self._mProbers: for prober in self._mProbers:
if not prober: continue if not prober:
if not prober.active: continue continue
if not prober.active:
continue
st = prober.feed(aBuf) st = prober.feed(aBuf)
if not st: continue if not st:
continue
if st == constants.eFoundIt: if st == constants.eFoundIt:
self._mBestGuessProber = prober self._mBestGuessProber = prober
return self.get_state() return self.get_state()
elif st == constants.eNotMe: elif st == constants.eNotMe:
prober.active = constants.False prober.active = False
self._mActiveNum -= 1 self._mActiveNum -= 1
if self._mActiveNum <= 0: if self._mActiveNum <= 0:
self._mState = constants.eNotMe self._mState = constants.eNotMe
@ -78,18 +84,22 @@ class CharSetGroupProber(CharSetProber):
bestConf = 0.0 bestConf = 0.0
self._mBestGuessProber = None self._mBestGuessProber = None
for prober in self._mProbers: for prober in self._mProbers:
if not prober: continue if not prober:
continue
if not prober.active: if not prober.active:
if constants._debug: if constants._debug:
sys.stderr.write(prober.get_charset_name() + ' not active\n') sys.stderr.write(prober.get_charset_name()
+ ' not active\n')
continue continue
cf = prober.get_confidence() cf = prober.get_confidence()
if constants._debug: if constants._debug:
sys.stderr.write('%s confidence = %s\n' % (prober.get_charset_name(), cf)) sys.stderr.write('%s confidence = %s\n' %
(prober.get_charset_name(), cf))
if bestConf < cf: if bestConf < cf:
bestConf = cf bestConf = cf
self._mBestGuessProber = prober self._mBestGuessProber = prober
if not self._mBestGuessProber: return 0.0 if not self._mBestGuessProber:
return 0.0
return bestConf return bestConf
# else: # else:
# self._mBestGuessProber = self._mProbers[0] # self._mBestGuessProber = self._mProbers[0]

24
libs/chardet/charsetprober.py

@ -1,11 +1,11 @@
######################## BEGIN LICENSE BLOCK ######################## ######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code. # The Original Code is Mozilla Universal charset detector code.
# #
# The Initial Developer of the Original Code is # The Initial Developer of the Original Code is
# Netscape Communications Corporation. # Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001 # Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved. # the Initial Developer. All Rights Reserved.
# #
# Contributor(s): # Contributor(s):
# Mark Pilgrim - port to Python # Mark Pilgrim - port to Python
# Shy Shalom - original C code # Shy Shalom - original C code
@ -14,27 +14,29 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
import constants, re from . import constants
import re
class CharSetProber: class CharSetProber:
def __init__(self): def __init__(self):
pass pass
def reset(self): def reset(self):
self._mState = constants.eDetecting self._mState = constants.eDetecting
def get_charset_name(self): def get_charset_name(self):
return None return None
@ -48,13 +50,13 @@ class CharSetProber:
return 0.0 return 0.0
def filter_high_bit_only(self, aBuf): def filter_high_bit_only(self, aBuf):
aBuf = re.sub(r'([\x00-\x7F])+', ' ', aBuf) aBuf = re.sub(b'([\x00-\x7F])+', b' ', aBuf)
return aBuf return aBuf
def filter_without_english_letters(self, aBuf): def filter_without_english_letters(self, aBuf):
aBuf = re.sub(r'([A-Za-z])+', ' ', aBuf) aBuf = re.sub(b'([A-Za-z])+', b' ', aBuf)
return aBuf return aBuf
def filter_with_english_letters(self, aBuf): def filter_with_english_letters(self, aBuf):
# TODO # TODO
return aBuf return aBuf

15
libs/chardet/codingstatemachine.py

@ -13,19 +13,21 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from constants import eStart, eError, eItsMe from .constants import eStart
from .compat import wrap_ord
class CodingStateMachine: class CodingStateMachine:
def __init__(self, sm): def __init__(self, sm):
@ -40,12 +42,15 @@ class CodingStateMachine:
def next_state(self, c): def next_state(self, c):
# for each byte we get its class # for each byte we get its class
# if it is first byte, we also get byte length # if it is first byte, we also get byte length
byteCls = self._mModel['classTable'][ord(c)] # PY3K: aBuf is a byte stream, so c is an int, not a byte
byteCls = self._mModel['classTable'][wrap_ord(c)]
if self._mCurrentState == eStart: if self._mCurrentState == eStart:
self._mCurrentBytePos = 0 self._mCurrentBytePos = 0
self._mCurrentCharLen = self._mModel['charLenTable'][byteCls] self._mCurrentCharLen = self._mModel['charLenTable'][byteCls]
# from byte's class and stateTable, we get its next state # from byte's class and stateTable, we get its next state
self._mCurrentState = self._mModel['stateTable'][self._mCurrentState * self._mModel['classFactor'] + byteCls] curr_state = (self._mCurrentState * self._mModel['classFactor']
+ byteCls)
self._mCurrentState = self._mModel['stateTable'][curr_state]
self._mCurrentBytePos += 1 self._mCurrentBytePos += 1
return self._mCurrentState return self._mCurrentState

34
libs/chardet/compat.py

@ -0,0 +1,34 @@
######################## BEGIN LICENSE BLOCK ########################
# Contributor(s):
# Ian Cordasco - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA
######################### END LICENSE BLOCK #########################
import sys
if sys.version_info < (3, 0):
base_str = (str, unicode)
else:
base_str = (bytes, str)
def wrap_ord(a):
if sys.version_info < (3, 0) and isinstance(a, base_str):
return ord(a)
else:
return a

8
libs/chardet/constants.py

@ -37,11 +37,3 @@ eError = 1
eItsMe = 2 eItsMe = 2
SHORTCUT_THRESHOLD = 0.95 SHORTCUT_THRESHOLD = 0.95
import __builtin__
if not hasattr(__builtin__, 'False'):
False = 0
True = 1
else:
False = __builtin__.False
True = __builtin__.True

44
libs/chardet/cp949prober.py

@ -0,0 +1,44 @@
######################## BEGIN LICENSE BLOCK ########################
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
# Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA
######################### END LICENSE BLOCK #########################
from .mbcharsetprober import MultiByteCharSetProber
from .codingstatemachine import CodingStateMachine
from .chardistribution import EUCKRDistributionAnalysis
from .mbcssm import CP949SMModel
class CP949Prober(MultiByteCharSetProber):
def __init__(self):
MultiByteCharSetProber.__init__(self)
self._mCodingSM = CodingStateMachine(CP949SMModel)
# NOTE: CP949 is a superset of EUC-KR, so the distribution should be
# not different.
self._mDistributionAnalyzer = EUCKRDistributionAnalysis()
self.reset()
def get_charset_name(self):
return "CP949"

39
libs/chardet/escprober.py

@ -13,39 +13,43 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
import constants, sys from . import constants
from escsm import HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel, ISO2022KRSMModel from .escsm import (HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel,
from charsetprober import CharSetProber ISO2022KRSMModel)
from codingstatemachine import CodingStateMachine from .charsetprober import CharSetProber
from .codingstatemachine import CodingStateMachine
from .compat import wrap_ord
class EscCharSetProber(CharSetProber): class EscCharSetProber(CharSetProber):
def __init__(self): def __init__(self):
CharSetProber.__init__(self) CharSetProber.__init__(self)
self._mCodingSM = [ \ self._mCodingSM = [
CodingStateMachine(HZSMModel), CodingStateMachine(HZSMModel),
CodingStateMachine(ISO2022CNSMModel), CodingStateMachine(ISO2022CNSMModel),
CodingStateMachine(ISO2022JPSMModel), CodingStateMachine(ISO2022JPSMModel),
CodingStateMachine(ISO2022KRSMModel) CodingStateMachine(ISO2022KRSMModel)
] ]
self.reset() self.reset()
def reset(self): def reset(self):
CharSetProber.reset(self) CharSetProber.reset(self)
for codingSM in self._mCodingSM: for codingSM in self._mCodingSM:
if not codingSM: continue if not codingSM:
codingSM.active = constants.True continue
codingSM.active = True
codingSM.reset() codingSM.reset()
self._mActiveSM = len(self._mCodingSM) self._mActiveSM = len(self._mCodingSM)
self._mDetectedCharset = None self._mDetectedCharset = None
@ -61,19 +65,22 @@ class EscCharSetProber(CharSetProber):
def feed(self, aBuf): def feed(self, aBuf):
for c in aBuf: for c in aBuf:
# PY3K: aBuf is a byte array, so c is an int, not a byte
for codingSM in self._mCodingSM: for codingSM in self._mCodingSM:
if not codingSM: continue if not codingSM:
if not codingSM.active: continue continue
codingState = codingSM.next_state(c) if not codingSM.active:
continue
codingState = codingSM.next_state(wrap_ord(c))
if codingState == constants.eError: if codingState == constants.eError:
codingSM.active = constants.False codingSM.active = False
self._mActiveSM -= 1 self._mActiveSM -= 1
if self._mActiveSM <= 0: if self._mActiveSM <= 0:
self._mState = constants.eNotMe self._mState = constants.eNotMe
return self.get_state() return self.get_state()
elif codingState == constants.eItsMe: elif codingState == constants.eItsMe:
self._mState = constants.eFoundIt self._mState = constants.eFoundIt
self._mDetectedCharset = codingSM.get_coding_state_machine() self._mDetectedCharset = codingSM.get_coding_state_machine() # nopep8
return self.get_state() return self.get_state()
return self.get_state() return self.get_state()

338
libs/chardet/escsm.py

@ -13,62 +13,62 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from constants import eStart, eError, eItsMe from .constants import eStart, eError, eItsMe
HZ_cls = ( \ HZ_cls = (
1,0,0,0,0,0,0,0, # 00 - 07 1,0,0,0,0,0,0,0, # 00 - 07
0,0,0,0,0,0,0,0, # 08 - 0f 0,0,0,0,0,0,0,0, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0,0,0,0,0,0,0,0, # 10 - 17
0,0,0,1,0,0,0,0, # 18 - 1f 0,0,0,1,0,0,0,0, # 18 - 1f
0,0,0,0,0,0,0,0, # 20 - 27 0,0,0,0,0,0,0,0, # 20 - 27
0,0,0,0,0,0,0,0, # 28 - 2f 0,0,0,0,0,0,0,0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0,0,0,0,0,0,0,0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0,0,0,0,0,0,0,0, # 38 - 3f
0,0,0,0,0,0,0,0, # 40 - 47 0,0,0,0,0,0,0,0, # 40 - 47
0,0,0,0,0,0,0,0, # 48 - 4f 0,0,0,0,0,0,0,0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0,0,0,0,0,0,0,0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0,0,0,0,0,0,0,0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0,0,0,0,0,0,0,0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0,0,0,0,0,0,0,0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0,0,0,0,0,0,0,0, # 70 - 77
0,0,0,4,0,5,2,0, # 78 - 7f 0,0,0,4,0,5,2,0, # 78 - 7f
1,1,1,1,1,1,1,1, # 80 - 87 1,1,1,1,1,1,1,1, # 80 - 87
1,1,1,1,1,1,1,1, # 88 - 8f 1,1,1,1,1,1,1,1, # 88 - 8f
1,1,1,1,1,1,1,1, # 90 - 97 1,1,1,1,1,1,1,1, # 90 - 97
1,1,1,1,1,1,1,1, # 98 - 9f 1,1,1,1,1,1,1,1, # 98 - 9f
1,1,1,1,1,1,1,1, # a0 - a7 1,1,1,1,1,1,1,1, # a0 - a7
1,1,1,1,1,1,1,1, # a8 - af 1,1,1,1,1,1,1,1, # a8 - af
1,1,1,1,1,1,1,1, # b0 - b7 1,1,1,1,1,1,1,1, # b0 - b7
1,1,1,1,1,1,1,1, # b8 - bf 1,1,1,1,1,1,1,1, # b8 - bf
1,1,1,1,1,1,1,1, # c0 - c7 1,1,1,1,1,1,1,1, # c0 - c7
1,1,1,1,1,1,1,1, # c8 - cf 1,1,1,1,1,1,1,1, # c8 - cf
1,1,1,1,1,1,1,1, # d0 - d7 1,1,1,1,1,1,1,1, # d0 - d7
1,1,1,1,1,1,1,1, # d8 - df 1,1,1,1,1,1,1,1, # d8 - df
1,1,1,1,1,1,1,1, # e0 - e7 1,1,1,1,1,1,1,1, # e0 - e7
1,1,1,1,1,1,1,1, # e8 - ef 1,1,1,1,1,1,1,1, # e8 - ef
1,1,1,1,1,1,1,1, # f0 - f7 1,1,1,1,1,1,1,1, # f0 - f7
1,1,1,1,1,1,1,1, # f8 - ff 1,1,1,1,1,1,1,1, # f8 - ff
) )
HZ_st = ( \ HZ_st = (
eStart,eError, 3,eStart,eStart,eStart,eError,eError,# 00-07 eStart,eError, 3,eStart,eStart,eStart,eError,eError,# 00-07
eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f
eItsMe,eItsMe,eError,eError,eStart,eStart, 4,eError,# 10-17 eItsMe,eItsMe,eError,eError,eStart,eStart, 4,eError,# 10-17
5,eError, 6,eError, 5, 5, 4,eError,# 18-1f 5,eError, 6,eError, 5, 5, 4,eError,# 18-1f
4,eError, 4, 4, 4,eError, 4,eError,# 20-27 4,eError, 4, 4, 4,eError, 4,eError,# 20-27
4,eItsMe,eStart,eStart,eStart,eStart,eStart,eStart,# 28-2f 4,eItsMe,eStart,eStart,eStart,eStart,eStart,eStart,# 28-2f
) )
HZCharLenTable = (0, 0, 0, 0, 0, 0) HZCharLenTable = (0, 0, 0, 0, 0, 0)
@ -79,50 +79,50 @@ HZSMModel = {'classTable': HZ_cls,
'charLenTable': HZCharLenTable, 'charLenTable': HZCharLenTable,
'name': "HZ-GB-2312"} 'name': "HZ-GB-2312"}
ISO2022CN_cls = ( \ ISO2022CN_cls = (
2,0,0,0,0,0,0,0, # 00 - 07 2,0,0,0,0,0,0,0, # 00 - 07
0,0,0,0,0,0,0,0, # 08 - 0f 0,0,0,0,0,0,0,0, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0,0,0,0,0,0,0,0, # 10 - 17
0,0,0,1,0,0,0,0, # 18 - 1f 0,0,0,1,0,0,0,0, # 18 - 1f
0,0,0,0,0,0,0,0, # 20 - 27 0,0,0,0,0,0,0,0, # 20 - 27
0,3,0,0,0,0,0,0, # 28 - 2f 0,3,0,0,0,0,0,0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0,0,0,0,0,0,0,0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0,0,0,0,0,0,0,0, # 38 - 3f
0,0,0,4,0,0,0,0, # 40 - 47 0,0,0,4,0,0,0,0, # 40 - 47
0,0,0,0,0,0,0,0, # 48 - 4f 0,0,0,0,0,0,0,0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0,0,0,0,0,0,0,0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0,0,0,0,0,0,0,0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0,0,0,0,0,0,0,0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0,0,0,0,0,0,0,0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0,0,0,0,0,0,0,0, # 70 - 77
0,0,0,0,0,0,0,0, # 78 - 7f 0,0,0,0,0,0,0,0, # 78 - 7f
2,2,2,2,2,2,2,2, # 80 - 87 2,2,2,2,2,2,2,2, # 80 - 87
2,2,2,2,2,2,2,2, # 88 - 8f 2,2,2,2,2,2,2,2, # 88 - 8f
2,2,2,2,2,2,2,2, # 90 - 97 2,2,2,2,2,2,2,2, # 90 - 97
2,2,2,2,2,2,2,2, # 98 - 9f 2,2,2,2,2,2,2,2, # 98 - 9f
2,2,2,2,2,2,2,2, # a0 - a7 2,2,2,2,2,2,2,2, # a0 - a7
2,2,2,2,2,2,2,2, # a8 - af 2,2,2,2,2,2,2,2, # a8 - af
2,2,2,2,2,2,2,2, # b0 - b7 2,2,2,2,2,2,2,2, # b0 - b7
2,2,2,2,2,2,2,2, # b8 - bf 2,2,2,2,2,2,2,2, # b8 - bf
2,2,2,2,2,2,2,2, # c0 - c7 2,2,2,2,2,2,2,2, # c0 - c7
2,2,2,2,2,2,2,2, # c8 - cf 2,2,2,2,2,2,2,2, # c8 - cf
2,2,2,2,2,2,2,2, # d0 - d7 2,2,2,2,2,2,2,2, # d0 - d7
2,2,2,2,2,2,2,2, # d8 - df 2,2,2,2,2,2,2,2, # d8 - df
2,2,2,2,2,2,2,2, # e0 - e7 2,2,2,2,2,2,2,2, # e0 - e7
2,2,2,2,2,2,2,2, # e8 - ef 2,2,2,2,2,2,2,2, # e8 - ef
2,2,2,2,2,2,2,2, # f0 - f7 2,2,2,2,2,2,2,2, # f0 - f7
2,2,2,2,2,2,2,2, # f8 - ff 2,2,2,2,2,2,2,2, # f8 - ff
) )
ISO2022CN_st = ( \ ISO2022CN_st = (
eStart, 3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07 eStart, 3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07
eStart,eError,eError,eError,eError,eError,eError,eError,# 08-0f eStart,eError,eError,eError,eError,eError,eError,eError,# 08-0f
eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17 eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17
eItsMe,eItsMe,eItsMe,eError,eError,eError, 4,eError,# 18-1f eItsMe,eItsMe,eItsMe,eError,eError,eError, 4,eError,# 18-1f
eError,eError,eError,eItsMe,eError,eError,eError,eError,# 20-27 eError,eError,eError,eItsMe,eError,eError,eError,eError,# 20-27
5, 6,eError,eError,eError,eError,eError,eError,# 28-2f 5, 6,eError,eError,eError,eError,eError,eError,# 28-2f
eError,eError,eError,eItsMe,eError,eError,eError,eError,# 30-37 eError,eError,eError,eItsMe,eError,eError,eError,eError,# 30-37
eError,eError,eError,eError,eError,eItsMe,eError,eStart,# 38-3f eError,eError,eError,eError,eError,eItsMe,eError,eStart,# 38-3f
) )
ISO2022CNCharLenTable = (0, 0, 0, 0, 0, 0, 0, 0, 0) ISO2022CNCharLenTable = (0, 0, 0, 0, 0, 0, 0, 0, 0)
@ -133,51 +133,51 @@ ISO2022CNSMModel = {'classTable': ISO2022CN_cls,
'charLenTable': ISO2022CNCharLenTable, 'charLenTable': ISO2022CNCharLenTable,
'name': "ISO-2022-CN"} 'name': "ISO-2022-CN"}
ISO2022JP_cls = ( \ ISO2022JP_cls = (
2,0,0,0,0,0,0,0, # 00 - 07 2,0,0,0,0,0,0,0, # 00 - 07
0,0,0,0,0,0,2,2, # 08 - 0f 0,0,0,0,0,0,2,2, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0,0,0,0,0,0,0,0, # 10 - 17
0,0,0,1,0,0,0,0, # 18 - 1f 0,0,0,1,0,0,0,0, # 18 - 1f
0,0,0,0,7,0,0,0, # 20 - 27 0,0,0,0,7,0,0,0, # 20 - 27
3,0,0,0,0,0,0,0, # 28 - 2f 3,0,0,0,0,0,0,0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0,0,0,0,0,0,0,0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0,0,0,0,0,0,0,0, # 38 - 3f
6,0,4,0,8,0,0,0, # 40 - 47 6,0,4,0,8,0,0,0, # 40 - 47
0,9,5,0,0,0,0,0, # 48 - 4f 0,9,5,0,0,0,0,0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0,0,0,0,0,0,0,0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0,0,0,0,0,0,0,0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0,0,0,0,0,0,0,0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0,0,0,0,0,0,0,0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0,0,0,0,0,0,0,0, # 70 - 77
0,0,0,0,0,0,0,0, # 78 - 7f 0,0,0,0,0,0,0,0, # 78 - 7f
2,2,2,2,2,2,2,2, # 80 - 87 2,2,2,2,2,2,2,2, # 80 - 87
2,2,2,2,2,2,2,2, # 88 - 8f 2,2,2,2,2,2,2,2, # 88 - 8f
2,2,2,2,2,2,2,2, # 90 - 97 2,2,2,2,2,2,2,2, # 90 - 97
2,2,2,2,2,2,2,2, # 98 - 9f 2,2,2,2,2,2,2,2, # 98 - 9f
2,2,2,2,2,2,2,2, # a0 - a7 2,2,2,2,2,2,2,2, # a0 - a7
2,2,2,2,2,2,2,2, # a8 - af 2,2,2,2,2,2,2,2, # a8 - af
2,2,2,2,2,2,2,2, # b0 - b7 2,2,2,2,2,2,2,2, # b0 - b7
2,2,2,2,2,2,2,2, # b8 - bf 2,2,2,2,2,2,2,2, # b8 - bf
2,2,2,2,2,2,2,2, # c0 - c7 2,2,2,2,2,2,2,2, # c0 - c7
2,2,2,2,2,2,2,2, # c8 - cf 2,2,2,2,2,2,2,2, # c8 - cf
2,2,2,2,2,2,2,2, # d0 - d7 2,2,2,2,2,2,2,2, # d0 - d7
2,2,2,2,2,2,2,2, # d8 - df 2,2,2,2,2,2,2,2, # d8 - df
2,2,2,2,2,2,2,2, # e0 - e7 2,2,2,2,2,2,2,2, # e0 - e7
2,2,2,2,2,2,2,2, # e8 - ef 2,2,2,2,2,2,2,2, # e8 - ef
2,2,2,2,2,2,2,2, # f0 - f7 2,2,2,2,2,2,2,2, # f0 - f7
2,2,2,2,2,2,2,2, # f8 - ff 2,2,2,2,2,2,2,2, # f8 - ff
) )
ISO2022JP_st = ( \ ISO2022JP_st = (
eStart, 3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07 eStart, 3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07
eStart,eStart,eError,eError,eError,eError,eError,eError,# 08-0f eStart,eStart,eError,eError,eError,eError,eError,eError,# 08-0f
eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17 eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17
eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,# 18-1f eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,# 18-1f
eError, 5,eError,eError,eError, 4,eError,eError,# 20-27 eError, 5,eError,eError,eError, 4,eError,eError,# 20-27
eError,eError,eError, 6,eItsMe,eError,eItsMe,eError,# 28-2f eError,eError,eError, 6,eItsMe,eError,eItsMe,eError,# 28-2f
eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,# 30-37 eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,# 30-37
eError,eError,eError,eItsMe,eError,eError,eError,eError,# 38-3f eError,eError,eError,eItsMe,eError,eError,eError,eError,# 38-3f
eError,eError,eError,eError,eItsMe,eError,eStart,eStart,# 40-47 eError,eError,eError,eError,eItsMe,eError,eStart,eStart,# 40-47
) )
ISO2022JPCharLenTable = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ISO2022JPCharLenTable = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
@ -188,47 +188,47 @@ ISO2022JPSMModel = {'classTable': ISO2022JP_cls,
'charLenTable': ISO2022JPCharLenTable, 'charLenTable': ISO2022JPCharLenTable,
'name': "ISO-2022-JP"} 'name': "ISO-2022-JP"}
ISO2022KR_cls = ( \ ISO2022KR_cls = (
2,0,0,0,0,0,0,0, # 00 - 07 2,0,0,0,0,0,0,0, # 00 - 07
0,0,0,0,0,0,0,0, # 08 - 0f 0,0,0,0,0,0,0,0, # 08 - 0f
0,0,0,0,0,0,0,0, # 10 - 17 0,0,0,0,0,0,0,0, # 10 - 17
0,0,0,1,0,0,0,0, # 18 - 1f 0,0,0,1,0,0,0,0, # 18 - 1f
0,0,0,0,3,0,0,0, # 20 - 27 0,0,0,0,3,0,0,0, # 20 - 27
0,4,0,0,0,0,0,0, # 28 - 2f 0,4,0,0,0,0,0,0, # 28 - 2f
0,0,0,0,0,0,0,0, # 30 - 37 0,0,0,0,0,0,0,0, # 30 - 37
0,0,0,0,0,0,0,0, # 38 - 3f 0,0,0,0,0,0,0,0, # 38 - 3f
0,0,0,5,0,0,0,0, # 40 - 47 0,0,0,5,0,0,0,0, # 40 - 47
0,0,0,0,0,0,0,0, # 48 - 4f 0,0,0,0,0,0,0,0, # 48 - 4f
0,0,0,0,0,0,0,0, # 50 - 57 0,0,0,0,0,0,0,0, # 50 - 57
0,0,0,0,0,0,0,0, # 58 - 5f 0,0,0,0,0,0,0,0, # 58 - 5f
0,0,0,0,0,0,0,0, # 60 - 67 0,0,0,0,0,0,0,0, # 60 - 67
0,0,0,0,0,0,0,0, # 68 - 6f 0,0,0,0,0,0,0,0, # 68 - 6f
0,0,0,0,0,0,0,0, # 70 - 77 0,0,0,0,0,0,0,0, # 70 - 77
0,0,0,0,0,0,0,0, # 78 - 7f 0,0,0,0,0,0,0,0, # 78 - 7f
2,2,2,2,2,2,2,2, # 80 - 87 2,2,2,2,2,2,2,2, # 80 - 87
2,2,2,2,2,2,2,2, # 88 - 8f 2,2,2,2,2,2,2,2, # 88 - 8f
2,2,2,2,2,2,2,2, # 90 - 97 2,2,2,2,2,2,2,2, # 90 - 97
2,2,2,2,2,2,2,2, # 98 - 9f 2,2,2,2,2,2,2,2, # 98 - 9f
2,2,2,2,2,2,2,2, # a0 - a7 2,2,2,2,2,2,2,2, # a0 - a7
2,2,2,2,2,2,2,2, # a8 - af 2,2,2,2,2,2,2,2, # a8 - af
2,2,2,2,2,2,2,2, # b0 - b7 2,2,2,2,2,2,2,2, # b0 - b7
2,2,2,2,2,2,2,2, # b8 - bf 2,2,2,2,2,2,2,2, # b8 - bf
2,2,2,2,2,2,2,2, # c0 - c7 2,2,2,2,2,2,2,2, # c0 - c7
2,2,2,2,2,2,2,2, # c8 - cf 2,2,2,2,2,2,2,2, # c8 - cf
2,2,2,2,2,2,2,2, # d0 - d7 2,2,2,2,2,2,2,2, # d0 - d7
2,2,2,2,2,2,2,2, # d8 - df 2,2,2,2,2,2,2,2, # d8 - df
2,2,2,2,2,2,2,2, # e0 - e7 2,2,2,2,2,2,2,2, # e0 - e7
2,2,2,2,2,2,2,2, # e8 - ef 2,2,2,2,2,2,2,2, # e8 - ef
2,2,2,2,2,2,2,2, # f0 - f7 2,2,2,2,2,2,2,2, # f0 - f7
2,2,2,2,2,2,2,2, # f8 - ff 2,2,2,2,2,2,2,2, # f8 - ff
) )
ISO2022KR_st = ( \ ISO2022KR_st = (
eStart, 3,eError,eStart,eStart,eStart,eError,eError,# 00-07 eStart, 3,eError,eStart,eStart,eStart,eError,eError,# 00-07
eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f
eItsMe,eItsMe,eError,eError,eError, 4,eError,eError,# 10-17 eItsMe,eItsMe,eError,eError,eError, 4,eError,eError,# 10-17
eError,eError,eError,eError, 5,eError,eError,eError,# 18-1f eError,eError,eError,eError, 5,eError,eError,eError,# 18-1f
eError,eError,eError,eItsMe,eStart,eStart,eStart,eStart,# 20-27 eError,eError,eError,eItsMe,eStart,eStart,eStart,eStart,# 20-27
) )
ISO2022KRCharLenTable = (0, 0, 0, 0, 0, 0) ISO2022KRCharLenTable = (0, 0, 0, 0, 0, 0)
@ -238,3 +238,5 @@ ISO2022KRSMModel = {'classTable': ISO2022KR_cls,
'stateTable': ISO2022KR_st, 'stateTable': ISO2022KR_st,
'charLenTable': ISO2022KRCharLenTable, 'charLenTable': ISO2022KRCharLenTable,
'name': "ISO-2022-KR"} 'name': "ISO-2022-KR"}
# flake8: noqa

45
libs/chardet/eucjpprober.py

@ -13,25 +13,26 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
import constants, sys import sys
from constants import eStart, eError, eItsMe from . import constants
from mbcharsetprober import MultiByteCharSetProber from .mbcharsetprober import MultiByteCharSetProber
from codingstatemachine import CodingStateMachine from .codingstatemachine import CodingStateMachine
from chardistribution import EUCJPDistributionAnalysis from .chardistribution import EUCJPDistributionAnalysis
from jpcntx import EUCJPContextAnalysis from .jpcntx import EUCJPContextAnalysis
from mbcssm import EUCJPSMModel from .mbcssm import EUCJPSMModel
class EUCJPProber(MultiByteCharSetProber): class EUCJPProber(MultiByteCharSetProber):
def __init__(self): def __init__(self):
@ -44,37 +45,41 @@ class EUCJPProber(MultiByteCharSetProber):
def reset(self): def reset(self):
MultiByteCharSetProber.reset(self) MultiByteCharSetProber.reset(self)
self._mContextAnalyzer.reset() self._mContextAnalyzer.reset()
def get_charset_name(self): def get_charset_name(self):
return "EUC-JP" return "EUC-JP"
def feed(self, aBuf): def feed(self, aBuf):
aLen = len(aBuf) aLen = len(aBuf)
for i in range(0, aLen): for i in range(0, aLen):
# PY3K: aBuf is a byte array, so aBuf[i] is an int, not a byte
codingState = self._mCodingSM.next_state(aBuf[i]) codingState = self._mCodingSM.next_state(aBuf[i])
if codingState == eError: if codingState == constants.eError:
if constants._debug: if constants._debug:
sys.stderr.write(self.get_charset_name() + ' prober hit error at byte ' + str(i) + '\n') sys.stderr.write(self.get_charset_name()
+ ' prober hit error at byte ' + str(i)
+ '\n')
self._mState = constants.eNotMe self._mState = constants.eNotMe
break break
elif codingState == eItsMe: elif codingState == constants.eItsMe:
self._mState = constants.eFoundIt self._mState = constants.eFoundIt
break break
elif codingState == eStart: elif codingState == constants.eStart:
charLen = self._mCodingSM.get_current_charlen() charLen = self._mCodingSM.get_current_charlen()
if i == 0: if i == 0:
self._mLastChar[1] = aBuf[0] self._mLastChar[1] = aBuf[0]
self._mContextAnalyzer.feed(self._mLastChar, charLen) self._mContextAnalyzer.feed(self._mLastChar, charLen)
self._mDistributionAnalyzer.feed(self._mLastChar, charLen) self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
else: else:
self._mContextAnalyzer.feed(aBuf[i-1:i+1], charLen) self._mContextAnalyzer.feed(aBuf[i - 1:i + 1], charLen)
self._mDistributionAnalyzer.feed(aBuf[i-1:i+1], charLen) self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
charLen)
self._mLastChar[0] = aBuf[aLen - 1] self._mLastChar[0] = aBuf[aLen - 1]
if self.get_state() == constants.eDetecting: if self.get_state() == constants.eDetecting:
if self._mContextAnalyzer.got_enough_data() and \ if (self._mContextAnalyzer.got_enough_data() and
(self.get_confidence() > constants.SHORTCUT_THRESHOLD): (self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
self._mState = constants.eFoundIt self._mState = constants.eFoundIt
return self.get_state() return self.get_state()

2
libs/chardet/euckrfreq.py

@ -592,3 +592,5 @@ EUCKRCharToFreqOrder = ( \
8704,8705,8706,8707,8708,8709,8710,8711,8712,8713,8714,8715,8716,8717,8718,8719, 8704,8705,8706,8707,8708,8709,8710,8711,8712,8713,8714,8715,8716,8717,8718,8719,
8720,8721,8722,8723,8724,8725,8726,8727,8728,8729,8730,8731,8732,8733,8734,8735, 8720,8721,8722,8723,8724,8725,8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,
8736,8737,8738,8739,8740,8741) 8736,8737,8738,8739,8740,8741)
# flake8: noqa

13
libs/chardet/euckrprober.py

@ -13,22 +13,23 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from mbcharsetprober import MultiByteCharSetProber from .mbcharsetprober import MultiByteCharSetProber
from codingstatemachine import CodingStateMachine from .codingstatemachine import CodingStateMachine
from chardistribution import EUCKRDistributionAnalysis from .chardistribution import EUCKRDistributionAnalysis
from mbcssm import EUCKRSMModel from .mbcssm import EUCKRSMModel
class EUCKRProber(MultiByteCharSetProber): class EUCKRProber(MultiByteCharSetProber):
def __init__(self): def __init__(self):

16
libs/chardet/euctwfreq.py

@ -13,12 +13,12 @@
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either # License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version. # version 2.1 of the License, or (at your option) any later version.
# #
# This library is distributed in the hope that it will be useful, # This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details. # Lesser General Public License for more details.
# #
# You should have received a copy of the GNU Lesser General Public # You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software # License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
@ -26,8 +26,8 @@
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
# EUCTW frequency table # EUCTW frequency table
# Converted from big5 work # Converted from big5 work
# by Taiwan's Mandarin Promotion Council # by Taiwan's Mandarin Promotion Council
# <http:#www.edu.tw:81/mandr/> # <http:#www.edu.tw:81/mandr/>
# 128 --> 0.42261 # 128 --> 0.42261
@ -38,15 +38,15 @@
# #
# Idea Distribution Ratio = 0.74851/(1-0.74851) =2.98 # Idea Distribution Ratio = 0.74851/(1-0.74851) =2.98
# Random Distribution Ration = 512/(5401-512)=0.105 # Random Distribution Ration = 512/(5401-512)=0.105
# #
# Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR # Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR
EUCTW_TYPICAL_DISTRIBUTION_RATIO = 0.75 EUCTW_TYPICAL_DISTRIBUTION_RATIO = 0.75
# Char to FreqOrder table , # Char to FreqOrder table ,
EUCTW_TABLE_SIZE = 8102 EUCTW_TABLE_SIZE = 8102
EUCTWCharToFreqOrder = ( \ EUCTWCharToFreqOrder = (
1,1800,1506, 255,1431, 198, 9, 82, 6,7310, 177, 202,3615,1256,2808, 110, # 2742 1,1800,1506, 255,1431, 198, 9, 82, 6,7310, 177, 202,3615,1256,2808, 110, # 2742
3735, 33,3241, 261, 76, 44,2113, 16,2931,2184,1176, 659,3868, 26,3404,2643, # 2758 3735, 33,3241, 261, 76, 44,2113, 16,2931,2184,1176, 659,3868, 26,3404,2643, # 2758
1198,3869,3313,4060, 410,2211, 302, 590, 361,1963, 8, 204, 58,4296,7311,1931, # 2774 1198,3869,3313,4060, 410,2211, 302, 590, 361,1963, 8, 204, 58,4296,7311,1931, # 2774
@ -424,3 +424,5 @@ EUCTWCharToFreqOrder = ( \
8694,8695,8696,8697,8698,8699,8700,8701,8702,8703,8704,8705,8706,8707,8708,8709, # 8710 8694,8695,8696,8697,8698,8699,8700,8701,8702,8703,8704,8705,8706,8707,8708,8709, # 8710
8710,8711,8712,8713,8714,8715,8716,8717,8718,8719,8720,8721,8722,8723,8724,8725, # 8726 8710,8711,8712,8713,8714,8715,8716,8717,8718,8719,8720,8721,8722,8723,8724,8725, # 8726
8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,8736,8737,8738,8739,8740,8741) # 8742 8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,8736,8737,8738,8739,8740,8741) # 8742
# flake8: noqa

8
libs/chardet/euctwprober.py

@ -25,10 +25,10 @@
# 02110-1301 USA # 02110-1301 USA
######################### END LICENSE BLOCK ######################### ######################### END LICENSE BLOCK #########################
from mbcharsetprober import MultiByteCharSetProber from .mbcharsetprober import MultiByteCharSetProber
from codingstatemachine import CodingStateMachine from .codingstatemachine import CodingStateMachine
from chardistribution import EUCTWDistributionAnalysis from .chardistribution import EUCTWDistributionAnalysis
from mbcssm import EUCTWSMModel from .mbcssm import EUCTWSMModel
class EUCTWProber(MultiByteCharSetProber): class EUCTWProber(MultiByteCharSetProber):
def __init__(self): def __init__(self):

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save