Ruud
14 years ago
27 changed files with 863 additions and 225 deletions
@ -0,0 +1,6 @@ |
|||
from .main import Scanner |
|||
|
|||
def start(): |
|||
return Scanner() |
|||
|
|||
config = [] |
|||
@ -0,0 +1,477 @@ |
|||
from couchpotato import get_session |
|||
from couchpotato.core.event import fireEvent, addEvent |
|||
from couchpotato.core.helpers.encoding import toUnicode |
|||
from couchpotato.core.helpers.variable import getExt |
|||
from couchpotato.core.logger import CPLog |
|||
from couchpotato.core.plugins.base import Plugin |
|||
from couchpotato.core.settings.model import File, Library, Release, Movie |
|||
from couchpotato.environment import Env |
|||
from flask.helpers import json |
|||
from themoviedb.tmdb import opensubtitleHashFile |
|||
import os |
|||
import re |
|||
import subprocess |
|||
import traceback |
|||
|
|||
log = CPLog(__name__) |
|||
|
|||
|
|||
class Scanner(Plugin): |
|||
|
|||
minimal_filesize = { |
|||
'media': 314572800, # 300MB |
|||
'trailer': 1048576, # 1MB |
|||
} |
|||
ignored_in_path = ['_unpack', '_failed_', '_unknown_', '_exists_', '.appledouble', '.appledb', '.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo'] #unpacking, smb-crap, hidden files |
|||
ignore_names = ['extract', 'extracting', 'extracted', 'movie', 'movies', 'film', 'films'] |
|||
extensions = { |
|||
'movie': ['mkv', 'wmv', 'avi', 'mpg', 'mpeg', 'mp4', 'm2ts', 'iso', 'img'], |
|||
'dvd': ['vts_*', 'vob'], |
|||
'nfo': ['nfo', 'txt', 'tag'], |
|||
'subtitle': ['sub', 'srt', 'ssa', 'ass'], |
|||
'subtitle_extra': ['idx'], |
|||
'trailer': ['mov', 'mp4', 'flv'] |
|||
} |
|||
file_types = { |
|||
'subtitle': ('subtitle', 'subtitle'), |
|||
'trailer': ('video', 'trailer'), |
|||
'nfo': ('nfo', 'nfo'), |
|||
'movie': ('video', 'movie'), |
|||
'backdrop': ('image', 'backdrop'), |
|||
} |
|||
|
|||
codecs = { |
|||
'audio': ['dts', 'ac3', 'ac3d', 'mp3'], |
|||
'video': ['x264', 'divx', 'xvid'] |
|||
} |
|||
|
|||
source_media = { |
|||
'bluray': ['bluray', 'blu-ray', 'brrip', 'br-rip'], |
|||
'hddvd': ['hddvd', 'hd-dvd'], |
|||
'dvd': ['dvd'], |
|||
'hdtv': ['hdtv'] |
|||
} |
|||
|
|||
clean = '(?i)[^\s](ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip|hdtvrip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|cd[1-9]|\[.*\])[^\s]*' |
|||
multipart_regex = [ |
|||
'[ _\.-]+cd[ _\.-]*([0-9a-d]+)', #*cd1 |
|||
'[ _\.-]+dvd[ _\.-]*([0-9a-d]+)', #*dvd1 |
|||
'[ _\.-]+part[ _\.-]*([0-9a-d]+)', #*part1.mkv |
|||
'[ _\.-]+dis[ck][ _\.-]*([0-9a-d]+)', #*disk1.mkv |
|||
'()[ _\.-]+([0-9]*[abcd]+)(\.....?)$', |
|||
'([a-z])([0-9]+)(\.....?)$', |
|||
'()([ab])(\.....?)$' #*a.mkv |
|||
] |
|||
|
|||
def __init__(self): |
|||
|
|||
addEvent('app.load', self.scan) |
|||
|
|||
def scan(self, folder = '/Volumes/Media/Test/'): |
|||
|
|||
""" |
|||
Get all files |
|||
|
|||
For each file larger then 350MB |
|||
create movie "group", this is where all movie files will be grouped |
|||
group multipart together |
|||
check if its DVD (VIDEO_TS) |
|||
|
|||
# This should work for non-folder based structure |
|||
for each moviegroup |
|||
|
|||
for each file smaller then 350MB, allfiles.filter(moviename*) |
|||
|
|||
# Assuming the beginning of the filename is the same for this structure |
|||
Movie is masterfile, moviename-cd1.ext -> moviename |
|||
Find other files connected to moviename, moviename*.nfo, moviename*.sub, moviename*trailer.ext |
|||
|
|||
Remove found file from allfiles |
|||
|
|||
# This should work for folder based structure |
|||
for each leftover file |
|||
Loop over leftover files, use dirname as moviename |
|||
|
|||
|
|||
For each found movie |
|||
|
|||
determine filetype |
|||
|
|||
Check if it's already in the db |
|||
|
|||
Add it to database |
|||
""" |
|||
|
|||
# Get movie "master" files |
|||
movie_files = {} |
|||
leftovers = [] |
|||
for root, dirs, files in os.walk(folder): |
|||
for filename in files: |
|||
|
|||
file_path = os.path.join(root, filename) |
|||
|
|||
# Remove ignored files |
|||
if not self.keepFile(file_path): |
|||
continue |
|||
|
|||
is_dvd_file = self.isDVDFile(file_path) |
|||
if os.path.getsize(file_path) > self.minimal_filesize['media'] or is_dvd_file: # Minimal 300MB files or is DVD file |
|||
|
|||
identifier = self.createFileIdentifier(file_path, folder, exclude_filename = is_dvd_file) |
|||
|
|||
if not movie_files.get(identifier): |
|||
movie_files[identifier] = { |
|||
'unsorted_files': [], |
|||
'identifiers': [], |
|||
'is_dvd': is_dvd_file, |
|||
} |
|||
|
|||
movie_files[identifier]['unsorted_files'].append(file_path) |
|||
else: |
|||
leftovers.append(file_path) |
|||
|
|||
# Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2" |
|||
# files will be grouped first. |
|||
leftovers = set(sorted(leftovers, reverse = True)) |
|||
|
|||
id_handles = [ |
|||
None, # Attach files to group by identifier |
|||
lambda x: os.path.split(x)[-1], # Attach files via filename of master_file name only |
|||
os.path.dirname, # Attach files via master_file dirname |
|||
] |
|||
|
|||
# Create identifier based on handle |
|||
for handler in id_handles: |
|||
for identifier, group in movie_files.iteritems(): |
|||
identifier = handler(identifier) if handler else identifier |
|||
if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) |
|||
|
|||
# Group the files based on the identifier |
|||
found_files = self.getGroupFiles(identifier, folder, leftovers) |
|||
group['unsorted_files'].extend(found_files) |
|||
|
|||
# Remove the found files from the leftover stack |
|||
leftovers = leftovers - found_files |
|||
|
|||
# Open up the db |
|||
db = get_session() |
|||
|
|||
# Mark all files as "offline" before a adding them to the database (again) |
|||
files_in_path = db.query(File).filter(File.path.like(toUnicode(folder) + u'%%')) |
|||
files_in_path.update({'available': 0}, synchronize_session = False) |
|||
db.commit() |
|||
|
|||
# Determine file types |
|||
update_after = [] |
|||
for identifier, group in movie_files.iteritems(): |
|||
|
|||
# Group extra (and easy) files first |
|||
images = self.getImages(group['unsorted_files']) |
|||
group['files'] = { |
|||
'subtitle': self.getSubtitles(group['unsorted_files']), |
|||
'nfo': self.getNfo(group['unsorted_files']), |
|||
'trailer': self.getTrailers(group['unsorted_files']), |
|||
'backdrop': images['backdrop'], |
|||
'leftover': set(group['unsorted_files']), |
|||
} |
|||
|
|||
# Media files |
|||
if group['is_dvd']: |
|||
group['files']['movie'] = self.getDVDFiles(group['unsorted_files']) |
|||
else: |
|||
group['files']['movie'] = self.getMediaFiles(group['unsorted_files']) |
|||
group['meta_data'] = self.getMetaData(group['files']['movie']) |
|||
|
|||
# Leftover "sorted" files |
|||
for type in group['files']: |
|||
group['files']['leftover'] -= set(group['files'][type]) |
|||
|
|||
# Delete the unsorted list |
|||
del group['unsorted_files'] |
|||
|
|||
# Determine movie |
|||
group['library'] = self.determineMovie(group) |
|||
|
|||
# Save to DB |
|||
if group['library']: |
|||
#library = db.query(Library).filter_by(id = library.get('id')).one() |
|||
|
|||
# Add release |
|||
release = self.addRelease(group) |
|||
return |
|||
|
|||
# Add identifier for library update |
|||
update_after.append(group['library'].get('identifier')) |
|||
|
|||
for identifier in update_after: |
|||
fireEvent('library.update', identifier = identifier) |
|||
|
|||
# If cleanup option is enabled, remove offline files from database |
|||
if self.conf('cleanup_offline'): |
|||
files_in_path = db.query(File).filter(File.path.like(folder + '%%')).filter_by(available = 0) |
|||
[db.delete(x) for x in files_in_path] |
|||
db.commit() |
|||
|
|||
db.remove() |
|||
|
|||
|
|||
def addRelease(self, group): |
|||
db = get_session() |
|||
|
|||
identifier = '%s.%s.%s' % (group['library']['identifier'], group['meta_data']['audio'], group['meta_data']['quality']) |
|||
|
|||
# Add movie |
|||
done_status = fireEvent('status.get', 'done', single = True) |
|||
movie = db.query(Movie).filter_by(library_id = group['library'].get('id')).first() |
|||
if not movie: |
|||
movie = Movie( |
|||
library_id = group['library'].get('id'), |
|||
profile_id = 0, |
|||
status_id = done_status.get('id') |
|||
) |
|||
db.add(movie) |
|||
db.commit() |
|||
|
|||
# Add release |
|||
quality = fireEvent('quality.single', group['meta_data']['quality'], single = True) |
|||
release = db.query(Release).filter_by(identifier = identifier).first() |
|||
if not release: |
|||
release = Release( |
|||
identifier = identifier, |
|||
movie = movie, |
|||
quality_id = quality.get('id'), |
|||
status_id = done_status.get('id') |
|||
) |
|||
db.add(release) |
|||
db.commit() |
|||
|
|||
# Add each file type |
|||
for type in group['files']: |
|||
|
|||
for file in group['files'][type]: |
|||
added_file = self.saveFile(file, type = type, include_media_info = type is 'movie') |
|||
try: |
|||
added_file = db.query(File).filter_by(id = added_file.get('id')).one() |
|||
release.files.append(added_file) |
|||
db.commit() |
|||
except Exception, e: |
|||
log.debug('Failed to attach "%s" to release: %s' % (file, e)) |
|||
|
|||
db.remove() |
|||
|
|||
def getMetaData(self, files): |
|||
|
|||
return { |
|||
'audio': 'AC3', |
|||
'quality': '720p', |
|||
'quality_type': 'HD', |
|||
'resolution_width': 1280, |
|||
'resolution_height': 720 |
|||
} |
|||
|
|||
for file in files: |
|||
self.getMeta(file) |
|||
|
|||
def getMeta(self, filename): |
|||
lib_dir = os.path.join(Env.get('app_dir'), 'libs') |
|||
script = os.path.join(lib_dir, 'getmeta.py') |
|||
|
|||
p = subprocess.Popen(["python", script, filename], stdout = subprocess.PIPE, stderr = subprocess.PIPE, cwd = lib_dir) |
|||
z = p.communicate()[0] |
|||
|
|||
try: |
|||
meta = json.loads(z) |
|||
log.info('Retrieved metainfo: %s' % meta) |
|||
return meta |
|||
except Exception, e: |
|||
print e |
|||
log.error('Couldn\'t get metadata from file') |
|||
|
|||
def determineMovie(self, group): |
|||
imdb_id = None |
|||
|
|||
files = group['files'] |
|||
# Check and see if nfo contains the imdb-id |
|||
try: |
|||
for nfo_file in files['nfo']: |
|||
imdb_id = self.getImdb(nfo_file) |
|||
if imdb_id: break |
|||
except: |
|||
pass |
|||
|
|||
# Check if path is already in db |
|||
db = get_session() |
|||
for file in files['movie']: |
|||
f = db.query(File).filter_by(path = toUnicode(file)).first() |
|||
try: |
|||
imdb_id = f.library[0].identifier |
|||
break |
|||
except: |
|||
pass |
|||
db.remove() |
|||
|
|||
# Search based on identifiers |
|||
if not imdb_id: |
|||
for identifier in group['identifiers']: |
|||
if len(identifier) > 2: |
|||
movie = fireEvent('provider.movie.search', q = identifier, merge = True, limit = 1) |
|||
if len(movie) > 0: |
|||
imdb_id = movie[0]['imdb'] |
|||
if imdb_id: break |
|||
else: |
|||
log.debug('Identifier to short to use for search: %s' % identifier) |
|||
|
|||
if imdb_id: |
|||
#movie = fireEvent('provider.movie.info', identifier = imdb_id, merge = True) |
|||
#if movie and movie.get('imdb'): |
|||
return fireEvent('library.add', attrs = { |
|||
'identifier': imdb_id |
|||
}, update_after = False, single = True) |
|||
|
|||
log.error('No imdb_id found for %s.' % group['identifiers']) |
|||
return False |
|||
|
|||
def saveFile(self, file, type = 'unknown', include_media_info = False): |
|||
|
|||
properties = {} |
|||
|
|||
# Get media info for files |
|||
if include_media_info: |
|||
properties = {} |
|||
|
|||
# Check database and update/insert if necessary |
|||
return fireEvent('file.add', path = file, part = self.getPartNumber(file), type = self.file_types[type], properties = properties, single = True) |
|||
|
|||
def getImdb(self, txt): |
|||
|
|||
if os.path.isfile(txt): |
|||
output = open(txt, 'r') |
|||
txt = output.read() |
|||
output.close() |
|||
|
|||
try: |
|||
m = re.search('(?P<id>tt[0-9{7}]+)', txt) |
|||
id = m.group('id') |
|||
if id: return id |
|||
except AttributeError: |
|||
pass |
|||
|
|||
return False |
|||
|
|||
def getMediaFiles(self, files): |
|||
|
|||
def test(s): |
|||
return self.filesizeBetween(s, 300, 100000) and getExt(s.lower()) in self.extensions['movie'] |
|||
|
|||
return set(filter(test, files)) |
|||
|
|||
def getDVDFiles(self, files): |
|||
|
|||
def test(s): |
|||
return self.isDVDFile(s) |
|||
|
|||
return set(filter(test, files)) |
|||
|
|||
def getSubtitles(self, files): |
|||
return set(filter(lambda s: getExt(s.lower()) in self.extensions['subtitle'], files)) |
|||
|
|||
def getNfo(self, files): |
|||
return set(filter(lambda s: getExt(s.lower()) in self.extensions['nfo'], files)) |
|||
|
|||
def getTrailers(self, files): |
|||
|
|||
def test(s): |
|||
return re.search('(^|[\W_])trailer\d*[\W_]', s.lower()) and self.filesizeBetween(s, 2, 250) |
|||
|
|||
return set(filter(test, files)) |
|||
|
|||
def getImages(self, files): |
|||
|
|||
def test(s): |
|||
return getExt(s.lower()) in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tbn'] |
|||
files = set(filter(test, files)) |
|||
|
|||
images = {} |
|||
|
|||
# Fanart |
|||
images['backdrop'] = set(filter(lambda s: re.search('(^|[\W_])fanart|backdrop\d*[\W_]', s.lower()) and self.filesizeBetween(s, 0, 5), files)) |
|||
|
|||
# Rest |
|||
images['rest'] = files - images['backdrop'] |
|||
|
|||
return images |
|||
|
|||
|
|||
def isDVDFile(self, file): |
|||
|
|||
if list(set(file.lower().split(os.path.sep)) & set(['video_ts', 'audio_ts'])): |
|||
return True |
|||
|
|||
for needle in ['vts_', 'video_ts', 'audio_ts']: |
|||
if needle in file.lower(): |
|||
return True |
|||
|
|||
return False |
|||
|
|||
def keepFile(self, file): |
|||
|
|||
# ignoredpaths |
|||
for i in self.ignored_in_path: |
|||
if i in file.lower(): |
|||
log.debug('Ignored "%s" contains "%s".' % (file, i)) |
|||
return False |
|||
|
|||
# Sample file |
|||
if re.search('(^|[\W_])sample\d*[\W_]', file.lower()): |
|||
log.debug('Is sample file "%s".' % file) |
|||
return False |
|||
|
|||
# Minimal size |
|||
if self.filesizeBetween(file, self.minimal_filesize['media']): |
|||
log.debug('File to small: %s' % file) |
|||
return False |
|||
|
|||
# All is OK |
|||
return True |
|||
|
|||
|
|||
def filesizeBetween(self, file, min = 0, max = 100000): |
|||
try: |
|||
return (min * 1048576) < os.path.getsize(file) < (max * 1048576) |
|||
except: |
|||
log.error('Couldn\'t get filesize of %s.' % file) |
|||
|
|||
return False |
|||
|
|||
def getGroupFiles(self, identifier, folder, file_pile): |
|||
return set(filter(lambda s:identifier in self.createFileIdentifier(s, folder), file_pile)) |
|||
|
|||
def createFileIdentifier(self, file_path, folder, exclude_filename = False): |
|||
identifier = file_path.replace(folder, '') # root folder |
|||
identifier = os.path.splitext(identifier)[0] # ext |
|||
if exclude_filename: |
|||
identifier = identifier[:len(identifier) - len(os.path.split(identifier)[-1])] |
|||
identifier = self.removeMultipart(identifier) # multipart |
|||
|
|||
return identifier |
|||
|
|||
def removeMultipart(self, name): |
|||
for regex in self.multipart_regex: |
|||
try: |
|||
found = re.sub(regex, '', name) |
|||
if found != name: |
|||
return found |
|||
except: |
|||
pass |
|||
return name |
|||
|
|||
def getPartNumber(self, name): |
|||
for regex in self.multipart_regex: |
|||
try: |
|||
found = re.search(regex, name) |
|||
if found: |
|||
return found.group(1) |
|||
return 1 |
|||
except: |
|||
pass |
|||
return name |
|||
Loading…
Reference in new issue