Joel Kåberg
12 years ago
23 changed files with 1122 additions and 374 deletions
@ -0,0 +1,6 @@ |
|||
from .main import Matcher |
|||
|
|||
def start(): |
|||
return Matcher() |
|||
|
|||
config = [] |
|||
@ -0,0 +1,109 @@ |
|||
from caper import Caper |
|||
from couchpotato import CPLog, tryInt |
|||
from couchpotato.core.event import addEvent, fireEvent |
|||
from couchpotato.core.helpers.encoding import simplifyString |
|||
from couchpotato.core.helpers.variable import possibleTitles, dictIsSubset |
|||
from couchpotato.core.plugins.base import Plugin |
|||
|
|||
log = CPLog(__name__) |
|||
|
|||
|
|||
class Matcher(Plugin): |
|||
def __init__(self): |
|||
self.caper = Caper() |
|||
|
|||
addEvent('matcher.parse', self.parse) |
|||
addEvent('matcher.best', self.best) |
|||
|
|||
addEvent('matcher.correct_title', self.correctTitle) |
|||
addEvent('matcher.correct_identifier', self.correctIdentifier) |
|||
addEvent('matcher.correct_quality', self.correctQuality) |
|||
|
|||
def parse(self, release): |
|||
return self.caper.parse(release['name']) |
|||
|
|||
def best(self, release, media, quality): |
|||
rel_info = fireEvent('matcher.parse', release, single = True) |
|||
|
|||
if len(rel_info.chains) < 1: |
|||
log.info2('Wrong: %s, unable to parse release name (no chains)', release['name']) |
|||
return False |
|||
|
|||
for chain in rel_info.chains: |
|||
if fireEvent('searcher.correct_match', chain, release, media, quality, single = True): |
|||
return chain |
|||
|
|||
return None |
|||
|
|||
def chainMatch(self, chain, group, tags): |
|||
found_tags = [] |
|||
|
|||
for match in chain.info[group]: |
|||
for ck, cv in match.items(): |
|||
if ck in tags and simplifyString(cv) in tags[ck]: |
|||
found_tags.append(ck) |
|||
|
|||
|
|||
if set(tags.keys()) == set(found_tags): |
|||
return True |
|||
|
|||
return set([key for key, value in tags.items() if None not in value]) == set(found_tags) |
|||
|
|||
def correctIdentifier(self, chain, media): |
|||
required_id = fireEvent('searcher.get_media_identifier', media['library'], single = True) |
|||
|
|||
if 'identifier' not in chain.info: |
|||
return False |
|||
|
|||
# TODO could be handled better? |
|||
if len(chain.info['identifier']) != 1: |
|||
return False |
|||
identifier = chain.info['identifier'][0] |
|||
|
|||
# TODO air by date episodes |
|||
|
|||
# TODO this should support identifiers with characters 'a', 'b', etc.. |
|||
for k, v in identifier.items(): |
|||
identifier[k] = tryInt(v, None) |
|||
|
|||
if not dictIsSubset(required_id, identifier): |
|||
log.info2('Wrong: required identifier %s does not match release identifier %s', (str(required_id), str(identifier))) |
|||
return False |
|||
|
|||
return True |
|||
|
|||
def correctTitle(self, chain, media): |
|||
root_library = fireEvent('searcher.get_media_root', media['library'], single = True) |
|||
|
|||
if 'show_name' not in chain.info or not len(chain.info['show_name']): |
|||
log.info('Wrong: missing show name in parsed result') |
|||
return False |
|||
|
|||
chain_words = [x.lower() for x in chain.info['show_name']] |
|||
|
|||
# Check show titles match |
|||
# TODO check xem names |
|||
for title in root_library['info']['titles']: |
|||
for valid_words in [x.split(' ') for x in possibleTitles(title)]: |
|||
|
|||
if valid_words == chain_words: |
|||
return True |
|||
|
|||
return False |
|||
|
|||
def correctQuality(self, chain, quality, quality_map): |
|||
if quality['identifier'] not in quality_map: |
|||
log.info2('Wrong: unknown preferred quality %s', quality['identifier']) |
|||
return False |
|||
|
|||
if 'video' not in chain.info: |
|||
log.info2('Wrong: no video tags found') |
|||
return False |
|||
|
|||
video_tags = quality_map[quality['identifier']] |
|||
|
|||
if not self.chainMatch(chain, 'video', video_tags): |
|||
log.info2('Wrong: %s tags not in chain', video_tags) |
|||
return False |
|||
|
|||
return True |
|||
@ -0,0 +1,42 @@ |
|||
# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
|
|||
|
|||
from qcond.transformers.merge import MergeTransformer |
|||
from qcond.transformers.slice import SliceTransformer |
|||
from qcond.transformers.strip_common import StripCommonTransformer |
|||
|
|||
|
|||
__version_info__ = ('0', '1', '0') |
|||
__version_branch__ = 'master' |
|||
|
|||
__version__ = "%s%s" % ( |
|||
'.'.join(__version_info__), |
|||
'-' + __version_branch__ if __version_branch__ else '' |
|||
) |
|||
|
|||
|
|||
class QueryCondenser(object): |
|||
def __init__(self): |
|||
self.transformers = [ |
|||
MergeTransformer(), |
|||
SliceTransformer(), |
|||
StripCommonTransformer() |
|||
] |
|||
|
|||
def distinct(self, titles): |
|||
for transformer in self.transformers: |
|||
titles = transformer.run(titles) |
|||
|
|||
return titles |
|||
@ -0,0 +1,23 @@ |
|||
# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
|
|||
|
|||
import sys |
|||
|
|||
PY3 = sys.version_info[0] == 3 |
|||
|
|||
if PY3: |
|||
xrange = range |
|||
else: |
|||
xrange = xrange |
|||
@ -0,0 +1,84 @@ |
|||
# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
|
|||
|
|||
from difflib import SequenceMatcher |
|||
import re |
|||
import sys |
|||
from logr import Logr |
|||
from qcond.compat import xrange |
|||
|
|||
|
|||
PY3 = sys.version_info[0] == 3 |
|||
|
|||
|
|||
def simplify(s): |
|||
s = s.lower() |
|||
s = re.sub(r"(\w)'(\w)", r"\1\2", s) |
|||
return s |
|||
|
|||
|
|||
def strip(s): |
|||
return re.sub(r"^(\W*)(.*?)(\W*)$", r"\2", s) |
|||
|
|||
|
|||
def create_matcher(a, b, swap_longest = True, case_sensitive = False): |
|||
# Ensure longest string is a |
|||
if swap_longest and len(b) > len(a): |
|||
a_ = a |
|||
a = b |
|||
b = a_ |
|||
|
|||
if not case_sensitive: |
|||
a = a.upper() |
|||
b = b.upper() |
|||
|
|||
return SequenceMatcher(None, a, b) |
|||
|
|||
|
|||
def first(function_or_none, sequence): |
|||
if PY3: |
|||
for item in filter(function_or_none, sequence): |
|||
return item |
|||
else: |
|||
result = filter(function_or_none, sequence) |
|||
if len(result): |
|||
return result[0] |
|||
|
|||
return None |
|||
|
|||
def sorted_append(sequence, item, func): |
|||
if not len(sequence): |
|||
sequence.insert(0, item) |
|||
return |
|||
|
|||
x = 0 |
|||
for x in xrange(len(sequence)): |
|||
if func(sequence[x]): |
|||
sequence.insert(x, item) |
|||
return |
|||
|
|||
sequence.append(item) |
|||
|
|||
def itemsMatch(L1, L2): |
|||
return len(L1) == len(L2) and sorted(L1) == sorted(L2) |
|||
|
|||
def distinct(sequence): |
|||
result = [] |
|||
|
|||
for item in sequence: |
|||
if item not in result: |
|||
result.append(item) |
|||
|
|||
return result |
|||
@ -0,0 +1,21 @@ |
|||
# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
|
|||
|
|||
class Transformer(object): |
|||
def __init__(self): |
|||
pass |
|||
|
|||
def run(self, titles): |
|||
raise NotImplementedError() |
|||
@ -0,0 +1,238 @@ |
|||
# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
|
|||
|
|||
from operator import itemgetter |
|||
from logr import Logr |
|||
from qcond.helpers import simplify, strip, first, sorted_append, distinct |
|||
from qcond.transformers.base import Transformer |
|||
from qcond.compat import xrange |
|||
|
|||
|
|||
class MergeTransformer(Transformer): |
|||
def __init__(self): |
|||
super(MergeTransformer, self).__init__() |
|||
|
|||
def run(self, titles): |
|||
titles = distinct([simplify(title) for title in titles]) |
|||
|
|||
Logr.info(str(titles)) |
|||
|
|||
Logr.debug("------------------------------------------------------------") |
|||
|
|||
root, tails = self.parse(titles) |
|||
|
|||
Logr.debug("--------------------------PARSE-----------------------------") |
|||
|
|||
for node in root: |
|||
print_tree(node) |
|||
|
|||
Logr.debug("--------------------------MERGE-----------------------------") |
|||
|
|||
self.merge(root) |
|||
|
|||
Logr.debug("--------------------------FINAL-----------------------------") |
|||
|
|||
for node in root: |
|||
print_tree(node) |
|||
|
|||
Logr.debug("--------------------------RESULT-----------------------------") |
|||
|
|||
scores = {} |
|||
results = [] |
|||
|
|||
for tail in tails: |
|||
score, value, original_value = tail.full_value() |
|||
|
|||
if value in scores: |
|||
scores[value] += score |
|||
else: |
|||
results.append((value, original_value)) |
|||
scores[value] = score |
|||
|
|||
Logr.debug("%s %s %s", score, value, original_value) |
|||
|
|||
sorted_results = sorted(results, key=lambda item: (scores[item[0]], item[1]), reverse = True) |
|||
|
|||
return [result[0] for result in sorted_results] |
|||
|
|||
def parse(self, titles): |
|||
root = [] |
|||
tails = [] |
|||
|
|||
for title in titles: |
|||
Logr.debug(title) |
|||
|
|||
cur = None |
|||
words = title.split(' ') |
|||
|
|||
for wx in xrange(len(words)): |
|||
word = strip(words[wx]) |
|||
|
|||
if cur is None: |
|||
cur = find_node(root, word) |
|||
|
|||
if cur is None: |
|||
cur = DNode(word, None, num_children=len(words) - wx, original_value=title) |
|||
root.append(cur) |
|||
else: |
|||
parent = cur |
|||
parent.weight += 1 |
|||
|
|||
cur = find_node(parent.right, word) |
|||
|
|||
if cur is None: |
|||
Logr.debug("%s %d", word, len(words) - wx) |
|||
cur = DNode(word, parent, num_children=len(words) - wx) |
|||
sorted_append(parent.right, cur, lambda a: a.num_children < cur.num_children) |
|||
else: |
|||
cur.weight += 1 |
|||
|
|||
tails.append(cur) |
|||
|
|||
return root, tails |
|||
|
|||
def merge(self, root): |
|||
for x in range(len(root)): |
|||
Logr.debug(root[x]) |
|||
root[x].right = self._merge(root[x].right) |
|||
Logr.debug('=================================================================') |
|||
|
|||
return root |
|||
|
|||
def get_nodes_right(self, value): |
|||
if type(value) is not list: |
|||
value = [value] |
|||
|
|||
nodes = [] |
|||
|
|||
for node in value: |
|||
nodes.append(node) |
|||
|
|||
for child in self.get_nodes_right(node.right): |
|||
nodes.append(child) |
|||
|
|||
return nodes |
|||
|
|||
def destroy_nodes_right(self, value): |
|||
nodes = self.get_nodes_right(value) |
|||
|
|||
for node in nodes: |
|||
node.value = None |
|||
node.dead = True |
|||
|
|||
def _merge(self, nodes, depth = 0): |
|||
Logr.debug(str('\t' * depth) + str(nodes)) |
|||
|
|||
top = nodes[0] |
|||
|
|||
# Merge into top |
|||
for x in range(len(nodes)): |
|||
# Merge extra results into top |
|||
if x > 0: |
|||
top.value = None |
|||
top.weight += nodes[x].weight |
|||
self.destroy_nodes_right(top.right) |
|||
|
|||
if len(nodes[x].right): |
|||
top.join_right(nodes[x].right) |
|||
|
|||
Logr.debug("= %s joined %s", nodes[x], top) |
|||
|
|||
nodes[x].dead = True |
|||
|
|||
nodes = [n for n in nodes if not n.dead] |
|||
|
|||
# Traverse further |
|||
for node in nodes: |
|||
if len(node.right): |
|||
node.right = self._merge(node.right, depth + 1) |
|||
|
|||
return nodes |
|||
|
|||
|
|||
def print_tree(node, depth = 0): |
|||
Logr.debug(str('\t' * depth) + str(node)) |
|||
|
|||
if len(node.right): |
|||
for child in node.right: |
|||
print_tree(child, depth + 1) |
|||
else: |
|||
Logr.debug(node.full_value()[1]) |
|||
|
|||
|
|||
def find_node(node_list, value): |
|||
# Try find adjacent node match |
|||
for node in node_list: |
|||
if node.value == value: |
|||
return node |
|||
|
|||
return None |
|||
|
|||
|
|||
class DNode(object): |
|||
def __init__(self, value, parent, right=None, weight=1, num_children=None, original_value=None): |
|||
self.value = value |
|||
|
|||
self.parent = parent |
|||
|
|||
if right is None: |
|||
right = [] |
|||
self.right = right |
|||
|
|||
self.weight = weight |
|||
|
|||
self.original_value = original_value |
|||
self.num_children = num_children |
|||
|
|||
self.dead = False |
|||
|
|||
def join_right(self, nodes): |
|||
for node in nodes: |
|||
duplicate = first(lambda x: x.value == node.value, self.right) |
|||
|
|||
if duplicate: |
|||
duplicate.weight += node.weight |
|||
duplicate.join_right(node.right) |
|||
else: |
|||
node.parent = self |
|||
self.right.append(node) |
|||
|
|||
def full_value(self): |
|||
words = [] |
|||
total_score = 0 |
|||
|
|||
cur = self |
|||
root = None |
|||
|
|||
while cur is not None: |
|||
if cur.value and not cur.dead: |
|||
words.insert(0, cur.value) |
|||
total_score += cur.weight |
|||
|
|||
if cur.parent is None: |
|||
root = cur |
|||
cur = cur.parent |
|||
|
|||
return float(total_score) / len(words), ' '.join(words), root.original_value if root else None |
|||
|
|||
def __repr__(self): |
|||
return '<%s value:"%s", weight: %s, num_children: %s%s%s>' % ( |
|||
'DNode', |
|||
self.value, |
|||
self.weight, |
|||
self.num_children, |
|||
(', original_value: %s' % self.original_value) if self.original_value else '', |
|||
' REMOVING' if self.dead else '' |
|||
) |
|||
@ -0,0 +1,280 @@ |
|||
# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
|
|||
|
|||
from logr import Logr |
|||
from qcond.helpers import create_matcher |
|||
from qcond.transformers.base import Transformer |
|||
|
|||
|
|||
class SliceTransformer(Transformer): |
|||
def __init__(self): |
|||
super(SliceTransformer, self).__init__() |
|||
|
|||
def run(self, titles): |
|||
nodes = [] |
|||
|
|||
# Create a node for each title |
|||
for title in titles: |
|||
nodes.append(SimNode(title)) |
|||
|
|||
# Calculate similarities between nodes |
|||
for node in nodes: |
|||
calculate_sim_links(node, [n for n in nodes if n != node]) |
|||
|
|||
kill_nodes_above(nodes, 0.90) |
|||
|
|||
Logr.debug('---------------------------------------------------------------------') |
|||
|
|||
print_link_tree(nodes) |
|||
Logr.debug('%s %s', len(nodes), [n.value for n in nodes]) |
|||
|
|||
Logr.debug('---------------------------------------------------------------------') |
|||
|
|||
kill_trailing_nodes(nodes) |
|||
|
|||
Logr.debug('---------------------------------------------------------------------') |
|||
|
|||
# Sort remaining nodes by 'num_merges' |
|||
nodes = sorted(nodes, key=lambda n: n.num_merges, reverse=True) |
|||
|
|||
print_link_tree(nodes) |
|||
|
|||
Logr.debug('---------------------------------------------------------------------') |
|||
|
|||
Logr.debug('%s %s', len(nodes), [n.value for n in nodes]) |
|||
|
|||
return [n.value for n in nodes] |
|||
|
|||
|
|||
class SimLink(object): |
|||
def __init__(self, similarity, opcodes, stats): |
|||
self.similarity = similarity |
|||
self.opcodes = opcodes |
|||
self.stats = stats |
|||
|
|||
|
|||
class SimNode(object): |
|||
def __init__(self, value): |
|||
self.value = value |
|||
|
|||
self.dead = False |
|||
self.num_merges = 0 |
|||
|
|||
self.links = {} # {<other SimNode>: <SimLink>} |
|||
|
|||
|
|||
def kill_nodes(nodes, killed_nodes): |
|||
# Remove killed nodes from root list |
|||
for node in killed_nodes: |
|||
if node in nodes: |
|||
nodes.remove(node) |
|||
|
|||
# Remove killed nodes from links |
|||
for killed_node in killed_nodes: |
|||
for node in nodes: |
|||
if killed_node in node.links: |
|||
node.links.pop(killed_node) |
|||
|
|||
|
|||
def kill_nodes_above(nodes, above_sim): |
|||
killed_nodes = [] |
|||
|
|||
for node in nodes: |
|||
if node.dead: |
|||
continue |
|||
|
|||
Logr.debug(node.value) |
|||
|
|||
for link_node, link in node.links.items(): |
|||
if link_node.dead: |
|||
continue |
|||
|
|||
Logr.debug('\t%0.2f -- %s', link.similarity, link_node.value) |
|||
|
|||
if link.similarity >= above_sim: |
|||
if len(link_node.value) > len(node.value): |
|||
Logr.debug('\t\tvery similar, killed this node') |
|||
link_node.dead = True |
|||
node.num_merges += 1 |
|||
killed_nodes.append(link_node) |
|||
else: |
|||
Logr.debug('\t\tvery similar, killed owner') |
|||
node.dead = True |
|||
link_node.num_merges += 1 |
|||
killed_nodes.append(node) |
|||
|
|||
kill_nodes(nodes, killed_nodes) |
|||
|
|||
|
|||
def print_link_tree(nodes): |
|||
for node in nodes: |
|||
Logr.debug(node.value) |
|||
Logr.debug('\tnum_merges: %s', node.num_merges) |
|||
|
|||
if len(node.links): |
|||
Logr.debug('\t========== LINKS ==========') |
|||
for link_node, link in node.links.items(): |
|||
Logr.debug('\t%0.2f -- %s', link.similarity, link_node.value) |
|||
|
|||
Logr.debug('\t---------------------------') |
|||
|
|||
|
|||
def kill_trailing_nodes(nodes): |
|||
killed_nodes = [] |
|||
|
|||
for node in nodes: |
|||
if node.dead: |
|||
continue |
|||
|
|||
Logr.debug(node.value) |
|||
|
|||
for link_node, link in node.links.items(): |
|||
if link_node.dead: |
|||
continue |
|||
|
|||
is_valid = link.stats.get('valid', False) |
|||
|
|||
has_deletions = False |
|||
has_insertions = False |
|||
has_replacements = False |
|||
|
|||
for opcode in link.opcodes: |
|||
if opcode[0] == 'delete': |
|||
has_deletions = True |
|||
if opcode[0] == 'insert': |
|||
has_insertions = True |
|||
if opcode[0] == 'replace': |
|||
has_replacements = True |
|||
|
|||
equal_perc = link.stats.get('equal', 0) / float(len(node.value)) |
|||
insert_perc = link.stats.get('insert', 0) / float(len(node.value)) |
|||
|
|||
Logr.debug('\t({0:<24}) [{1:02d}:{2:02d} = {3:02d} {4:3.0f}% {5:3.0f}%] -- {6:<45}'.format( |
|||
'd:%s, i:%s, r:%s' % (has_deletions, has_insertions, has_replacements), |
|||
len(node.value), len(link_node.value), link.stats.get('equal', 0), |
|||
equal_perc * 100, insert_perc * 100, |
|||
'"{0}"'.format(link_node.value) |
|||
)) |
|||
|
|||
Logr.debug('\t\t%s', link.stats) |
|||
|
|||
kill = all([ |
|||
is_valid, |
|||
equal_perc >= 0.5, |
|||
insert_perc < 2, |
|||
has_insertions, |
|||
not has_deletions, |
|||
not has_replacements |
|||
]) |
|||
|
|||
if kill: |
|||
Logr.debug('\t\tkilled this node') |
|||
|
|||
link_node.dead = True |
|||
node.num_merges += 1 |
|||
killed_nodes.append(link_node) |
|||
|
|||
kill_nodes(nodes, killed_nodes) |
|||
|
|||
stats_print_format = "\t{0:<8} ({1:2d}:{2:2d}) ({3:2d}:{4:2d})" |
|||
|
|||
|
|||
def get_index_values(iterable, a, b): |
|||
return ( |
|||
iterable[a] if a else None, |
|||
iterable[b] if b else None |
|||
) |
|||
|
|||
|
|||
def get_indices(iterable, a, b): |
|||
return ( |
|||
a if 0 < a < len(iterable) else None, |
|||
b if 0 < b < len(iterable) else None |
|||
) |
|||
|
|||
|
|||
def get_opcode_stats(for_node, node, opcodes): |
|||
stats = {} |
|||
|
|||
for tag, i1, i2, j1, j2 in opcodes: |
|||
Logr.debug(stats_print_format.format( |
|||
tag, i1, i2, j1, j2 |
|||
)) |
|||
|
|||
if tag in ['insert', 'delete']: |
|||
ax = None, None |
|||
bx = None, None |
|||
|
|||
if tag == 'insert': |
|||
ax = get_indices(for_node.value, i1 - 1, i1) |
|||
bx = get_indices(node.value, j1, j2 - 1) |
|||
|
|||
if tag == 'delete': |
|||
ax = get_indices(for_node.value, j1 - 1, j1) |
|||
bx = get_indices(node.value, i1, i2 - 1) |
|||
|
|||
av = get_index_values(for_node.value, *ax) |
|||
bv = get_index_values(node.value, *bx) |
|||
|
|||
Logr.debug( |
|||
'\t\t%s %s [%s><%s] <---> %s %s [%s><%s]', |
|||
ax, av, av[0], av[1], |
|||
bx, bv, bv[0], bv[1] |
|||
) |
|||
|
|||
head_valid = av[0] in [None, ' '] or bv[0] in [None, ' '] |
|||
tail_valid = av[1] in [None, ' '] or bv[1] in [None, ' '] |
|||
valid = head_valid and tail_valid |
|||
|
|||
if 'valid' not in stats or (stats['valid'] and not valid): |
|||
stats['valid'] = valid |
|||
|
|||
Logr.debug('\t\t' + ('VALID' if valid else 'INVALID')) |
|||
|
|||
if tag not in stats: |
|||
stats[tag] = 0 |
|||
|
|||
stats[tag] += (i2 - i1) or (j2 - j1) |
|||
|
|||
return stats |
|||
|
|||
|
|||
def calculate_sim_links(for_node, other_nodes): |
|||
for node in other_nodes: |
|||
if node in for_node.links: |
|||
continue |
|||
|
|||
Logr.debug('calculating similarity between "%s" and "%s"', for_node.value, node.value) |
|||
|
|||
# Get similarity |
|||
similarity_matcher = create_matcher(for_node.value, node.value) |
|||
similarity = similarity_matcher.quick_ratio() |
|||
|
|||
# Get for_node -> node opcodes |
|||
a_opcodes_matcher = create_matcher(for_node.value, node.value, swap_longest = False) |
|||
a_opcodes = a_opcodes_matcher.get_opcodes() |
|||
a_stats = get_opcode_stats(for_node, node, a_opcodes) |
|||
|
|||
Logr.debug('-' * 100) |
|||
|
|||
# Get node -> for_node opcodes |
|||
b_opcodes_matcher = create_matcher(node.value, for_node.value, swap_longest = False) |
|||
b_opcodes = b_opcodes_matcher.get_opcodes() |
|||
b_stats = get_opcode_stats(for_node, node, b_opcodes) |
|||
|
|||
for_node.links[node] = SimLink(similarity, a_opcodes, a_stats) |
|||
node.links[for_node] = SimLink(similarity, b_opcodes, b_stats) |
|||
|
|||
#raw_input('Press ENTER to continue') |
|||
@ -0,0 +1,26 @@ |
|||
# Copyright 2013 Dean Gardiner <gardiner91@gmail.com> |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
|
|||
|
|||
from qcond.transformers.base import Transformer |
|||
|
|||
|
|||
COMMON_WORDS = [ |
|||
'the' |
|||
] |
|||
|
|||
|
|||
class StripCommonTransformer(Transformer): |
|||
def run(self, titles): |
|||
return [title for title in titles if title.lower() not in COMMON_WORDS] |
|||
Loading…
Reference in new issue