Check for broken indexes and reindex if needed

couchpotato/core/database.py

@@ -3,10 +3,11 @@ import os
 import time
 import traceback
 
+from CodernityDB.index import IndexException, IndexNotFoundException, IndexConflict
 from couchpotato import CPLog
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, sp
 from couchpotato.core.helpers.variable import getImdb, tryInt
 
 
@@ -15,11 +16,13 @@ log = CPLog(__name__)
 
 class Database(object):
 
-    indexes = []
+    indexes = None
     db = None
 
     def __init__(self):
 
+        self.indexes = {}
+
         addApiView('database.list_documents', self.listDocuments)
         addApiView('database.reindex', self.reindex)
         addApiView('database.compact', self.compact)
@@ -45,26 +48,45 @@ class Database(object):
 
     def setupIndex(self, index_name, klass):
 
-        self.indexes.append(index_name)
+        self.indexes[index_name] = klass
 
         db = self.getDB()
 
         # Category index
         index_instance = klass(db.path, index_name)
         try:
-            db.add_index(index_instance)
+
-            db.reindex_index(index_name)
+            # Make sure store and bucket don't exist
-        except:
+            exists = []
-            previous = db.indexes_names[index_name]
+            for x in ['buck', 'stor']:
-            previous_version = previous._version
+                full_path = os.path.join(db.path, '%s_%s' % (index_name, x))
-            current_version = klass._version
+                if os.path.exists(full_path):
-
+                    exists.append(full_path)
-            # Only edit index if versions are different
+
-            if previous_version < current_version:
+            if index_name not in db.indexes_names:
-                log.debug('Index "%s" already exists, updating and reindexing', index_name)
+
-                db.destroy_index(previous)
+                # Remove existing buckets if index isn't there
+                for x in exists:
+                    os.unlink(x)
+
+                # Add index (will restore buckets)
                 db.add_index(index_instance)
                 db.reindex_index(index_name)
+            else:
+                # Previous info
+                previous = db.indexes_names[index_name]
+                previous_version = previous._version
+                current_version = klass._version
+
+                # Only edit index if versions are different
+                if previous_version < current_version:
+                    log.debug('Index "%s" already exists, updating and reindexing', index_name)
+                    db.destroy_index(previous)
+                    db.add_index(index_instance)
+                    db.reindex_index(index_name)
+
+        except:
+            log.error('Failed adding index %s: %s', (index_name, traceback.format_exc()))
 
     def deleteDocument(self, **kwargs):
 
@@ -138,21 +160,62 @@ class Database(object):
             'success': success
         }
 
-    def compact(self, **kwargs):
+    def compact(self, try_repair = True, **kwargs):
+
+        success = False
+        db = self.getDB()
+
+        # Removing left over compact files
+        db_path = sp(db.path)
+        for f in os.listdir(sp(db.path)):
+            for x in ['_compact_buck', '_compact_stor']:
+                if f[-len(x):] == x:
+                    os.unlink(os.path.join(db_path, f))
 
-        success = True
         try:
             start = time.time()
-            db = self.getDB()
             size = float(db.get_db_details().get('size', 0))
             log.debug('Compacting database, current size: %sMB', round(size/1048576, 2))
 
             db.compact()
             new_size = float(db.get_db_details().get('size', 0))
             log.debug('Done compacting database in %ss, new size: %sMB, saved: %sMB', (round(time.time()-start, 2), round(new_size/1048576, 2), round((size-new_size)/1048576, 2)))
+            success = True
+        except (IndexException, AttributeError):
+            if try_repair:
+                log.error('Something wrong with indexes, trying repair')
+
+                # Remove all indexes
+                old_indexes = self.indexes.keys()
+                for index_name in old_indexes:
+                    try:
+                        db.destroy_index(index_name)
+                    except IndexNotFoundException:
+                        pass
+                    except:
+                        log.error('Failed removing old index %s', index_name)
+
+                # Add them again
+                for index_name in self.indexes:
+                    klass = self.indexes[index_name]
+
+                    # Category index
+                    index_instance = klass(db.path, index_name)
+                    try:
+                        db.add_index(index_instance)
+                        db.reindex_index(index_name)
+                    except IndexConflict:
+                        pass
+                    except:
+                        log.error('Failed adding index %s', index_name)
+                        raise
+
+                self.compact(try_repair = False)
+            else:
+                log.error('Failed compact: %s', traceback.format_exc())
+
         except:
             log.error('Failed compact: %s', traceback.format_exc())
-            success = False
 
         return {
             'success': success
@@ -166,6 +229,7 @@ class Database(object):
         size = db.get_db_details().get('size')
         prop_name = 'last_db_compact'
         last_check = int(Env.prop(prop_name, default = 0))
+
         if size > 26214400 and last_check < time.time()-604800: # 25MB / 7 days
             self.compact()
             Env.prop(prop_name, value = int(time.time()))