Changeset 204831 in indico

Timestamp:

07/22/11 09:31:29 (22 months ago)

Author:

Pedro Ferreira <jose.pedro.ferreira@…>

Branches:

master, burotel, hello-world-walkthrough, ipv6, v0.98-series, v0.98.2, v0.98.3, v0.98b1, v0.98b2, v0.99, 051b2622c51afb171a1dedb46a0df4fbb0cbd02e, 0da0c1403bae8e51d8229f460181c71b9e6dda72

Children:

fa8006

Parents:

3cbd3e

git-author:

Pedro Ferreira <jose.pedro.ferreira@…> (07/19/11 17:59:20)

git-committer:

Pedro Ferreira <jose.pedro.ferreira@…> (07/22/11 09:31:29)

Message:

[IMP] Index checking functions/script

Files:

• bin/maintenance/indexes/all_index_check.py (added)
• indico/MaKaC/common/indexes.py (modified) (5 diffs)
• indico/core/index/base.py (modified) (2 diffs)
• indico/core/index/event.py (modified) (1 diff)

indico/MaKaC/common/indexes.py

@@ -35 +35 @@
 from MaKaC.plugins.base import PluginsHolder
 from zope.index.text import textindex
-
+import pytz
+import itertools
 
 # BTrees are 32 bit by default
@@ -298 +299 @@
             return []
 
+    def _check(self, dbi=None):
+        """
+        Performs some sanity checks
+        """
+        i = 0
+        from MaKaC.conference import ConferenceHolder
+        confIdx = ConferenceHolder()._getIdx()
+
+        for cid, confs in self._idxCategItem.iteritems():
+            for confId in confs:
+                # it has to be in the conference holder
+                if confId not in confIdx:
+                    yield "[%s] '%s' not in ConferenceHolder" % (cid, confId)
+                # the category has to be one of the owners
+                elif cid not in (map(lambda x:x.id, ConferenceHolder().getById(confId).getOwnerPath()) + ['0']):
+                    yield "[%s] Conference '%s' is not owned" % (cid, confId)
+            if dbi and i % 100 == 99:
+                dbi.sync()
+            i += 1
+
+
 class CalendarIndex(Persistent):
     """Implements a persistent calendar-like index. Objects (need to implement
@@ -564 +586 @@
         return self._getObjectsStartingAfter( date )
 
-    #def getObjectsStartingInYear( self, year ):
-    #    lastDay = datetime( year, 12, 31 )
-    #    s1 = sets.Set()
-    #    # Objects starting before last day of year (included):
-    #    s1 = self._getObjectsStartingBefore(lastDay + timedelta(days=1))
-    #    if not s1:
-    #        return s1
-    #    firstDay = datetime( year, 1, 1 )
-    #    s2 = sets.Set()
-    #    # Objects starting before first day of year (excluded):
-    #    s2 = self._getObjectsStartingBefore(firstDay)
-    #    # Difference:
-    #    s1.difference_update( s2 )
-    #    return s1
+    def _check(self, dbi=None):
+        """
+        Performs some sanity checks
+        """
+
+        from MaKaC.conference import ConferenceHolder
+        confIdx = ConferenceHolder()._getIdx()
+
+        def _check_index(desc, index, func):
+            i = 0
+            for ts, confs in index.iteritems():
+                for confId in confs:
+                    # it has to be in the conference holder
+                    if confId not in confIdx:
+                        yield "[%s][%s] '%s' not in ConferenceHolder" % (desc, ts, confId)
+                    else:
+
+                        conf = ConferenceHolder().getById(confId)
+                        try:
+                            expectedDate = date2utctimestamp(func(conf))
+                        except OverflowError:
+                            expectedDate = 'overflow'
+
+                        # ts must be ok
+                        if ts != expectedDate:
+                            yield "[%s][%s] Conference '%s' has bogus date (should be '%s')" % (desc, ts, confId, expectedDate)
+                    if dbi and i % 100 == 99:
+                        dbi.sync()
+                    i += 1
+
+        return itertools.chain(
+            _check_index('sdate', self._idxSdate, lambda x: x.getStartDate()),
+            _check_index('edate', self._idxEdate, lambda x: x.getEndDate()))
+
+
 class CalendarDayIndex(Persistent):
     def __init__( self ):
@@ -763 +806 @@
             for event in day:
                 yield event
+
+        i = 0
+        confIdx = ConferenceHolder()._getIdx()
+
+    def _check(self, dbi=None, categId=''):
+        """
+        Performs some sanity checks
+        """
+        i = 0
+        from MaKaC.conference import ConferenceHolder
+        confIdx = ConferenceHolder()._getIdx()
+
+        for ts, confs in self._idxDay.iteritems():
+            dt = pytz.timezone('UTC').localize(datetime.utcfromtimestamp(ts))
+
+            for conf in confs:
+                # it has to be in the conference holder
+                if conf.getId() not in confIdx:
+                    yield "[%s][%s] '%s' not in ConferenceHolder" % (ts, categId, conf.getId())
+                else:
+                    # date must be ok
+                    if dt > conf.getEndDate().replace(hour=23, minute=59, second=59) \
+                           or dt < conf.getStartDate().replace(hour=0, minute=0, second=0):
+                        yield "[%s] '%s' has date out of bounds '%s'(%s)" % (categId, conf.getId(), ts, dt)
+                    elif categId not in (map(lambda x:x.id, conf.getOwnerPath()) + ['0']):
+                        yield "[%s] Conference '%s' is not owned" % (categId, conf.getId())
+
+                if dbi and i % 100 == 99:
+                    dbi.sync()
+                i += 1
 
 
@@ -915 +988 @@
         else:
             return []
+
+    def _check(self, dbi=None):
+        """
+        Performs some sanity checks
+        """
+        for categId, calDayIdx in self._idxCategItem.iteritems():
+            for problem in calDayIdx._check(dbi=dbi, categId=categId):
+                yield problem
 
 

indico/core/index/base.py

@@ -152 +152 @@
 
 
-
 class DIndex(SIndex):
     """
@@ -211 +210 @@
         self._num_objs.change(-1)
 
+    def _check(self, dbi=None):
+        """
+        Simple sanity check
+        """
+        i = 0
+        # check that the elements in fwd index are also in rev index
+        cls = self.__class__.__name__
+        for ts, eset in self._fwd_index.iteritems():
+            if not eset:
+                yield "[%s] Element set at '%s' is empty" % ts
+            for elem in eset:
+                uid = elem.getUniqueId()
+                key = self._adapter(elem)
+                if uid not in self._rev_index:
+                    yield "[%s] An entry for '%s'(%s) should exist in _rev_index" \
+                          % (cls, uid, ts)
+                elif key not in self._rev_index[uid]:
+                    yield "[%s] Element '%s'(%s) should be in _rev_index['%s']" \
+                          % (cls, uid, ts, key)
+            if dbi and i % 1000 == 999:
+                dbi.abort()
+            i += 1
+
+        # now the opposite
+        for uid, kset in self._rev_index.iteritems():
+            for key in kset:
+                if key not in self._fwd_index:
+                    yield "[%s] Key '%s' not found in _fwd_index" % (cls, key)
+                elif uid not in map(lambda x:x.getUniqueId(), self._fwd_index[key]):
+                    yield "[%s] Object '%s' not in _fwd_index[%s]" % (cls, uid, key)
+            if dbi and i % 1000 == 999:
+                dbi.abort()
+            i += 1
+
 
 class SIOIndex(DIndex):

indico/core/index/event.py

@@ -60 +60 @@
             if dbi:
                 dbi.commit()
+
+    def _check(self, dbi=None):
+        from MaKaC.conference import CategoryManager, ConferenceHolder
+        confIdx = ConferenceHolder()._getIdx()
+        categIdx = CategoryManager()._getIdx()
+
+        i = 0
+
+        for cid, index in self._container.iteritems():
+            # simple data structure check
+            for problem in index._check():
+                yield problem
+
+            # consistency with CategoryManager
+            if cid not in categIdx:
+                yield "Category '%s' not in CategoryManager" % cid
+
+            # consistency with ConferenceHolder
+            for ts, conf in index.iteritems():
+                if conf.getId() not in confIdx:
+                    yield "[%s] Conference '%s'(%s) not in ConferenceHolder" \
+                          % (cid, conf.getId(), ts)
+
+                if dbi and i % 100 == 99:
+                    dbi.abort()
+
+                i += 1