source: indico/bin/migration/migrate_0.97_0.98.py @ 6b4092

hello-world-walkthroughipv6v0.98-seriesv0.98.2v0.98.3v0.99v1.0v1.1
Last change on this file since 6b4092 was 6b4092, checked in by Pedro Ferreira <jose.pedro.ferreira@…>, 18 months ago

[IMP] Use (room, day) index when appropriate
  • Property mode set to 100644
File size: 15.9 KB
Line 
1# -*- coding: utf-8 -*-
2##
3## This file is part of CDS Indico.
4## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
5##
6## CDS Indico is free software; you can redistribute it and/or
7## modify it under the terms of the GNU General Public License as
8## published by the Free Software Foundation; either version 2 of the
9## License, or (at your option) any later version.
10##
11## CDS Indico is distributed in the hope that it will be useful, but
12## WITHOUT ANY WARRANTY; without even the implied warranty of
13## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14## General Public License for more details.
15##
16## You should have received a copy of the GNU General Public License
17## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
18## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
19from MaKaC.user import AvatarHolder
20from MaKaC.rb_location import CrossLocationQueries
21
22"""
23Migration script: v0.97 -> v0.98
24
25NOTE: Methods should be specified in order of execution, since @since adds them to
26the task list in the order it is called.
27"""
28
29
30import sys, traceback, argparse
31from BTrees.OOBTree import OOTreeSet, OOBTree
32from BTrees.IOBTree import IOBTree
33from dateutil import rrule
34from pkg_resources import parse_version
35
36from MaKaC import __version__
37from MaKaC.common.indexes import IndexesHolder, CategoryDayIndex, CalendarDayIndex
38from MaKaC.common import DBMgr
39from MaKaC.common.info import HelperMaKaCInfo
40from MaKaC.common.Counter import Counter
41from MaKaC.conference import ConferenceHolder, CategoryManager, Conference
42from MaKaC.common.timerExec import HelperTaskList
43from MaKaC.plugins.base import PluginType, PluginsHolder
44from MaKaC.registration import RegistrantSession, RegistrationSession
45from MaKaC.plugins.RoomBooking.default.dalManager import DALManager
46from MaKaC.plugins.RoomBooking.default.room import Room
47from MaKaC.webinterface import displayMgr
48
49from indico.core.index import Catalog
50from indico.ext import livesync
51from indico.util import console, i18n
52from indico.modules.scheduler.tasks import AlarmTask, FoundationSyncTask, \
53     CategoryStatisticsUpdaterTask
54
55from indico.modules.scheduler import Client
56
57
58MIGRATION_TASKS = []
59
60i18n.setLocale('en_GB')
61
62def since(version, always=False):
63    def _since(f):
64        MIGRATION_TASKS.append((version, f, always))
65        return f
66    return _since
67
68
69def _fixAC(obj):
70    ac = obj.getAccessController()
71    ac.setOwner(obj)
72
73
74def _fixAccessController(obj, fixSelf=True):
75    # i.e. subcontributions do not have their own AccessController
76    if fixSelf:
77        _fixAC(obj)
78
79    for mat in obj.getAllMaterialList():
80        for res in mat.getResourceList():
81            _fixAC(res)
82        _fixAC(mat)
83
84
85def _convertAlarms(obj):
86    """
87    Take the alarms in an event and convert them to the new format
88    """
89    alarms = {}
90    obj._legacyAlarmList = obj.getAlarmList()
91
92    for alarm in obj.getAlarmList():
93        if alarm.timeBefore:
94            newTask = AlarmTask(obj, alarm.id, relative=alarm.timeBefore)
95        elif alarm.getStartDate():
96            newTask = AlarmTask(obj, alarm.id, alarm.getStartDate())
97        else:
98            continue
99        newTask.setSubject(alarm.getSubject())
100        newTask.setText(alarm.getText())
101
102        # define directly, otherwise _setText will be triggered!
103        newTask.note = alarm.getNote()
104        newTask.confSummary = alarm.getConfSumary()
105
106        newTask.setToAllParticipants(alarm.getToAllParticipants())
107        alarms[alarm.id] = newTask
108
109        newTask.setFromAddr(alarm.getFromAddr())
110        for addr in alarm.getToAddrList():
111            newTask.addToAddr(addr)
112
113    obj.alarmList = alarms
114
115
116def _fixDefaultStyle(conf, cdmr):
117    confDM = cdmr.getDisplayMgr(conf, True)
118    if confDM.getDefaultStyle() == 'administrative3':
119        confDM.setDefaultStyle('administrative')
120    if confDM.getDefaultStyle() == 'it':
121        confDM.setDefaultStyle('standard')
122
123
124@since('0.98b2')
125def pluginMigration(dbi, withRBDB, prevVersion):
126    """
127    Adding new plugins and adapting existing ones to new name policies
128    """
129
130    PLUGIN_REMAP = {
131        'PayPal': 'payPal',
132        'WorldPay': 'worldPay',
133        'YellowPay': 'yellowPay',
134        "Dummyimporter": "dummy",
135        "CDSInvenio": "invenio",
136        "CERNSearchPOST": "cern_search",
137        "InvenioBatchUploader": "invenio"
138    }
139
140    root = dbi.getDBConnection().root()
141    if 'plugins' in root:
142        ptl = []
143        ps = root['plugins']
144        for k, v in ps.iteritems():
145            if isinstance(v, PluginType):
146                ptl.append(v)
147        for pt in ptl:
148            pt.setUsable(True)
149            for p in pt.getPluginList(includeNonPresent=True,
150                                      includeTestPlugins=True,
151                                      includeNonActive=True):
152                if hasattr(p, '_Plugin__id'):
153                    pid = p.getId()
154                else:
155                    pid = p.getName()
156
157                if pid in PLUGIN_REMAP:
158                    pid = PLUGIN_REMAP[pid]
159
160                p.setId(pid)
161                p.setUsable(True)
162
163    dbi.commit()
164    if withRBDB:
165        DALManager.commit()
166
167    # load new plugins, so that we can update them after
168    PluginsHolder().reloadAllPlugins()
169    dbi.commit()
170    if withRBDB:
171        DALManager.commit()
172
173    if prevVersion < parse_version("0.98b1"):
174        # update db for specific plugins
175        livesync.db.updateDBStructures(root)
176        dbi.commit()
177        if withRBDB:
178            DALManager.commit()
179
180
181@since('0.98b')
182def categoryACMigration(dbi, withRBDB, prevVersion):
183    """
184    Fixing AccessController for categories
185    """
186    for categ in CategoryManager()._getIdx().itervalues():
187        _fixAccessController(categ)
188        dbi.commit()
189
190
191@since('0.98b2')
192def conferenceMigration(dbi, withRBDB, prevVersion):
193    """
194    Adding missing attributes to conference objects and children
195    """
196
197    cdmr = displayMgr.ConfDisplayMgrRegistery()
198    ch = ConferenceHolder()
199    i = 0
200
201    from97 = prevVersion < parse_version("0.98b1")
202
203    # migrating from <=0.97.1 requires smaller granularity
204    for (level, obj) in console.conferenceHolderIterator(ch, deepness='subcontrib' if from97 else 'event'):
205        # only for conferences
206        if level == 'event':
207
208            if from97:
209                # handle sessions, that our iterator ignores
210                for session in obj.getSessionList():
211                    _fixAccessController(session)
212
213                if hasattr(obj, '_Conference__alarmCounter'):
214                    raise Exception("Conference Object %s (%s) seems to have been "
215                                    "already converted" % (obj, obj.id))
216
217                existingKeys = obj.alarmList.keys()
218                existingKeys.sort()
219                nstart = int(existingKeys[-1]) + 1 if existingKeys else 0
220                obj._Conference__alarmCounter = Counter(nstart)
221
222                # For each conference, take the existing tasks and
223                # convert them to the new object classes.
224                _convertAlarms(obj)
225
226            # convert registration form's "Personal Data" section to new format
227            obj.getRegistrationForm()._convertPersonalData()
228
229            # For each conference, fix the default style
230            _fixDefaultStyle(obj, cdmr)
231
232        if from97:
233            _fixAccessController(obj,
234                                 fixSelf=(level != 'subcontrib'))
235
236            # Convert RegistrationSessions to RegistrantSessions
237            if isinstance(obj, Conference):
238                for reg in obj.getRegistrants().itervalues():
239                    if reg._sessions and \
240                           isinstance(reg._sessions[0], RegistrationSession):
241                        reg._sessions = [RegistrantSession(ses, reg) \
242                                         for ses in reg._sessions]
243
244        if i % 1000 == 999:
245            dbi.commit()
246            if withRBDB and from97:
247                DALManager.commit()
248
249        i += 1
250
251    dbi.commit()
252    if withRBDB and from97:
253        DALManager.commit()
254
255
256@since('0.98b')
257def taskMigration(dbi, withRBDB, prevVersion):
258    """
259    Migrating database tasks from the old format to the new one
260    """
261
262    c = Client()
263
264    for t in HelperTaskList().getTaskListInstance().getTasks():
265        for obj in t.listObj.values():
266            print console.colored("   * %s" % obj.__class__.__name__, 'blue')
267            if obj.__class__.__name__ == 'FoundationSync':
268                c.enqueue(
269                    FoundationSyncTask(rrule.DAILY, byhour=0, byminute=0))
270            elif obj.__class__.__name__ == 'StatisticsUpdater':
271                c.enqueue(CategoryStatisticsUpdaterTask(
272                    CategoryManager().getById('0'),
273                    rrule.DAILY,
274                    byhour=0, byminute=0))
275            elif obj.__class__.__name__ == 'sendMail':
276                # they have to be somewhere in the conference
277                alarm = t.conf.alarmList[t.id]
278                c.enqueue(alarm)
279            else:
280                raise Exception("Unknown task type!")
281
282    if withRBDB:
283        DALManager.commit()
284    dbi.commit()
285
286
287@since('0.98b')
288def categoryConfDictToTreeSet(dbi, withRBDB, prevVersion):
289    """
290    Replacing the conference dictionary in the Category objects by a OOTreeSet.
291    """
292    for categ in CategoryManager()._getIdx().itervalues():
293        categ.conferencesBackup = categ.conferences.values()
294        categ.conferences = OOTreeSet(categ.conferences.itervalues())
295        if len(categ.conferences) != len(categ.conferencesBackup):
296            print "Problem migrating conf dict to tree set: %s" % categ.getId()
297
298
299@since('0.98b')
300def categoryDateIndexMigration(dbi, withRBDB, prevVersion):
301    """
302    Replacing category date indexes.
303    """
304    if "backupCategoryDate" not in IndexesHolder()._getIdx():
305        categoryDate = IndexesHolder().getIndex("categoryDate")
306        IndexesHolder()._getIdx()["backupCategoryDate"] = categoryDate
307        newIdx = CategoryDayIndex()
308        newIdx.buildIndex()
309        IndexesHolder()._getIdx()["categoryDate"] = newIdx
310    else:
311        print """categoryDateIndexMigration: new categoryDate index has """ \
312        """NOT been generated because the index backup already exists.
313
314If you still want to regenerate it, please, do it manually using """ \
315        """bin/migration/CategoryDate.py"""
316
317
318@since('0.98.1')
319def categoryDateIndexWithoutVisibility(dbi, withRBDB, prevVersion):
320    """
321    Create category date index without visiblity.
322    """
323    IndexesHolder()._getIdx()['categoryDate']._useVisibility = True
324    if 'categoryDateAll' not in IndexesHolder()._getIdx():
325        newIdx = CategoryDayIndex(visibility=False)
326        newIdx.buildIndex()
327        IndexesHolder()._getIdx()['categoryDateAll'] = newIdx
328
329
330@since('0.98b', always=True)
331def catalogMigration(dbi, withRBDB, prevVersion):
332    """
333    Initializing/updating index catalog
334    """
335    Catalog.updateDB(dbi=dbi)
336
337
338@since('0.98b2')
339def roomBlockingInit(dbi, withRBDB, prevVersion):
340    """
341    Initializing room blocking indexes.
342    """
343    if not withRBDB:
344        return
345
346    root = DALManager().getRoot()
347    if not root.has_key( 'RoomBlocking' ):
348        root['RoomBlocking'] = OOBTree()
349        root['RoomBlocking']['Blockings'] = IOBTree()
350        root['RoomBlocking']['Indexes'] = OOBTree()
351        root['RoomBlocking']['Indexes']['OwnerBlockings'] = OOBTree()
352        root['RoomBlocking']['Indexes']['DayBlockings'] = CalendarDayIndex()
353        root['RoomBlocking']['Indexes']['RoomBlockings'] = OOBTree()
354
355@since('0.98b2')
356def runRoomDayIndexInit(dbi, withRBDB, prevVersion):
357    """
358    Initializing room+day => reservation index.
359    """
360    if not withRBDB:
361        return
362
363    root = DALManager().getRoot()
364    if not root.has_key('RoomDayReservationsIndex'):
365        root['RoomDayReservationsIndex'] = OOBTree()
366        for i, resv in enumerate(CrossLocationQueries.getReservations()):
367            resv._addToRoomDayReservationsIndex()
368            if i % 1000 == 0:
369                DALManager.commit()
370    DALManager.commit()
371
372@since('0.98b2')
373def langToGB(dbi, withRBDB, prevVersion):
374    """
375    Replacing en_US with en_GB.
376    """
377    avatars = AvatarHolder().getList()
378    for av in avatars:
379        if av.getLang() == "en_US":
380            av.setLang("en_GB")
381
382
383@since('0.98b2')
384def makoMigration(dbi, withRBDB, prevVersion):
385    """
386    Installing new TPLs for meeting/lecture styles
387    """
388    info = HelperMaKaCInfo().getMaKaCInfoInstance()
389    sm = info.getStyleManager()
390    try:
391        del sm._stylesheets
392    except:
393        pass
394    for lid in ['meeting', 'simple_event', 'conference']:
395        l = sm._eventStylesheets[lid]
396        if 'it' in l:
397            l.remove('it')
398        if 'administrative3' in l:
399            l.remove('administrative3')
400        sm._eventStylesheets[lid] = l
401    styles = sm.getStyles()
402    styles['xml'] = ('xml','XML.xsl',None)
403    sm.setStyles(styles)
404
405
406@since('0.98b2')
407def pluginOptionsRoomGUIDs(dbi, withRBDB, prevVersion):
408    """
409    Modifying Room GUIDs
410    """
411    if not withRBDB:
412        return
413
414    ph = PluginsHolder()
415    for pluginName, roomsOpt in [('WebcastRequest', 'webcastCapableRooms'),
416                                 ('RecordingRequest', 'recordingCapableRooms')]:
417        opt = ph.getPluginType('Collaboration').getPlugin(pluginName).getOption(roomsOpt)
418        newValue = []
419        for name in opt.getValue():
420            loc, name = name.split(':')
421            room = CrossLocationQueries.getRooms(location=loc, roomName=name)
422            if room:
423                newValue.append(str(room.guid))
424        opt.setValue(newValue)
425
426
427def runMigration(withRBDB=False, prevVersion=parse_version(__version__),
428                 specified=[]):
429
430    print "\nExecuting migration...\n"
431
432    dbi = DBMgr.getInstance()
433
434    # go from older to newer version and execute corresponding tasks
435    for version, task, always in MIGRATION_TASKS:
436        if specified and task.__name__ not in specified:
437            continue
438        if parse_version(version) > prevVersion or always:
439            print console.colored("->", 'green', attrs=['bold']), \
440                  task.__doc__.replace('\n', '').strip(),
441            print console.colored("(%s)" % version, 'yellow')
442            dbi.startRequest()
443            if withRBDB:
444                DALManager.connect()
445
446            task(dbi, withRBDB, prevVersion)
447
448            if withRBDB:
449                DALManager.commit()
450            dbi.endRequest()
451
452            print console.colored("   DONE\n", 'green', attrs=['bold'])
453
454    print console.colored("Database Migration successful!\n",
455                          'green', attrs=['bold'])
456
457
458def main():
459    """
460    Main program cycle
461    """
462
463    print console.colored("""\nThis script will migrate the Indico DB from v0.97.x to v0.98. We recommend that
464this operation be executed while the web server is down, in order to avoid
465concurrency problems and DB conflicts.\n\n""", 'yellow')
466
467    parser = argparse.ArgumentParser(description='Execute migration')
468    parser.add_argument('--with-rb', dest='useRBDB', action='store_true',
469                        help='Use the Room Booking DB')
470    parser.add_argument('--run-only', dest='specified', default='',
471                        help='Specify which step(s) to run (comma-separated)')
472    parser.add_argument('--prev-version', dest='prevVersion', help='Previous version of Indico (used by DB)', default=__version__)
473
474    args = parser.parse_args()
475
476    if console.yesno("Are you sure you want to execute the "
477                     "migration now?"):
478        try:
479            return runMigration(withRBDB=args.useRBDB,
480                                prevVersion=parse_version(args.prevVersion),
481                                specified=filter(lambda x: x, map(lambda x: x.strip(), args.specified.split(','))))
482        except:
483            print console.colored("\nMigration failed! DB may be in "
484                                  " an inconsistent state:", 'red', attrs=['bold'])
485            print console.colored(traceback.format_exc(), 'red')
486    else:
487        return 1
488
489
490if __name__ == "__main__":
491    sys.exit(main())
Note: See TracBrowser for help on using the repository browser.