source: indico/setup.py @ 49bc7c

burotelhello-world-walkthroughipv6v0.98-seriesv0.98.2v0.98.3v0.98b1v0.98b2v0.99v1.0v1.1
Last change on this file since 49bc7c was 49bc7c, checked in by Pedro Ferreira <jose.pedro.ferreira@…>, 3 years ago

[FIX] Test Framework: Adapted functional tests

  • Functional tests working;
  • Database connectivity issue (hostname) solved;
  • Reworked example tests so that they work with the most recent version;
  • Added "record" mode that allows an empty database to be open, so that tests can be recorded using it;
  • Property mode set to 100644
File size: 20.9 KB
Line 
1# -*- coding: utf-8 -*-
2##
3##
4## This file is part of CDS Indico.
5## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
6##
7## CDS Indico is free software; you can redistribute it and/or
8## modify it under the terms of the GNU General Public License as
9## published by the Free Software Foundation; either version 2 of the
10## License, or (at your option) any later version.
11##
12## CDS Indico is distributed in the hope that it will be useful, but
13## WITHOUT ANY WARRANTY; without even the implied warranty of
14## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15## General Public License for more details.
16##
17## You should have received a copy of the GNU General Public License
18## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
19## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
20
21# Autoinstalls setuptools if the user doesn't have them already
22import ez_setup
23ez_setup.use_setuptools()
24
25import commands
26import getopt
27import os
28import re
29import shutil
30import string
31import sys
32from distutils.sysconfig import get_python_lib
33from distutils.cmd import Command
34from distutils.command import bdist
35
36import pkg_resources
37from setuptools.command import develop, install, sdist, bdist_egg, easy_install
38from setuptools import setup, find_packages, findall
39
40EXTRA_RESOURCES_URL = "http://cdswaredev.cern.ch/indico/wiki/Admin/Installation/IndicoExtras"
41
42if sys.platform == 'linux2':
43    import pwd
44    import grp
45
46
47class vars(object):
48    '''Variable holder.'''
49    packageDir = None
50    versionVal = 'None'
51    accessuser = None
52    accessgroup = None
53    dbInstalledBySetupPy = False
54    binDir = None
55    documentationDir = None
56    configurationDir = None
57    htdocsDir = None
58
59###  Methods required by setup() ##############################################
60
61def _generateDataPaths(x):
62
63    dataFilesDict = {}
64
65    for (baseDstDir, files, remove_first_x_chars) in x:
66        for f in files:
67            dst_dir = os.path.join(baseDstDir, os.path.dirname(f)[remove_first_x_chars:])
68            if dst_dir not in dataFilesDict:
69                dataFilesDict[dst_dir] = []
70            dataFilesDict[dst_dir].append(f)
71
72    dataFiles = []
73    for k, v in dataFilesDict.items():
74        dataFiles.append((k, v))
75
76    return dataFiles
77
78def _getDataFiles(x):
79    """
80    Returns a fully populated data_files ready to be fed to setup()
81
82    WARNING: when creating a bdist_egg we need to include files inside bin,
83    doc, config & htdocs into the egg therefore we cannot fetch indico.conf
84    values directly because they will not refer to the proper place. We
85    include those files in the egg's root folder.
86    """
87
88    # setup expects a list like this (('foo/bar/baz', 'wiki.py'),
89    #                                 ('a/b/c', 'd.jpg'))
90    #
91    # What we do below is transform a list like this:
92    #                                (('foo', 'bar/baz/wiki.py'),
93    #                                 ('a', 'b/c/d.jpg'))
94    #
95    # first into a dict and then into a pallatable form for setuptools.
96
97    # This re will be used to filter out etc/*.conf files and therefore not overwritting them
98    isAConfRe = re.compile('etc\/[^/]+\.conf$')
99
100    dataFiles = _generateDataPaths((('bin',           findall('bin'), 4),
101                                    ('doc', findall('doc'), 4),
102                                    ('etc', [xx for xx in findall('etc') if not isAConfRe.search(xx)], 4),
103                                    ('htdocs',        findall('indico/htdocs'), 14)))
104    return dataFiles
105
106
107
108
109
110def _getInstallRequires():
111    '''Returns external packages required by Indico
112
113    These are the ones needed for runtime.'''
114
115    base =  ['pytz', 'zope.index', 'zope.interface', 'simplejson', 'suds', 'cds-indico-extras']
116    if sys.version_info[1] < 5: #for Python older than 2.5
117        base.append('hashlib') # hashlib isn't a part of Python older than 2.5
118        base.append('ZODB3>=3.8,<3.9.0a')
119    else:                       #for Python 2.5+
120        base.append('ZODB3>=3.8')
121
122    return base
123
124
125def _versionInit():
126        '''Retrieves the version number from indico/MaKaC/__init__.py and returns it'''
127
128        import datetime
129        from indico.MaKaC import __version__
130        v = __version__
131
132        print('Version being packaged: %s' % v)
133
134        return v
135
136###  Commands ###########################################################
137class sdist_indico(sdist.sdist):
138    user_options = sdist.sdist.user_options + \
139                   [('version=', None, 'version to distribute')]
140    version = 'dev'
141
142    def run(self):
143        global x
144        sdist.sdist.run(self)
145
146
147class jsdist_indico:
148    def jsCompress(self):
149        from MaKaC.consoleScripts.installBase import jsCompress
150        jsCompress()
151        self.dataFiles += _generateDataPaths([('htdocs/js/presentation/pack', findall('indico/htdocs/js/presentation/pack'), 35),
152                                             ('htdocs/js/indico/pack', findall('indico/htdocs/js/indico/pack'), 29)])
153
154
155def _bdist_indico(dataFiles):
156    class bdist_indico(bdist.bdist, jsdist_indico):
157        def run(self):
158            self.jsCompress()
159            compileAllLanguages()
160            bdist.bdist.run(self)
161
162    bdist_indico.dataFiles = dataFiles
163    return bdist_indico
164
165def _bdist_egg_indico(dataFiles):
166    class bdist_egg_indico(bdist_egg.bdist_egg, jsdist_indico):
167        def run(self):
168            self.jsCompress()
169            compileAllLanguages()
170            bdist_egg.bdist_egg.run(self)
171
172    bdist_egg_indico.dataFiles = dataFiles
173    return bdist_egg_indico
174
175class jsbuild(Command):
176    description = "minifies and packs javascript files"
177    user_options = []
178    boolean_options = []
179
180    def initialize_options(self):
181        pass
182
183    def finalize_options(self):
184        pass
185
186    def run(self):
187        from MaKaC.consoleScripts.installBase import jsCompress
188        jsCompress()
189
190class fetchdeps:
191    def run(self):
192        print "Checking if dependencies need to be installed..."
193
194        wset = pkg_resources.working_set
195
196        wset.resolve(map(pkg_resources.Requirement.parse, _getInstallRequires()),
197                           installer = self._installMissing)
198
199        print "Done!"
200
201
202    def _installMissing(self, dist):
203        env = pkg_resources.Environment()
204        print dist, EXTRA_RESOURCES_URL
205        easy_install.main(["-f", EXTRA_RESOURCES_URL, "-U", str(dist)])
206        env.scan()
207        return env[str(dist)][0]
208
209
210class fetchdeps_indico(fetchdeps, Command):
211    description = "fetch all the dependencies needed to run Indico"
212    user_options = []
213    boolean_options = []
214
215    def initialize_options(self):
216        pass
217
218    def finalize_options(self):
219        pass
220
221
222class develop_indico(Command):
223    description = "prepares the current directory for Indico development"
224    user_options = []
225    boolean_options = []
226
227    def initialize_options(self):
228        pass
229
230    def finalize_options(self):
231        pass
232
233    def run(self):
234
235        fetchdeps().run()
236
237        local = 'etc/indico.conf'
238        if os.path.exists(local):
239            print 'Upgrading existing etc/indico.conf..'
240            upgrade_indico_conf(local, 'etc/indico.conf.sample')
241        else:
242            print 'Creating new etc/indico.conf..'
243            shutil.copy('etc/indico.conf.sample', local)
244
245        for f in [x for x in ('etc/zdctl.conf', 'etc/zodb.conf') if not os.path.exists(x)]:
246            shutil.copy('%s.sample' % f, f)
247
248        print """\nIndico needs to store some information in the filesystem (database, cache, temporary files, logs...)
249Please specify the directory where you'd like it to be placed.
250(Note that putting it outside of your sourcecode tree is recommended)"""
251        prefixDir = raw_input('[%s]: ' % os.getcwd()).strip()
252
253        if prefixDir == '':
254            prefixDir = os.getcwd()
255
256        directories = dict((d, os.path.join(prefixDir, d)) for d in
257                           ['db', 'log', 'tmp', 'cache', 'archive'])
258
259        print 'Creating directories...',
260        for d in directories.values():
261            if not os.path.exists(d):
262                os.makedirs(d)
263        print 'Done!'
264
265        directories['htdocs'] = os.path.join(os.getcwd(), 'indico', 'htdocs')
266        directories['bin'] = os.path.join(os.getcwd(), 'bin')
267        directories['etc'] = os.path.join(os.getcwd(), 'etc')
268        directories['doc'] = os.path.join(os.getcwd(), 'doc')
269
270        self._update_conf_dir_paths(local, directories)
271
272        from MaKaC.consoleScripts.installBase import _databaseText, _findApacheUserGroup, _checkDirPermissions, _updateDbConfigFiles, _updateMaKaCEggCache
273
274        user = ''
275
276        sourcePath = os.getcwd()
277
278        if sys.platform == "linux2":
279            # find the apache user/group
280            user, group = _findApacheUserGroup(None, None)
281            _checkDirPermissions(directories, dbInstalledBySetupPy = directories['db'], accessuser = user, accessgroup = group)
282
283        _updateDbConfigFiles(directories['db'], directories['log'], os.path.join(sourcePath, 'etc'), directories['tmp'], user)
284
285        _updateMaKaCEggCache(os.path.join(os.path.dirname(__file__), 'indico', 'MaKaC', '__init__.py'), directories['tmp'])
286
287        updateIndicoConfPathInsideMaKaCConfig(os.path.join(os.path.dirname(__file__), ''), 'indico/MaKaC/common/MaKaCConfig.py')
288        compileAllLanguages()
289        print '''
290%s
291        ''' % _databaseText('etc')
292
293    def _update_conf_dir_paths(self, filepath, dirs):
294        fdata = open(filePath).read()
295        for dir in dirs.items():
296            d = dir[1].replace("\\","/") # For Windows users
297            fdata = re.sub('\/opt\/indico\/%s'%dir[0], d, fdata)
298        open(filePath, 'w').write(fdata)
299
300class test_indico(Command):
301    """
302    Test command for Indico
303    """
304
305    description = "Test Suite Framework"
306    user_options = [('specify=', None, "Use nosetests style (file.class:testcase)"),
307                    ('coverage', None, "Output coverage report in html"),
308                    ('unit', None, "Run only Unit tests"),
309                    ('functional', None, "Run only Functional tests"),
310                    ('pylint', None, "Run python source analysis"),
311                    ('jsunit', None, "Run js unit tests"),
312                    ('jslint', None, "Run js source analysis"),
313                    ('jscoverage', None, "Output coverage report in html for js"),
314                    ('jsspecify=', None, "Use js-test-driver style (TestCaseName.testName)"),
315                    ('grid', None, "Use Selenium Grid"),
316                    ('html', None, "Make an HTML report (when possible)"),
317                    ('record', None, "Record tests (for --functional)"),
318                    ('full-output', None, "Write the results to the console, in addition to the log file")]
319    boolean_options = []
320
321    specify = None
322    coverage = False
323    unit = False
324    functional = False
325    pylint = False
326    jsunit = False
327    jslint = False
328    jscoverage = False
329    jsspecify = None
330    grid = None
331    full_output = False
332    html = False
333    record = False
334
335    def initialize_options(self):
336        pass
337
338    def finalize_options(self):
339        pass
340
341    def run(self):
342
343        if not self.checkIndicopPackages():
344            print "Please install those missing packages before launching the tests again"
345            sys.exit(-1)
346
347        #missing jars will be downloaded automatically
348        if not self.checkIndicopJars():
349            print "Some jars could not be downloaded. Please download the missing jars manually"
350            sys.exit(-1)
351
352        from indico.tests import TestManager, TEST_RUNNERS
353        testsToRun = []
354
355        allTests = TEST_RUNNERS.keys()
356
357        for testType in allTests:
358            if getattr(self, testType):
359                testsToRun.append(testType)
360
361        if self.jsspecify and 'jsunit' not in testsToRun:
362            testsToRun.append('jsunit')
363        if self.specify != None and 'specify' not in testsToRun:
364            testsToRun.append('specify')
365
366        if testsToRun == []:
367            testsToRun = allTests
368
369
370        options = {'verbose': self.full_output,
371                   'html': self.html,
372                   'specify': self.specify,
373                   'coverage': self.coverage,
374                   'record': self.record}
375
376        #this variable will tell what to do with the databases
377        fakeDBPolicy = self.checkDBStatus(testsToRun, self.specify)
378
379        manager = TestManager()
380        result = manager.main(fakeDBPolicy, testsToRun, options)
381
382        print result
383
384
385    def checkDBStatus(self, testsToRun, specify):
386        from indico.tests import TestConfig
387        from indico.tests.util import TestZEOServer
388        from MaKaC.common.Configuration import Config
389
390        fakeDBPolicy = 0
391        if ('functional' in testsToRun) or ('grid' in testsToRun) or ((specify != None) and (specify.find('unit/') < 0)):
392
393            params = Config.getInstance().getDBConnectionParams()
394
395            #checking if production db is running
396            server = TestZEOServer(params[1],
397                                   'test',
398                                   hostname = params[0])
399
400
401            if server.server.can_connect(server.options.family, server.options.address):
402                print """Your production database is currently running.
403Do you want to stop it using this command '%s' and run the tests?
404(We will restart your produduction after the tests with this command '%s')""" % \
405(TestConfig.getInstance().getStopDBCmd(), TestConfig.getInstance().getStartDBCmd())
406
407                userInput = raw_input("Press enter or type 'yes' to accept: ")
408                if userInput == 'yes' or userInput == '':
409                    fakeDBPolicy = 3
410                else:
411                    print "Exiting testing framework..."
412                    sys.exit(1)
413
414            else:
415                fakeDBPolicy = 2
416        elif 'unit' in testsToRun or 'specify' in testsToRun:
417            fakeDBPolicy = 1
418
419        return fakeDBPolicy
420
421    def checkIndicopPackages(self):
422        packagesList = ['figleaf',
423                        'nose',
424                        'selenium',
425                        'twill']
426        validPackages = True
427
428        for package in packagesList:
429            try:
430                pkg_resources.require(package)
431            except pkg_resources.DistributionNotFound:
432                print """
433%s not found! Please install it.
434i.e. try 'easy_install %s'""" % (package, package)
435                validPackages = False
436        return validPackages
437
438    def checkIndicopJars(self):
439        from indico.tests import TestConfig
440
441        """check if needed jars are here, if not, dowloading them and unzip a file if necessary"""
442        jarsList = {}
443        currentFilePath = os.path.dirname(__file__)
444        testModPath = os.path.join(currentFilePath, 'indico', 'tests')
445
446        try:
447            jarsList['jsunit'] = {'path':     os.path.join(testModPath,
448                                                           'javascript',
449                                                           'unit'),
450                                  'url':      TestConfig.getInstance().getJsunitURL(),
451                                  'filename': TestConfig.getInstance().getJsunitFilename()}
452
453            jarsList['jscoverage'] = {'path':     os.path.join(testModPath,
454                                                               'javascript',
455                                                               'unit',
456                                                               'plugins'),
457                                      'url':      TestConfig.getInstance().getJscoverageURL(),
458                                      'filename': TestConfig.getInstance().getJscoverageFilename()}
459
460            jarsList['selenium'] = {'path':      os.path.join(testModPath,
461                                                              'python',
462                                                              'functional'),
463                                    'url':       TestConfig.getInstance().getSeleniumURL(),
464                                    'inZipPath': TestConfig.getInstance().getSeleniumInZipPath(),
465                                    'zipname':   TestConfig.getInstance().getSeleniumZipname(),
466                                    'filename':  TestConfig.getInstance().getSeleniumFilename()}
467        except KeyError, key:
468            print "[ERR] Please specify a value for %s in tests.conf" % key
469            sys.exit(1)
470
471        validJars = True
472
473        for name in jarsList:
474            jar = jarsList[name]
475            #check if jar is already here
476            if not os.path.exists(os.path.join(jar['path'], jar['filename'])):
477                print "Downloading %s..." % jar['filename']
478                try:
479                    self.download(jar['url'], jar['path'])
480
481                    #if a zipname is specified, we will unzip the target file pointed by inZipPath variable
482                    try:
483                        if jar['zipname'] != None:
484                            self.unzip(os.path.join(jar['path'], jar['zipname']), jar['inZipPath'], os.path.join(jar['path'], jar['filename']))
485                    except KeyError:
486                        pass
487
488                except IOError, e:
489                    validJars = validJars and False
490                    print 'Could not download %s from %s (%s)' % (jar['filename'], jar['url'], e)
491
492        return validJars
493
494    def download(self, url, path):
495        """Copy the contents of a file from a given URL
496        to a local file.
497        """
498        import urllib
499        webFile = urllib.urlopen(url)
500        localFile = open(os.path.join(path, url.split('/')[-1]), 'w')
501        localFile.write(webFile.read())
502        webFile.close()
503        localFile.close()
504
505    def unzip(self, zipPath, inZipPath, targetFile):
506        """extract the needed file from zip and then delete the zip"""
507        import zipfile
508        try:
509            zfobj = zipfile.ZipFile(zipPath)
510            outfile = open(targetFile, 'wb')
511            outfile.write(zfobj.read(inZipPath))
512            outfile.flush()
513            outfile.close()
514
515            #delete zip file
516            os.unlink(zipPath)
517        except NameError, e:
518            print e
519
520if __name__ == '__main__':
521    # Always load source from the current folder
522    sys.path = [os.path.abspath('indico')] + sys.path
523
524    #PWD_INDICO_CONF = 'etc/indico.conf'
525    #if not os.path.exists(PWD_INDICO_CONF):
526    #    shutil.copy('etc/indico.conf.sample', PWD_INDICO_CONF)
527
528    from MaKaC.consoleScripts.installBase import *
529
530
531    #Dirty trick: For running tests, we need to load all the modules and get rid of unnecessary outputs
532    tempLoggingDir = None
533    if 'test' in sys.argv:
534        import logging
535        import tempfile
536        tempLoggingDir = tempfile.mkdtemp()
537        logging.basicConfig(filename=os.path.join(tempLoggingDir, 'logging'), level=logging.DEBUG)
538        setIndicoInstallMode(False)
539    else:
540        setIndicoInstallMode(True)
541
542    x = vars()
543    x.packageDir = os.path.join(get_python_lib(), 'MaKaC')
544
545
546    x.binDir = 'bin'
547    x.documentationDir = 'doc'
548    x.configurationDir = 'etc'
549    x.htdocsDir = 'htdocs'
550
551    dataFiles = _getDataFiles(x)
552
553    setup(name = "cds-indico",
554          cmdclass = {'sdist': sdist_indico,
555                    'bdist': _bdist_indico(dataFiles),
556                    'bdist_egg': _bdist_egg_indico(dataFiles),
557                    'jsbuild': jsbuild,
558                    'fetchdeps': fetchdeps_indico,
559                    'develop': develop_indico,
560                    'test': test_indico,
561                    },
562
563          version = _versionInit(),
564          description = "Indico is a full-featured conference lifecycle management and meeting/lecture scheduling tool",
565          author = "Indico Team",
566          author_email = "indico-team@cern.ch",
567          url = "http://cdswaredev.cern.ch/indico",
568          download_url = "http://cdswaredev.cern.ch/indico/wiki/Releases/Indico0.97.0",
569          platforms = ["any"],
570          long_description = "Indico allows you to schedule conferences, from single talks to complex meetings with sessions and contributions. It also includes an advanced user delegation mechanism, allows paper reviewing, archival of conference information and electronic proceedings",
571          license = "http://www.gnu.org/licenses/gpl-2.0.txt",
572          package_dir = { '': 'indico' },
573          entry_points = {
574            'console_scripts': [ 'taskDaemon           = MaKaC.consoleScripts.taskDaemon:main',
575                                 'indico_initial_setup = MaKaC.consoleScripts.indicoInitialSetup:main',
576                                 'indico_ctl           = MaKaC.consoleScripts.indicoCtl:main',
577                                 ]
578          },
579          zip_safe = False,
580          packages = find_packages(where = 'indico', exclude = ('htdocs',)),
581          install_requires = _getInstallRequires(),
582          data_files = dataFiles,
583          package_data = {'indico': ['*.*'] },
584          include_package_data = True,
585          dependency_links = [
586                              EXTRA_RESOURCES_URL
587                              ],
588          )
589
590    #delete the temp folder used for logging
591    if 'test' in sys.argv:
592        shutil.rmtree(tempLoggingDir)
Note: See TracBrowser for help on using the repository browser.