[galaxy-dev] [hg] galaxy 3452: Egg support overhaul, introduces a number of f...

details: http://www.bx.psu.edu/hg/galaxy/rev/0cb68935eede changeset: 3452:0cb68935eede user: Nate Coraor <nate@bx.psu.edu> date: Mon Mar 01 15:41:30 2010 -0500 description: Egg support overhaul, introduces a number of fixes to the ABI recognition, better build scripts, a rewrite of the eggs library and eliminition of the need for python -ES diffstat: dist-eggs.ini | 79 +- eggs.ini | 63 +- lib/galaxy/__init__.py | 78 +- lib/galaxy/eggs/__init__.py | 769 ++++++++-------------- lib/galaxy/eggs/dist.py | 86 ++ lib/galaxy/eggs/scramble.py | 234 ++++++ manage_db.sh | 2 +- run.sh | 8 +- run_functional_tests.sh | 10 +- run_reports.sh | 3 +- run_unit_tests.sh | 2 +- scripts/check_eggs.py | 10 +- scripts/check_python.py | 13 +- scripts/dist-scramble.py | 22 +- scripts/fetch_eggs.py | 56 +- scripts/get_platforms.py | 6 +- scripts/paster.py | 1 + scripts/scramble.py | 44 +- scripts/scramble/lib/get_platform.py | 18 - scripts/scramble/lib/scramble_lib.py | 164 ++++ scripts/scramble/patches/Cheetah/SetupConfig.py | 103 +++ scripts/scramble/patches/GeneTrack/setup.py | 15 + scripts/scramble/patches/MySQL_python/site.cfg | 18 + scripts/scramble/patches/PasteScript/setup.py | 145 ++++ scripts/scramble/patches/psycopg2/setup.py | 252 +++++++ scripts/scramble/scripts/Cheetah.py | 62 - scripts/scramble/scripts/DRMAA_python-macosx.py | 29 +- scripts/scramble/scripts/DRMAA_python-solaris.py | 44 +- scripts/scramble/scripts/DRMAA_python.py | 30 +- scripts/scramble/scripts/GeneTrack.py | 58 - scripts/scramble/scripts/MySQL_python-solaris.py | 81 ++ scripts/scramble/scripts/MySQL_python.py | 188 +--- scripts/scramble/scripts/PasteScript.py | 53 - scripts/scramble/scripts/generic.py | 28 +- scripts/scramble/scripts/pbs_python.py | 57 +- scripts/scramble/scripts/psycopg2-cygwin.py | 39 - scripts/scramble/scripts/psycopg2.py | 177 +--- scripts/scramble/scripts/pysam-solaris.py | 29 + scripts/scramble/scripts/pysqlite.py | 43 +- scripts/scramble/scripts/python_lzo.py | 129 --- set_metadata.sh | 2 +- setup.sh | 4 +- setup_paths.sh | 46 - 43 files changed, 1817 insertions(+), 1483 deletions(-) diffs (truncated from 3938 to 3000 lines): diff -r a6c9862ece3d -r 0cb68935eede dist-eggs.ini --- a/dist-eggs.ini Mon Mar 01 13:30:37 2010 -0500 +++ b/dist-eggs.ini Mon Mar 01 15:41:30 2010 -0500 @@ -1,9 +1,9 @@ ; ; Config for building eggs for distribution (via a site such as ; eggs.g2.bx.psu.edu). Probably only useful to Galaxy developers at -; Penn State. This file is used by dist-scramble.py +; Penn State. This file is used by scripts/dist-scramble.py ; -; More information: http://g2.trac.bx.psu.edu/wiki/GalaxyEggs +; More information: http://bitbucket.org/galaxy/galaxy-central/wiki/Config/Eggs ; [hosts] @@ -22,16 +22,32 @@ py2.4-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.4 py2.5-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.5 py2.6-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.6 -py2.5-macosx-10.5-i386-ucs2 = lion.bx.psu.edu /usr/bin/python2.5 -py2.4-solaris-2.10-i86pc-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc-ucs2/bin/python2.4 -py2.5-solaris-2.10-i86pc-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc-ucs2/bin/python2.5 -py2.6-solaris-2.10-i86pc-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc-ucs2/bin/python2.6 -py2.4-solaris-2.11-i86pc-ucs2 = victory.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.11-i86pc-ucs2/bin/python2.4 -py2.5-solaris-2.11-i86pc-ucs2 = victory.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.11-i86pc-ucs2/bin/python2.5 -py2.6-solaris-2.11-i86pc-ucs2 = victory.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.11-i86pc-ucs2/bin/python2.6 -py2.4-solaris-2.10-sun4u-ucs2 = v880.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-sun4u-ucs2/bin/python2.4 -py2.5-solaris-2.10-sun4u-ucs2 = v880.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-sun4u-ucs2/bin/python2.5 -py2.6-solaris-2.10-sun4u-ucs2 = v880.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-sun4u-ucs2/bin/python2.6 +py2.6-macosx-10.6-universal-ucs2 = lion.bx.psu.edu /usr/bin/python2.6 +py2.4-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.4 +py2.5-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.5 +py2.6-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.6 +py2.4-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.4 +py2.5-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.5 +py2.6-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.6 +py2.4-solaris-2.8-sun4u_32-ucs2 = troegs.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.4 +py2.5-solaris-2.8-sun4u_32-ucs2 = troegs.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.5 +py2.6-solaris-2.8-sun4u_32-ucs2 = troegs.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.6 +py2.4-solaris-2.8-sun4u_64-ucs2 = troegs.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.4 +py2.5-solaris-2.8-sun4u_64-ucs2 = troegs.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.5 +py2.6-solaris-2.8-sun4u_64-ucs2 = troegs.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.6 + +; pysam doesn't build on solaris < 10, so for it alone we need these hosts +py2.4-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.4 +py2.5-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.5 +py2.6-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.6 +py2.4-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.4 +py2.5-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.5 +py2.6-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.6 + +; these hosts are used to build eggs with no C extensions +py2.4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.4 +py2.5 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.5 +py2.6 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.6 [groups] py2.4-linux-i686 = py2.4-linux-i686-ucs2 py2.4-linux-i686-ucs4 @@ -47,20 +63,37 @@ linux-x86_64 = py2.4-linux-x86_64 py2.5-linux-x86_64 py2.6-linux-x86_64 linux = linux-i686 linux-x86_64 py2.4-macosx = py2.4-macosx-10.3-fat-ucs2 -py2.5-macosx = py2.5-macosx-10.3-fat-ucs2 py2.5-macosx-10.5-i386-ucs2 -py2.6-macosx = py2.6-macosx-10.3-fat-ucs2 +py2.5-macosx = py2.5-macosx-10.3-fat-ucs2 +py2.6-macosx = py2.6-macosx-10.3-fat-ucs2 py2.6-macosx-10.6-universal-ucs2 macosx = py2.4-macosx py2.5-macosx py2.6-macosx -py2.4-solaris = py2.4-solaris-2.10-i86pc-ucs2 py2.4-solaris-2.11-i86pc-ucs2 py2.4-solaris-2.10-sun4u-ucs2 -py2.5-solaris = py2.5-solaris-2.10-i86pc-ucs2 py2.5-solaris-2.11-i86pc-ucs2 py2.5-solaris-2.10-sun4u-ucs2 -py2.6-solaris = py2.6-solaris-2.10-i86pc-ucs2 py2.6-solaris-2.11-i86pc-ucs2 py2.6-solaris-2.10-sun4u-ucs2 -solaris = py2.4-solaris py2.5-solaris py2.6-solaris +py2.4-solaris-i86pc = py2.4-solaris-2.10-i86pc_32-ucs2 py2.4-solaris-2.10-i86pc_64-ucs2 +py2.5-solaris-i86pc = py2.5-solaris-2.10-i86pc_32-ucs2 py2.5-solaris-2.10-i86pc_64-ucs2 +py2.6-solaris-i86pc = py2.6-solaris-2.10-i86pc_32-ucs2 py2.6-solaris-2.10-i86pc_64-ucs2 +py2.4-solaris-sun4u = py2.4-solaris-2.8-sun4u_32-ucs2 py2.4-solaris-2.8-sun4u_64-ucs2 +py2.5-solaris-sun4u = py2.5-solaris-2.8-sun4u_32-ucs2 py2.5-solaris-2.8-sun4u_64-ucs2 +py2.6-solaris-sun4u = py2.6-solaris-2.8-sun4u_32-ucs2 py2.6-solaris-2.8-sun4u_64-ucs2 +py2.4-solaris = py2.4-solaris-i86pc py2.4-solaris-sun4u +py2.5-solaris = py2.5-solaris-i86pc py2.5-solaris-sun4u +py2.6-solaris = py2.6-solaris-i86pc py2.6-solaris-sun4u +solaris-i86pc = py2.4-solaris-i86pc py2.5-solaris-i86pc py2.6-solaris-i86pc +solaris-sun4u = py2.4-solaris-sun4u py2.5-solaris-sun4u py2.6-solaris-sun4u +solaris = solaris-i86pc solaris-sun4u py2.4-all = py2.4-linux py2.4-macosx py2.4-solaris py2.5-all = py2.5-linux py2.5-macosx py2.5-solaris py2.6-all = py2.6-linux py2.6-macosx py2.6-solaris -all = py2.4-all py2.5-all py2.6-all -; default hosts for platform-inspecific eggs -noplatform = py2.4-linux-i686-ucs4 py2.5-linux-i686-ucs4 py2.6-linux-i686-ucs4 +; group for building pysam on solaris 10 sparc +solaris-2.10-sun4u = py2.4-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_32-ucs2 py2.4-solaris-2.10-sun4u_64-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 + +; the 'all' key is used internally by the build system to specify which hosts +; to build on when no hosts are specified on the dist-eggs.py command line. +all = linux macosx solaris + +; the 'noplatform' key, likewise, is for which build hosts should be used when +; building pure python (noplatform) eggs. +noplatform = py2.4 py2.5 py2.6 + +; don't build these eggs on these platforms: [ignore] -; Don't build these eggs on these platforms: -GeneTrack = py2.4-noplatform +GeneTrack = py2.4 +pysam = py2.4-solaris-2.8-sun4u_32-ucs2 py2.5-solaris-2.8-sun4u_32-ucs2 py2.6-solaris-2.8-sun4u_32-ucs2 py2.4-solaris-2.8-sun4u_64-ucs2 py2.5-solaris-2.8-sun4u_64-ucs2 py2.6-solaris-2.8-sun4u_64-ucs2 diff -r a6c9862ece3d -r 0cb68935eede eggs.ini --- a/eggs.ini Mon Mar 01 13:30:37 2010 -0500 +++ b/eggs.ini Mon Mar 01 15:41:30 2010 -0500 @@ -3,11 +3,11 @@ ; ; This file is version controlled and should not be edited by hand! ; For more information, see: -; http://g2.trac.bx.psu.edu/wiki/GalaxyEggs +; http://bitbucket.org/galaxy/galaxy-central/wiki/Config/Eggs ; [general] -repository = http://eggs.g2.bx.psu.edu +repository = http://eggs.g2.bx.psu.edu/new ; these eggs must be scrambled for your local environment no_auto = pbs_python DRMAA_python @@ -15,13 +15,13 @@ bx_python = 0.5.0 Cheetah = 2.2.2 DRMAA_python = 0.2 -MySQL_python = 1.2.2 +MySQL_python = 1.2.3c1 pbs_python = 2.9.4 psycopg2 = 2.0.6 pycrypto = 2.0.1 pysam = 0.1.1 pysqlite = 2.5.6 -python_lzo = 1.08 +python_lzo = 1.08_2.03_static threadframe = 0.2 guppy = 0.1.8 @@ -33,10 +33,9 @@ elementtree = 1.2.6_20050316 GeneTrack = 2.0.0_beta_1 lrucache = 0.2 -;lsprof - james Mako = 0.2.5 nose = 0.11.1 -NoseHTML = 0.3 +NoseHTML = 0.3.1 Paste = 1.6 PasteDeploy = 1.3.3 PasteScript = 1.7.3 @@ -58,52 +57,16 @@ ; extra version information [tags] psycopg2 = _8.2.6_static -pysqlite = _static -MySQL_python = _5.0.67_static -python_lzo = _static +pysqlite = _3.6.17_static +MySQL_python = _5.1.41_static bx_python = _dev_3b9d30e47619 GeneTrack = _dev_bf44f7054c30f19bd9f79106c1fd69849562b361 SQLAlchemy = _dev_r6498 -; nose = .dev_r7156749efc58 +pysam = _kanwei_90e03180969d -; source location, necessary for scrambling +; dependency source urls, necessary for scrambling. for an explanation, see +; the wiki page above [source] -bx_python = http://bitbucket.org/james_taylor/bx-python/get/3b9d30e47619.bz2 -Cheetah = http://pypi.python.org/packages/source/C/Cheetah/Cheetah-2.2.2.tar.gz -DRMAA_python = http://gridengine.sunsource.net/files/documents/7/36/DRMAA-python-0.2.tar.gz -MySQL_python = http://downloads.sourceforge.net/project/mysql-python/mysql-python/1.2.2/MyS... http://downloads.mysql.com/archives/mysql-5.0/mysql-5.0.67.tar.gz -pbs_python = http://ftp.sara.nl/pub/outgoing/pbs_python-2.9.4.tar.gz -pexpect = http://pypi.python.org/packages/source/p/pexpect/pexpect-2.4.tar.gz -psycopg2 = http://initd.org/pub/software/psycopg/PSYCOPG-2-0/psycopg2-2.0.6.tar.gz ftp://ftp-archives.postgresql.org/pub/source/v8.2.6/postgresql-8.2.6.tar.bz2 -pycrypto = http://www.amk.ca/files/python/crypto/pycrypto-2.0.1.tar.gz -pysam = http://bitbucket.org/kanwei/kanwei-pysam/get/e3c601a062fd.gz -pysqlite = http://pypi.python.org/packages/source/p/pysqlite/pysqlite-2.5.6.tar.gz -python_lzo = http://www.oberhumer.com/opensource/lzo/download/LZO-v1/python-lzo-1.08.tar.... http://www.oberhumer.com/opensource/lzo/download/LZO-v1/lzo-1.08.tar.gz -threadframe = http://www.majid.info/python/threadframe/threadframe-0.2.tar.gz -guppy = http://pypi.python.org/packages/source/g/guppy/guppy-0.1.8.tar.gz -amqplib = http://py-amqplib.googlecode.com/files/amqplib-0.6.1.tgz -Beaker = http://cheeseshop.python.org/packages/source/B/Beaker/Beaker-1.4.tar.gz -decorator = http://pypi.python.org/packages/source/d/decorator/decorator-3.1.2.tar.gz -docutils = http://downloads.sourceforge.net/docutils/docutils-0.4.tar.gz -elementtree = http://effbot.org/downloads/elementtree-1.2.6-20050316.tar.gz -GeneTrack = http://github.com/ialbert/genetrack-central/tarball/bf44f7054c30f19bd9f79106... -lrucache = http://evan.prodromou.name/lrucache/lrucache-0.2.tar.gz -Mako = http://www.makotemplates.org/downloads/Mako-0.2.5.tar.gz -nose = http://pypi.python.org/packages/source/n/nose/nose-0.11.1.tar.gz -NoseHTML = http://bitbucket.org/james_taylor/nosehtml/get/c46a54d569ae.bz2 -Paste = http://cheeseshop.python.org/packages/source/P/Paste/Paste-1.6.tar.gz -PasteDeploy = http://cheeseshop.python.org/packages/source/P/PasteDeploy/PasteDeploy-1.3.3... -PasteScript = http://cheeseshop.python.org/packages/source/P/PasteScript/PasteScript-1.7.3... -Routes = http://pypi.python.org/packages/source/R/Routes/Routes-1.11.tar.gz -simplejson = http://cheeseshop.python.org/packages/source/s/simplejson/simplejson-1.5.tar... -SQLAlchemy = http://dist.g2.bx.psu.edu/SQLAlchemy-0.5.6_r6498.tar.bz2 -sqlalchemy_migrate = http://pypi.python.org/packages/source/s/sqlalchemy-migrate/sqlalchemy-migra... -Tempita = http://pypi.python.org/packages/source/T/Tempita/Tempita-0.1.tar.gz -twill = http://darcs.idyll.org/~t/projects/twill-0.9.tar.gz -WebError = http://pypi.python.org/packages/source/W/WebError/WebError-0.8a.tar.gz -WebHelpers = http://pypi.python.org/packages/source/W/WebHelpers/WebHelpers-0.2.tar.gz -WebOb = http://pypi.python.org/packages/source/W/WebOb/WebOb-0.8.5.tar.gz -wsgiref = http://pypi.python.org/packages/source/w/wsgiref/wsgiref-0.1.2.zip -Babel = http://ftp.edgewall.com/pub/babel/Babel-0.9.4.zip -wchartype = http://ginstrom.com/code/wchartype-0.1.zip -boto = http://boto.googlecode.com/files/boto-1.8d.tar.gz +MySQL_python = mysql-5.1.41 +psycopg2 = postgresql-8.2.6 +pysqlite = sqlite-amalgamation-3_6_17 diff -r a6c9862ece3d -r 0cb68935eede lib/galaxy/__init__.py --- a/lib/galaxy/__init__.py Mon Mar 01 13:30:37 2010 -0500 +++ b/lib/galaxy/__init__.py Mon Mar 01 15:41:30 2010 -0500 @@ -2,30 +2,62 @@ Galaxy root package -- this is a namespace package. """ -# Starting somewhere in 2.5.x, Python on Mac became broken - despite being fat, -# the machine portion of the platform is not set to 'fat'. -# -# For more, see: -# -# http://bugs.python.org/setuptools/issue19 -# -import os, sys -from distutils.sysconfig import get_config_vars +__import__( "pkg_resources" ).declare_namespace( __name__ ) -if sys.version_info[:2] == ( 2, 5 ) and \ +import os, sys, re +from distutils.sysconfig import get_config_var, get_config_vars + +import pkg_resources + +# patch get_platform() for better ABI recognition +def _get_build_platform(): + plat = pkg_resources._get_build_platform() + if sys.version_info[:2] == ( 2, 5 ) and \ ( ( os.uname()[-1] in ( 'i386', 'ppc' ) and sys.platform == 'darwin' and os.path.abspath( sys.prefix ).startswith( '/System' ) ) or \ ( sys.platform == 'darwin' and get_config_vars().get('UNIVERSALSDK', '').strip() ) ): - # Has to be before anything imports pkg_resources - def _get_platform_monkeypatch(): - plat = distutils.util._get_platform() - if plat.startswith( 'macosx-' ): - plat = 'macosx-10.3-fat' - return plat - import distutils.util - try: - assert distutils.util._get_platform - except: - distutils.util._get_platform = distutils.util.get_platform - distutils.util.get_platform = _get_platform_monkeypatch + plat = 'macosx-10.3-fat' + if sys.platform == "sunos5" and not (plat.endswith('_32') or plat.endswith('_64')): + if sys.maxint > 2**31: + plat += '_64' + else: + plat += '_32' + if not (plat.endswith('-ucs2') or plat.endswith('-ucs4')): + if sys.maxunicode > 2**16: + plat += '-ucs4' + else: + plat += '-ucs2' + return plat +try: + assert pkg_resources._get_build_platform +except: + pkg_resources._get_build_platform = pkg_resources.get_build_platform + pkg_resources.get_build_platform = _get_build_platform + pkg_resources.get_platform = _get_build_platform -__import__( "pkg_resources" ).declare_namespace( __name__ ) +# patch compatible_platforms() to allow for Solaris binary compatibility +solarisVersionString = re.compile(r"solaris-(\d)\.(\d+)-(.*)") +def _compatible_platforms(provided,required): + # this is a bit kludgey since we need to know a bit about what happened in + # the original method + if provided is None or required is None or provided==required: + return True # easy case + reqMac = pkg_resources.macosVersionString.match(required) + if reqMac: + return pkg_resources._compatible_platforms(provided,required) + reqSol = solarisVersionString.match(required) + if reqSol: + provSol = solarisVersionString.match(provided) + if not provSol: + return False + if provSol.group(1) != reqSol.group(1) or \ + provSol.group(3) != reqSol.group(3): + return False + if int(provSol.group(2)) > int(reqSol.group(2)): + return False + return True + return False +try: + assert pkg_resources._compatible_platforms +except: + pkg_resources._compatible_platforms = pkg_resources.compatible_platforms + pkg_resources.compatible_platforms = _compatible_platforms diff -r a6c9862ece3d -r 0cb68935eede lib/galaxy/eggs/__init__.py --- a/lib/galaxy/eggs/__init__.py Mon Mar 01 13:30:37 2010 -0500 +++ b/lib/galaxy/eggs/__init__.py Mon Mar 01 15:41:30 2010 -0500 @@ -2,17 +2,15 @@ Manage Galaxy eggs """ -import os, sys, shutil, tarfile, zipfile, zipimport, subprocess, ConfigParser, glob, urllib2, shutil -from types import ModuleType +import os, sys, glob, urllib, urllib2, ConfigParser, HTMLParser, zipimport, zipfile import logging log = logging.getLogger( __name__ ) import pkg_resources -# we MUST have the top level galaxy dir for automatic egg fetching -# within tools. i don't know of any way around this. -ndc -galaxy_dir = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "..", ".." ) ) +galaxy_dir = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', '..', '..' ) ) +py = 'py%s' % sys.version[:3] class EggNotFetchable( Exception ): def __init__( self, eggs ): @@ -23,452 +21,287 @@ def __str__( self ): return ' '.join( self.eggs ) -class PlatformNotSupported( Exception ): - pass - -# TODO: we should really be using exceptions instead of returns for everything - # need the options to remain case sensitive -class CSConfigParser( ConfigParser.SafeConfigParser ): +class CaseSensitiveConfigParser( ConfigParser.SafeConfigParser ): def optionxform( self, optionstr ): return optionstr +# so we can actually detect failures +class URLRetriever( urllib.FancyURLopener ): + def http_error_default( *args ): + urllib.URLopener.http_error_default( *args ) + class Egg( object ): """ - Contains information about locating, downloading and scrambling eggs. + Contains information about locating and downloading eggs. """ - archive_dir = os.path.join( galaxy_dir, "scripts", "scramble", "archives" ) - script_dir = os.path.join( galaxy_dir, "scripts", "scramble", "scripts" ) - build_dir = os.path.join( galaxy_dir, "scripts", "scramble", "build" ) - ez_setup = os.path.join( galaxy_dir, "scripts", "scramble", "lib", "ez_setup.py" ) - ez_setup_url = "http://peak.telecommunity.com/dist/ez_setup.py" - def __init__( self ): - self.name = None - self.version = None - self.tag = None - self.sources = [] - self.platform = {} - self.url = None - self.have = False - self.path = None - self.doppelgangers = [] - self.buildpath = None + def __init__( self, name=None, version=None, tag=None, url=None, platform=None ): + self.name = name + self.version = version + self.tag = tag + self.url = url + self.platform = platform + self.distribution = None self.dir = None - self.build_host = None - self.python = sys.executable + if self.name is not None and self.version is not None: + self.set_distribution() def set_dir( self ): - if self.build_host is not None: - self.dir = os.path.join( galaxy_dir, "dist-eggs", self.platform['galaxy'] ) - else: - self.dir = os.path.join( galaxy_dir, "eggs", self.platform['galaxy'] ) - def get_namever( self ): - return( "%s-%s" %( self.name, self.version ) ) - def get_namevertag( self ): - if self.tag is None: - return( "%s-%s" %( self.name, self.version ) ) - else: - return( "%s-%s%s" %( self.name, self.version, self.tag ) ) - def get_vertag( self ): - if self.tag is None: - return self.version - else: - return( "%s%s" %( self.version, self.tag ) ) - def get_filename( self ): - if self.tag is None: - return( "%s-%s-%s.egg" %( self.name, self.version, self.platform['peak'] ) ) - else: - return( "%s-%s%s-%s.egg" %( self.name, self.version, self.tag, self.platform['peak'] ) ) - def find( self ): - # TODO: should be able to set a search path in eggs.ini + self.dir = os.path.join( galaxy_dir, 'eggs' ) + if not os.path.exists( self.dir ): + os.makedirs( self.dir ) + def set_distribution( self ): + """ + Stores a pkg_resources Distribution object for reference later + """ if self.dir is None: self.set_dir() - self.path = os.path.join( self.dir, self.get_filename() ) - self.doppelgangers = glob.glob( os.path.join( self.dir, "%s-*-%s.egg" % (self.name, self.platform['peak'] ) ) ) - if os.access( self.path, os.F_OK ): - self.have = True - self.doppelgangers.remove( self.path ) - def fetch( self ): - if self.path is None: - self.find() - if not self.have: - if not os.access( os.path.dirname( self.path ), os.F_OK ): - os.makedirs( os.path.dirname( self.path ) ) + tag = self.tag or '' + self.distribution = pkg_resources.Distribution.from_filename( + os.path.join( self.dir, '-'.join( ( self.name, self.version + tag, self.platform ) ) + '.egg' ) ) + @property + def path( self ): + """ + Return the path of the egg, if it exists, or None + """ + if env[self.distribution.project_name]: + return env[self.distribution.project_name][0].location + return None + def fetch( self, requirement ): + """ + fetch() serves as the install method to pkg_resources.working_set.resolve() + """ + def find_alternative(): + """ + Some platforms (e.g. Solaris) support eggs compiled on older platforms + """ + class LinkParser( HTMLParser.HTMLParser ): + """ + Finds links in what should be an Apache-style directory index + """ + def __init__( self ): + HTMLParser.HTMLParser.__init__( self ) + self.links = [] + def handle_starttag( self, tag, attrs ): + if tag == 'a' and 'href' in dict( attrs ): + self.links.append( dict( attrs )['href'] ) + parser = LinkParser() try: - inf = urllib2.urlopen( self.url ) - otf = open( self.path, 'wb' ) - otf.write( inf.read() ) - inf.close() - otf.close() - log.debug( "Fetched %s" % self.url ) + parser.feed( urllib2.urlopen( self.url + '/' ).read() ) except urllib2.HTTPError, e: - raise EggNotFetchable( self.name ) - #if e.code == 404: - # return False - self.unpack_if_needed() - for doppelganger in self.doppelgangers: - remove_file_or_path( doppelganger ) - log.debug( "Removed conflicting egg: %s" % doppelganger ) - return True + if e.code == 404: + return None + parser.close() + for link in parser.links: + file = urllib.unquote( link ).rsplit( '/', 1 )[-1] + tmp_dist = pkg_resources.Distribution.from_filename( file ) + if tmp_dist.platform is not None and \ + self.distribution.project_name == tmp_dist.project_name and \ + self.distribution.version == tmp_dist.version and \ + pkg_resources.compatible_platforms( tmp_dist.platform, pkg_resources.get_platform() ): + return file + return None + if self.url is None: + return None + alternative = None + try: + url = self.url + '/' + self.distribution.egg_name() + '.egg' + URLRetriever().retrieve( url, self.distribution.location ) + log.debug( "Fetched %s" % url ) + except IOError, e: + if e[1] == 404 and self.distribution.platform != py: + alternative = find_alternative() + if alternative is None: + return None + else: + return None + if alternative is not None: + try: + url = '/'.join( ( self.url, alternative ) ) + URLRetriever().retrieve( url, os.path.join( self.dir, alternative ) ) + log.debug( "Fetched %s" % url ) + except IOError, e: + return None + self.platform = alternative.split( '-', 2 )[-1].rsplit( '.egg', 1 )[0] + self.set_distribution() + self.unpack_if_needed() + self.remove_doppelgangers() + global env + env = get_env() # reset the global Environment object now that we've obtained a new egg + return self.distribution def unpack_if_needed( self ): - meta = pkg_resources.EggMetadata( zipimport.zipimporter( self.path ) ) + meta = pkg_resources.EggMetadata( zipimport.zipimporter( self.distribution.location ) ) if meta.has_metadata( 'not-zip-safe' ): - unpack_zipfile( self.path, self.path + "-tmp" ) - os.remove( self.path ) - os.rename( self.path + "-tmp", self.path ) - def scramble( self, dist=False ): - if self.path is None: - self.find() - if os.access( self.path, os.F_OK ): - log.warning( "scramble(): Egg already exists, remove to force rebuild:" ) - log.warning( " %s" % self.path ) - return True - self.fetch_source() - self.unpack_source() - self.copy_build_script() - if not os.access( Egg.ez_setup, os.F_OK ): - if not os.access( os.path.dirname( Egg.ez_setup ), os.F_OK ): - os.makedirs( os.path.dirname( Egg.ez_setup ) ) - inf = urllib2.urlopen( Egg.ez_setup_url ) - otf = open( Egg.ez_setup, 'wb' ) - otf.write( inf.read() ) - inf.close() - otf.close() - shutil.copyfile( Egg.ez_setup, os.path.join( self.buildpath, "ez_setup.py" ) ) - log.warning( "scramble(): Beginning build" ) - # subprocessed to sterilize the env - if self.build_host is not None: - cmd = "ssh %s 'cd %s; %s -ES %s'" % ( self.build_host, self.buildpath, self.python, "scramble.py" ) - else: - cmd = "%s -ES %s" % ( self.python, "scramble.py" ) - log.debug( 'Executing: %s' % cmd ) - p = subprocess.Popen( args = cmd, shell = True, cwd = self.buildpath ) - r = p.wait() - if r != 0: - log.error( "scramble(): Egg build failed for %s" % self.get_namevertag() ) - return False - new_egg = os.path.join( self.buildpath, "dist", os.path.basename( self.path ) ) - if not os.access( os.path.dirname( self.path ), os.F_OK ): - os.makedirs( os.path.dirname( self.path ) ) - shutil.copyfile( new_egg, self.path ) - log.warning( "scramble(): Copied egg to:" ) - log.warning( " %s" % self.path ) - if not dist: - self.unpack_if_needed() - for doppelganger in self.doppelgangers: + unpack_zipfile( self.distribution.location, self.distribution.location + "-tmp" ) + os.remove( self.distribution.location ) + os.rename( self.distribution.location + "-tmp", self.distribution.location ) + def remove_doppelgangers( self ): + doppelgangers = glob.glob( os.path.join( self.dir, "%s-*-%s.egg" % ( self.name, self.platform ) ) ) + if self.distribution.location in doppelgangers: + doppelgangers.remove( self.distribution.location ) + for doppelganger in doppelgangers: remove_file_or_path( doppelganger ) - log.warning( "Removed conflicting egg: %s" % doppelganger ) - return True - # scramble helper methods - def get_archive_path( self, url ): - return os.path.join( Egg.archive_dir, (url.rsplit( '/', 1 ))[1] ) - def get_tld( self, names ): - tld = names[0].split( os.path.sep, 1 )[0] - for name in names: - try: - assert tld == name.split( os.path.sep, 1 )[0] - except: - raise Exception( "get_tld(): Archive contains multiple top-level directories!" ) - return tld - def fetch_source( self ): - if not os.access( Egg.archive_dir, os.F_OK ): - os.makedirs( Egg.archive_dir ) - for source_url in self.sources: - source_path = self.get_archive_path( source_url ) - if os.access( source_path, os.F_OK ): - log.warning( "fetch_source(): Using existing source, remove to download again:" ) - log.warning( " %s" % source_path ) - continue - log.warning( "fetch_source(): Attempting to download" ) - log.warning( " %s" % source_url ) - inf = urllib2.urlopen( source_url ) - otf = open( source_path, 'wb' ) - otf.write( inf.read() ) - inf.close() - otf.close() - log.warning( "fetch_source(): Fetched %s" % source_url ) - def unpack_source( self ): - unpack_dir = os.path.join( Egg.build_dir, self.platform['galaxy'] ) - if not os.access( unpack_dir, os.F_OK ): - os.makedirs( unpack_dir ) - self.buildpath = os.path.join( unpack_dir, self.name ) - if os.access( self.buildpath, os.F_OK ): - log.warning( "Removing old build directory at:" ) - log.warning( " %s" % self.buildpath ) - shutil.rmtree( self.buildpath ) - source_path = self.get_archive_path( self.sources[0] ) - if tarfile.is_tarfile( source_path ): - self.unpack = self.unpack_tar - elif zipfile.is_zipfile( source_path ): - self.unpack = self.unpack_zip - else: - raise Exception( "unpack_source(): Unknown archive file type for %s" % source_path ) - self.unpack( source_path, unpack_dir ) - log.warning( "unpack_source(): Unpacked to:" ) - log.warning( " %s" % self.buildpath ) - def unpack_zip( self, source_path, unpack_path ): - z = zipfile.ZipFile( source_path, "r" ) - tld = self.get_tld( z.namelist() ) - cur = os.getcwd() - os.chdir( unpack_path ) - for fn in z.namelist(): - if "ez_setup" in fn: - continue - if not os.access( os.path.dirname( fn ), os.F_OK ): - os.makedirs( os.path.dirname( fn ) ) - otf = open( fn, "wb" ) - otf.write( z.read( fn ) ) - otf.close() - z.close() - os.rename( tld, self.name ) - os.chdir( cur ) - def unpack_tar( self, source_path, unpack_path ): - t = tarfile.open( source_path, "r" ) - members = filter( lambda x: "ez_setup" not in x.name and "pax_global_header" != x.name, t.getmembers() ) - tld = self.get_tld( [ x.name for x in members ] ) - cur = os.getcwd() - os.chdir( unpack_path ) - for member in members: - t.extract( member ) - t.close() - os.rename( tld, self.name ) - os.chdir( cur ) - def copy_build_script( self ): - peak_platform = get_platform( platform=True, peak=True ) - nopy_platform = ( peak_platform.split( "-", 1 ) )[1] - just_os = ( nopy_platform.split( "-", 1 ) )[0] - just_py = ( peak_platform.split( "-", 1 ) )[0] - # will try: - # bx_python-py2.4-solaris-2.11-i86pc.py - # bx_python-py2.4-solaris.py - # bx_python-solaris-2.11-i86pc.py - # bx_python-solaris.py - # bx_python-py2.4.py - # bx_python.py - # generic.py - build_scripts = [ - os.path.join( Egg.script_dir, "%s-%s.py" % ( self.name, peak_platform ) ), - os.path.join( Egg.script_dir, "%s-%s-%s.py" % ( self.name, just_py, just_os ) ), - os.path.join( Egg.script_dir, "%s-%s.py" % ( self.name, nopy_platform ) ), - os.path.join( Egg.script_dir, "%s-%s.py" % ( self.name, just_os ) ), - os.path.join( Egg.script_dir, "%s-%s.py" % ( self.name, just_py ) ), - os.path.join( Egg.script_dir, "%s.py" % self.name ), - os.path.join( Egg.script_dir, "generic.py" ) - ] - for build_script in build_scripts: - try: - f = open( build_script, "r" ) - f.close() - log.warning( "scramble(): Using build script %s" % build_script ) - break - except IOError: - pass - shutil.copyfile( build_script, os.path.join( self.buildpath, "scramble.py" ) ) - if self.tag is not None: - tagfile = open( os.path.join( self.buildpath, ".galaxy_tag" ), "w" ) - print >>tagfile, self.tag - tagfile.close() + log.debug( "Removed conflicting egg: %s" % doppelganger ) + def resolve( self ): + try: + return pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) + except pkg_resources.DistributionNotFound, e: + # If this statement is true, it means we do have the requested egg, + # just not one (or more) of its deps. + if e.args[0].project_name != self.distribution.project_name: + log.warning( "Warning: %s (a dependant egg of %s) cannot be fetched" % ( e.args[0].project_name, self.distribution.project_name ) ) + return ( self.distribution, ) + else: + raise EggNotFetchable( self ) + except pkg_resources.VersionConflict, e: + # there's a conflicting egg on the path, remove it + dist = e.args[0] + # use the canonical path for comparisons + location = os.path.realpath( dist.location ) + for entry in pkg_resources.working_set.entries: + if os.path.realpath( entry ) == location: + pkg_resources.working_set.entries.remove( entry ) + break + else: + location = entry = None + del pkg_resources.working_set.by_key[dist.key] + if entry is not None: + pkg_resources.working_set.entry_keys[entry] = [] + if entry in sys.path: + sys.path.remove(entry) + r = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) + if location is not None and not location.endswith( '.egg' ): + # re-add the path if it's a non-egg dir, in case more deps live there + pkg_resources.working_set.entries.append( location ) + sys.path.append( location ) + return r + def require( self ): + try: + dists = self.resolve() + for dist in dists: + pkg_resources.working_set.add( dist ) + return dists + except: + raise class Crate( object ): """ - Reads the eggs.ini file for use with checking, fetching and scrambling eggs. + Reads the eggs.ini file for use with checking and fetching. """ - config_file = os.path.join( galaxy_dir, "eggs.ini" ) + config_file = os.path.join( galaxy_dir, 'eggs.ini' ) def __init__( self ): self.eggs = {} - self.config = CSConfigParser() + self.config = CaseSensitiveConfigParser() self.repo = None self.no_auto = [] - self.platform = { 'peak' : get_platform( platform=True, peak=True ), 'galaxy' : get_platform( platform=True, peak=False ) } - self.noplatform = { 'peak' : get_platform( platform=False, peak=True ), 'galaxy' : get_platform( platform=False, peak=False ) } + self.galaxy_config = GalaxyConfig() + self.parse() def parse( self ): - if self.config.read( Crate.config_file ) == []: - raise Exception( "unable to read egg config from %s" % Crate.config_file ) - try: - self.repo = self.config.get( "general", "repository" ) - self.no_auto = self.config.get( "general", "no_auto" ).split() - except ConfigParser.NoSectionError: - raise Exception( "eggs.ini is missing required section [general]" ) - #except ConfigParser.NoOptionError: - # raise Exception( "eggs.ini is missing required [general] option 'repository'" ) - try: - platform_eggs = self.config.items( "eggs:platform" ) - noplatform_eggs = self.config.items( "eggs:noplatform" ) - except ConfigParser.NoSectionError, e: - raise Exception( "eggs.ini is missing required section: %s" % e ) - self.parse_egg_section( platform_eggs, self.platform ) - self.parse_egg_section( noplatform_eggs, self.noplatform ) - def parse_egg_section( self, eggs, platform ): + self.config.read( Crate.config_file ) + self.repo = self.config.get( 'general', 'repository' ) + self.no_auto = self.config.get( 'general', 'no_auto' ).split() + self.parse_egg_section( self.config.items( 'eggs:platform' ), self.config.items( 'tags' ), True ) + self.parse_egg_section( self.config.items( 'eggs:noplatform' ), self.config.items( 'tags' ) ) + def parse_egg_section( self, eggs, tags, full_platform=False, egg_class=Egg ): for name, version in eggs: - egg = Egg() + tag = dict( tags ).get( name, '' ) + url = '/'.join( ( self.repo, name ) ) + if full_platform: + platform = '-'.join( ( py, pkg_resources.get_platform() ) ) + else: + platform = py + egg = egg_class( name, version, tag, url, platform ) + self.eggs[name] = egg + @property + def config_missing( self ): + """ + Return true if any eggs are missing, conditional on options set in the + Galaxy config file. + """ + for egg in self.config_eggs: + if not egg.path: + return True + return False + @property + def all_missing( self ): + """ + Return true if any eggs in the eggs config file are missing. + """ + for egg in self.all_eggs: + if not os.path.exists( egg.distribution.location ): + return True + return False + @property + def config_names( self ): + """ + Return a list of names of all eggs in the crate that are needed based + on the options set in the Galaxy config file. + """ + return [ egg.name for egg in self.config_eggs ] + @property + def all_names( self ): + """ + Return a list of names of all eggs in the crate. + """ + return [ egg.name for egg in self.all_eggs ] + @property + def config_eggs( self ): + """ + Return a list of all eggs in the crate that are needed based on the + options set in the Galaxy config file. + """ + return [ egg for egg in self.eggs.values() if self.galaxy_config.check_conditional( egg.name ) ] + @property + def all_eggs( self ): + """ + Return a list of all eggs in the crate. + """ + rval = [] + for egg in self.eggs.values(): + if egg.name not in self.galaxy_config.always_conditional: + rval.append( egg ) + elif self.galaxy_config.check_conditional( egg.name ): + rval.append( egg ) + return rval + def __getitem__( self, name ): + """ + Return a specific egg. + """ + name = name.replace( '-', '_' ) + return self.eggs[name] + def resolve( self, all=False ): + """ + Try to resolve (e.g. fetch) all eggs in the crate. + """ + if all: + eggs = self.all_eggs + else: + eggs = self.config_eggs + eggs = filter( lambda x: x.name not in self.no_auto, eggs ) + missing = [] + for egg in eggs: try: - egg.tag = self.config.get( "tags", name ) + egg.resolve() except: - egg.tag = None - try: - egg.sources = self.config.get( "source", name ).split() - except: - egg.sources = None - egg.name = name - egg.version = version - egg.platform['galaxy'] = platform['galaxy'] - egg.platform['peak'] = platform['peak'] - egg.url = "%s/%s/%s" %( self.repo, platform['galaxy'], egg.get_filename() ) - self.eggs[name] = egg - def find( self, ignore=None ): - missing = [] - for egg in self.eggs.itervalues(): - if ignore is not None: - if egg.name in ignore: - continue - egg.find() - if not egg.have: - missing.append( egg.name ) - if len( missing ): - return False - return True - def fetch( self, ignore=[] ): - """ - Fetch all eggs in the crate (ignoring any that you want to - ignore). If your platform isn't available, it'll attempt to - download all the noplatform eggs before failing. - """ - skip_platform = False - ignore.extend( self.no_auto ) - missing = [] - try: - f = urllib2.urlopen( "%s/%s" % ( self.repo, self.platform['galaxy'] ) ) - f.close() - except urllib2.HTTPError, e: - if e.code == 404: - skip_platform = True - for egg in self.eggs.itervalues(): - if ignore is not None: - if egg.name in ignore: - continue - if skip_platform and egg.platform['galaxy'] == self.platform['galaxy']: - missing.append( egg.name ) - continue - try: - egg.fetch() - except EggNotFetchable: - missing.append( egg.name ) - if skip_platform: - raise PlatformNotSupported( self.platform['galaxy'] ) + missing.append( egg ) if missing: raise EggNotFetchable( missing ) - return True - def scramble( self, ignore=None ): - # Crate-scrambling the no_auto eggs makes no sense - ignore.extend( self.no_auto ) - for egg in self.eggs.itervalues(): - if ignore is not None: - if egg.name in ignore: - continue - if not egg.scramble(): - return False - return True - def get_names( self ): - return self.eggs.keys() - def get( self, name ): - if self.eggs.has_key( name ): - return self.eggs[name] - else: - return None - def get_for_require( self, name ): - """ - return an egg based on a lowercase name comparo and with _ instead of -. used by require. - """ - for key in self.eggs.keys(): - if key.lower() == name.lower().replace( '-', '_' ): - return self.eggs[key] -class DistCrate( Crate ): - """ - A subclass of Crate that holds eggs with info on how to build them for distribution. - """ - dist_config_file = os.path.join( galaxy_dir, "dist-eggs.ini" ) - def __init__( self, build_on="all" ): - self.eggs = {} - self.config = CSConfigParser() - self.repo = None - self.build_on = build_on - self.platform = 'platform' - self.noplatform = 'noplatform' - def parse( self ): - if self.config.read( DistCrate.dist_config_file ) == []: - raise Exception( "unable to read dist egg config from %s" % DistCrate.dist_config_file ) - try: - self.hosts = dict( self.config.items( "hosts" ) ) - self.groups = dict( self.config.items( "groups" ) ) - self.ignore = dict( self.config.items( "ignore" ) ) - except ConfigParser.NoSectionError, e: - raise Exception( "eggs.ini is missing required section: %s" % e ) - self.platforms = self.get_platforms( self.build_on ) - self.noplatforms = self.get_platforms( 'noplatform' ) - Crate.parse( self ) - def get_platforms( self, wanted ): - # find all the members of a group and process them - if self.groups.has_key( wanted ): - platforms = [] - for name in self.groups[wanted].split(): - for platform in self.get_platforms( name ): - if platform not in platforms: - platforms.append( platform ) - return platforms - elif self.hosts.has_key( wanted ): - return [ wanted ] - else: - raise Exception( "unknown platform: %s" % wanted ) - def parse_egg_section( self, eggs, type ): - """ - Overrides the base class's method. Here we use the third arg - to find out what type of egg we'll be building. - """ - if type == "platform": - platforms = self.platforms - elif type == "noplatform": - platforms = self.noplatforms - for name, version in eggs: - for platform in platforms: - # can't use the regular methods here because we're not - # actually ON the target platform - if type == "platform": - gplat = platform - pplat = platform.rsplit('-', 1)[0] - elif type == "noplatform": - gplat = "%s-noplatform" % platform.split('-', 1)[0] - pplat = platform.split('-', 1)[0] - if name in self.ignore and gplat in self.ignore[name].split(): - continue - egg = Egg() - try: - egg.tag = self.config.get( "tags", name ) - except: - egg.tag = None - try: - egg.sources = self.config.get( "source", name ).split() - except: - egg.sources = None - egg.name = name - egg.version = version - egg.platform['galaxy'] = gplat - egg.platform['peak'] = pplat - egg.url = "%s/%s/%s" %( self.repo, gplat, egg.get_filename() ) - egg.build_host, egg.python = self.hosts[platform].split() - if not self.eggs.has_key( name ): - self.eggs[name] = [ egg ] - else: - self.eggs[name].append( egg ) - -class GalaxyConfig: +class GalaxyConfig( object ): config_file = os.path.join( galaxy_dir, "universe_wsgi.ini" ) - always_conditional = ( 'GeneTrack', ) + always_conditional = ( 'GeneTrack', 'pysam' ) def __init__( self ): self.config = ConfigParser.ConfigParser() if self.config.read( GalaxyConfig.config_file ) == []: raise Exception( "error: unable to read Galaxy config from %s" % GalaxyConfig.config_file ) - # TODO: conditionals should really be handled better than this def check_conditional( self, egg_name ): + def check_pysam(): + # can't build pysam on solaris < 10 + plat = pkg_resources.get_platform().split( '-' ) + if plat[0] == 'solaris': + minor = plat[1].split('.')[1] + if int( minor ) < 10: + return False + return True if egg_name == "pysqlite": # SQLite is different since it can be specified in two config vars and defaults to True try: @@ -484,102 +317,45 @@ "threadframe": lambda: self.config.get( "app:main", "use_heartbeat" ), "guppy": lambda: self.config.get( "app:main", "use_memdump" ), "GeneTrack": lambda: sys.version_info[:2] >= ( 2, 5 ), + "pysam": check_pysam() }.get( egg_name, lambda: True )() except: return False -def require( pkg ): - # add the egg dirs to sys.path if they're not already there - for platform in [ get_platform(), get_platform( platform=True ) ]: - path = os.path.join( galaxy_dir, "eggs", platform ) - if path not in sys.path: - new_path = [ path ] - new_path.extend( sys.path ) - sys.path = new_path - pkg_resources.working_set.add_entry(path) - name = pkg_resources.Requirement.parse( pkg ).project_name +def get_env(): + env = pkg_resources.Environment( platform=pkg_resources.get_platform() ) + for dist in pkg_resources.find_distributions( os.path.join( galaxy_dir, 'eggs' ), False ): + env.add( dist ) + return env +env = get_env() + +def require( req_str ): c = Crate() - c.parse() - egg = c.get_for_require( name ) + req = pkg_resources.Requirement.parse( req_str ) try: - if egg is None: - return pkg_resources.working_set.require( pkg ) - else: - return pkg_resources.working_set.require( "%s==%s" % ( name, egg.get_vertag() ) ) - except pkg_resources.VersionConflict, e: - # there's a conflicting egg on the pythonpath, remove it - dist = e.args[0] - working_set = pkg_resources.working_set - # use the canonical path for comparisons - location = os.path.realpath( dist.location ) - for entry in working_set.entries: - if os.path.realpath( entry ) == location: - working_set.entries.remove( entry ) - break - else: - location = None - del working_set.by_key[dist.key] - working_set.entry_keys[entry] = [] - try: - sys.path.remove(entry) - except ValueError: - raise Exception( "Cannot remove entry: %s" % str(entry) ) - r = require( pkg ) - if location is not None and not location.endswith( '.egg' ): - # re-add the path if it's a non-egg dir, in case more deps live there - working_set.entries.append( location ) # re-add to the set if it's a dir. - sys.path.append( location ) # re-add to the set if it's a dir. - return r - except pkg_resources.DistributionNotFound, e: - # the initial require itself is the first dep, but it can have - # multiple deps, which will be fetched by the require below. - dep = pkg_resources.Requirement.parse( str( e ) ).project_name - egg = c.get_for_require( dep ) - if egg is None: - # we don't have it and we can't get it - raise - egg.find() - if not egg.have: - if not egg.fetch(): - raise EggNotFetchable( egg.name ) - return require( pkg ) + return c[req.project_name].require() + except KeyError: + # not a galaxy-owned dependency + return pkg_resources.working_set.require( req_str ) + except EggNotFetchable, e: + raise EggNotFetchable( str( [ egg.name for egg in e.eggs ] ) ) +pkg_resources.require = require -# convenience stuff -def get_ucs(): - if sys.maxunicode > 65535: - return "ucs4" - else: - return "ucs2" - -def get_py(): - return "py%s" % sys.version[:3] - -def get_noplatform(): - return "%s-noplatform" % get_py() - -def get_platform( platform=False, peak=False ): - if platform: - if peak: - return "%s-%s" % ( get_py(), pkg_resources.get_platform() ) - else: - return "%s-%s-%s" % ( get_py(), pkg_resources.get_platform(), get_ucs() ) - else: - if peak: - return get_py() - else: - return "%s-noplatform" % get_py() - -def unpack_zipfile( filename, extract_dir): +def unpack_zipfile( filename, extract_dir, ignores=[] ): z = zipfile.ZipFile(filename) try: for info in z.infolist(): name = info.filename + perm = (info.external_attr >> 16L) & 0777 # don't extract absolute paths or ones with .. in them if name.startswith('/') or '..' in name: continue target = os.path.join(extract_dir, *name.split('/')) if not target: continue + for ignore in ignores: + if ignore in name: + continue if name.endswith('/'): # directory pkg_resources.ensure_directory(target) @@ -593,6 +369,11 @@ finally: f.close() del data + try: + if not os.path.islink(): + os.chmod(target, mode) + except: + pass finally: z.close() @@ -601,5 +382,3 @@ shutil.rmtree( f ) else: os.remove( f ) - -pkg_resources.require = require diff -r a6c9862ece3d -r 0cb68935eede lib/galaxy/eggs/dist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/eggs/dist.py Mon Mar 01 15:41:30 2010 -0500 @@ -0,0 +1,86 @@ +""" +Manage Galaxy eggs +""" + +import os, sys, subprocess +from scramble import ScrambleEgg, ScrambleCrate, ScrambleFailure, galaxy_dir, py +from __init__ import Crate, CaseSensitiveConfigParser + +import logging +log = logging.getLogger( __name__ ) + +class DistScrambleEgg( ScrambleEgg ): + def set_dir( self ): + self.dir = os.path.join( galaxy_dir, 'dist-eggs', self.name ) + if not os.path.exists( self.dir ): + os.makedirs( self.dir ) + @property + def path( self ): + # don't look for compatible eggs, look for exact matches + if os.path.exists( self.distribution.location ): + return self.distribution.location + return None + def run_scramble_script( self ): + log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name ) + # subprocessed to sterilize the env + cmd = "ssh %s 'cd %s; %s -ES %s'" % ( self.build_host, self.buildpath, self.python, 'scramble.py' ) + log.debug( '%s(): Executing:' % sys._getframe().f_code.co_name ) + log.debug( ' %s' % cmd ) + p = subprocess.Popen( args = cmd, shell = True ) + r = p.wait() + if r != 0: + raise ScrambleFailure( "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) ) + def unpack_if_needed( self ): + return # do not unpack dist eggs + +class DistScrambleCrate( ScrambleCrate ): + """ + Holds eggs with info on how to build them for distribution. + """ + dist_config_file = os.path.join( galaxy_dir, 'dist-eggs.ini' ) + def __init__( self, build_on='all' ): + self.dist_config = CaseSensitiveConfigParser() + self.build_on = build_on + ScrambleCrate.__init__( self ) + def parse( self ): + self.dist_config.read( DistScrambleCrate.dist_config_file ) + self.hosts = dict( self.dist_config.items( 'hosts' ) ) + self.groups = dict( self.dist_config.items( 'groups' ) ) + self.ignore = dict( self.dist_config.items( 'ignore' ) ) + self.platforms = self.get_platforms( self.build_on ) + self.noplatforms = self.get_platforms( 'noplatform' ) + Crate.parse( self ) + def get_platforms( self, wanted ): + # find all the members of a group and process them + if self.groups.has_key( wanted ): + platforms = [] + for name in self.groups[wanted].split(): + for platform in self.get_platforms( name ): + if platform not in platforms: + platforms.append( platform ) + return platforms + elif self.hosts.has_key( wanted ): + return [ wanted ] + else: + raise Exception( "unknown platform: %s" % wanted ) + def parse_egg_section( self, eggs, tags, full_platform=False ): + for name, version in eggs: + self.eggs[name] = [] + tag = dict( tags ).get( name, '' ) + url = '/'.join( ( self.repo, name ) ) + try: + sources = self.config.get( 'source', name ).split() + except: + sources = [] + if full_platform: + platforms = self.platforms + else: + platforms = self.noplatforms + for platform in platforms: + if name in self.ignore and platform in self.ignore[name].split(): + continue + egg = DistScrambleEgg( name, version, tag, url, platform ) + host_info = self.hosts[platform].split() + egg.build_host, egg.python = host_info[:2] + egg.sources = sources + self.eggs[name].append( egg ) diff -r a6c9862ece3d -r 0cb68935eede lib/galaxy/eggs/scramble.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/eggs/scramble.py Mon Mar 01 15:41:30 2010 -0500 @@ -0,0 +1,234 @@ +""" +Manage Galaxy eggs +""" + +import os, sys, shutil, tempfile, subprocess, urlparse, urllib +from __init__ import Egg, Crate, URLRetriever, galaxy_dir, py, unpack_zipfile +from distutils.sysconfig import get_config_var + +import tarfile, zipfile, zlib +arctypes = ( 'tar.gz', 'tgz', 'zip' ) + +import logging +log = logging.getLogger( __name__ ) + +import pkg_resources + +class ScrambleFailure( Exception ): + def __init__( self, eggs, msg=None ): + if type( eggs ) in ( list, tuple ): + self.eggs = eggs + else: + self.eggs = [ eggs ] + self.msg = msg + def __str__( self ): + return self.msg or ' '.join( self.eggs ) + +class ScrambleEgg( Egg ): + """ + Contains information about scrambling eggs. + """ + scramble_dir = os.path.join( galaxy_dir, 'scripts', 'scramble' ) + archive_dir = os.path.join( scramble_dir, 'archives' ) + script_dir = os.path.join( scramble_dir, 'scripts' ) + build_dir = os.path.join( scramble_dir, 'build' ) + ez_setup = os.path.join( scramble_dir, 'lib', 'ez_setup.py' ) + ez_setup_url = 'http://peak.telecommunity.com/dist/ez_setup.py' + def __init__( self, *args, **kwargs ): + Egg.__init__( self, *args, **kwargs ) + self.sources = [] + self.buildpath = None + self.source_path = None + self.py = py + self.build_host = None + self.python = sys.executable + def scramble( self ): + if self.path: + log.warning( "%s(): Egg already exists, remove to force rebuild:" % sys._getframe().f_code.co_name ) + log.warning( " %s" % self.path ) + return + self.fetch_source() + self.unpack_source() + self.copy_build_script() + if not os.path.exists( ScrambleEgg.ez_setup ): + URLRetriever().retrieve( ScrambleEgg.ez_setup_url, ScrambleEgg.ez_setup ) + shutil.copyfile( ScrambleEgg.ez_setup, os.path.join( self.buildpath, 'ez_setup.py' ) ) + self.run_scramble_script() + new_egg = os.path.join( self.buildpath, 'dist', os.path.basename( self.distribution.location ) ) + if not os.path.exists( new_egg ): + raise ScrambleFailure( self, "%s(): Egg build for %s did not appear to fail, but no egg found to copy from expected path:\n %s" % ( sys._getframe().f_code.co_name, self.name, egg_name ) ) + shutil.copyfile( new_egg, self.distribution.location ) + log.warning( "%s(): Copied egg to:" % sys._getframe().f_code.co_name ) + log.warning( " %s" % self.distribution.location ) + self.unpack_if_needed() + self.remove_doppelgangers() + # scramble helper methods + def get_tld( self, names ): + tld = names[0].split( os.path.sep, 1 )[0] + for name in names: + try: + assert tld == name.split( os.path.sep, 1 )[0] + except: + raise Exception( "%s(): Archive contains multiple top-level directories!" % sys._getframe().f_code.co_name ) + return tld + def fetch_one( self, urls ): + """ + Fetches the first available archive out of a list. + """ + for url in urls: + file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] ) + if os.path.exists( file ): + log.warning( "%s(): Using existing source, remove to download again:" % sys._getframe().f_code.co_name ) + log.warning( " %s" % file ) + return file + # if we don't have one, get one + for url in urls: + file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] ) + try: + log.debug( "%s(): Trying to fetch:" % sys._getframe().f_code.co_name ) + log.debug( " %s" % url ) + URLRetriever().retrieve( url, file + '.download' ) + shutil.move( file + '.download', file ) + log.debug( "%s(): Fetched to:" % sys._getframe().f_code.co_name ) + log.debug( " %s" % file ) + return file + except IOError, e: + if e[1] != 404: + raise + else: + return None + def fetch_source( self ): + """ + Get egg (and dependent) source + """ + if not os.path.exists( ScrambleEgg.archive_dir ): + os.makedirs( ScrambleEgg.archive_dir ) + urls = [] + url_base = self.url + '/' + '-'.join( ( self.name, self.version ) ) + urls.extend( map( lambda x: '.'.join( ( url_base, x ) ), arctypes ) ) + if self.tag: + urls.extend( map( lambda x: '.'.join( ( url_base + self.tag, x ) ), arctypes ) ) + self.source_path = self.fetch_one( urls ) + if self.source_path is None: + raise Exception( "%s(): Couldn't find a suitable source archive for %s %s from %s" % ( sys._getframe().f_code.co_name, self.name, self.version, self.url ) ) + for url in self.sources: + if not urlparse.urlparse( url )[0]: + url = self.url + '/' + url.lstrip( '/' ) + urls = [ url ] + urls.extend( map( lambda x: '.'.join( ( url, x ) ), arctypes ) ) # allows leaving off the extension and we'll try to find one + file = self.fetch_one( urls ) + if file is None: + raise Exception( "%s(): Couldn't fetch extra source for %s, check path in %s. URL(s) attempted output above." % ( sys._getframe().f_code.co_name, self.name, Crate.config_file, ) ) + def unpack_source( self ): + unpack_dir = os.path.join( ScrambleEgg.build_dir, self.platform ) + if not os.path.exists( unpack_dir ): + os.makedirs( unpack_dir ) + self.buildpath = os.path.join( unpack_dir, self.name ) + if os.path.exists( self.buildpath ): + log.warning( "%s(): Removing old build directory at:" % sys._getframe().f_code.co_name ) + log.warning( " %s" % self.buildpath ) + shutil.rmtree( self.buildpath ) + if tarfile.is_tarfile( self.source_path ): + self.unpack_tar() + elif zipfile.is_zipfile( self.source_path ): + self.unpack_zip() + else: + raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, source_path ) ) + log.warning( "%s(): Unpacked to:" % sys._getframe().f_code.co_name ) + log.warning( " %s" % self.buildpath ) + def unpack_zip( self ): + unpack_path = os.path.dirname( self.buildpath ) + tld = self.get_tld( zipfile.ZipFile( self.source_path, 'r' ).namelist() ) + unpack_zipfile( self.source_path, unpack_path, ( 'ez_setup', ) ) + os.rename( os.path.join( unpack_path, tld ), self.buildpath ) + def unpack_tar( self ): + unpack_path = os.path.dirname( self.buildpath ) + t = tarfile.open( self.source_path, "r" ) + members = filter( lambda x: "ez_setup" not in x.name and "pax_global_header" != x.name, t.getmembers() ) + tld = self.get_tld( [ x.name for x in members ] ) + cur = os.getcwd() + os.chdir( unpack_path ) + for member in members: + t.extract( member ) + t.close() + os.rename( tld, self.name ) + os.chdir( cur ) + def copy_build_script( self ): + # will try: + # bx_python-py2.4-solaris-2.11-i86pc.py + # bx_python-py2.4-solaris.py + # bx_python-solaris-2.11-i86pc.py + # bx_python-solaris.py + # bx_python-py2.4.py + # bx_python.py + # generic.py + platform = self.platform.replace( '-ucs2', '' ).replace( '-ucs4', '' ) # ucs is unimportant here + build_scripts = ( + "%s-%s.py" % ( self.name, platform ), + "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[:2] ) ), + "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[1:] ) ), + "%s-%s.py" % ( self.name, self.py ), + "%s-%s.py" % ( self.name, platform.split( '-' )[0] ), + "%s.py" % self.name, + "generic.py" ) + for build_script in build_scripts: + build_script = os.path.join( ScrambleEgg.script_dir, build_script ) + if os.path.exists( build_script ): + log.warning( "%s(): Using build script %s" % ( sys._getframe().f_code.co_name, build_script ) ) + break + shutil.copyfile( build_script, os.path.join( self.buildpath, "scramble.py" ) ) + verfile = open( os.path.join( self.buildpath, ".galaxy_ver" ), "w" ) + verfile.write( self.version + '\n' ) + verfile.close() + if self.tag is not None: + tagfile = open( os.path.join( self.buildpath, ".galaxy_tag" ), "w" ) + tagfile.write( self.tag + '\n' ) + tagfile.close() + def run_scramble_script( self ): + log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name ) + # subprocessed to sterilize the env + cmd = "%s -ES %s" % ( self.python, "scramble.py" ) + log.debug( '%s(): Executing in %s:' % ( sys._getframe().f_code.co_name, self.buildpath ) ) + log.debug( ' %s' % cmd ) + p = subprocess.Popen( args = cmd, shell = True, cwd = self.buildpath ) + r = p.wait() + if r != 0: + if sys.platform == 'sunos5' and get_config_var('CC').endswith('pycc') and not os.environ.get( 'PYCC_CC', None ): + log.error( "%s(): Your python interpreter was compiled with Sun's pycc" % sys._getframe().f_code.co_name ) + log.error( " pseudo-compiler. You may need to set PYCC_CC and PYCC_CXX in your" ) + log.error( " environment if your compiler is in a non-standard location." ) + raise ScrambleFailure( self, "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) ) + +class ScrambleCrate( Crate ): + """ + Reads the eggs.ini file for use with scrambling eggs. + """ + def parse( self ): + Crate.parse( self ) + # get dependency sources + for egg in self.eggs.values(): + try: + egg.sources = self.config.get( "source", egg.name ).split() + except: + egg.sources = [] + def parse_egg_section( self, *args, **kwargs ): + kwargs['egg_class'] = ScrambleEgg + Crate.parse_egg_section( self, *args, **kwargs ) + def scramble( self, all=False ): + if all: + eggs = self.all_eggs + else: + eggs = self.config_eggs + eggs = filter( lambda x: x.name not in self.no_auto, eggs ) + failed = [] + for egg in eggs: + try: + egg.scramble() + except Exception, e: + failed.append( egg ) + last_exc = e + if failed: + if len( failed ) == 1: + raise last_exc # only 1 failure out of the crate, be more informative + else: + raise ScrambleFailure( failed ) diff -r a6c9862ece3d -r 0cb68935eede manage_db.sh --- a/manage_db.sh Mon Mar 01 13:30:37 2010 -0500 +++ b/manage_db.sh Mon Mar 01 15:41:30 2010 -0500 @@ -6,4 +6,4 @@ ####### cd `dirname $0` -`python ./scripts/check_python.py` ./scripts/manage_db.py $@ +python ./scripts/manage_db.py $@ diff -r a6c9862ece3d -r 0cb68935eede run.sh --- a/run.sh Mon Mar 01 13:30:37 2010 -0500 +++ b/run.sh Mon Mar 01 15:41:30 2010 -0500 @@ -2,18 +2,16 @@ cd `dirname $0` -CLEAN_PYTHON=`python ./scripts/check_python.py` - # explicitly attempt to fetch eggs before running FETCH_EGGS=1 for arg in "$@"; do [ "$arg" = "--stop-daemon" ] && FETCH_EGGS=0; break done if [ $FETCH_EGGS -eq 1 ]; then - $CLEAN_PYTHON ./scripts/check_eggs.py quiet + python ./scripts/check_eggs.py quiet if [ $? -ne 0 ]; then echo "Some eggs are out of date, attempting to fetch..." - $CLEAN_PYTHON ./scripts/fetch_eggs.py + python ./scripts/fetch_eggs.py if [ $? -eq 0 ]; then echo "Fetch successful." else @@ -22,4 +20,4 @@ fi fi fi -$CLEAN_PYTHON ./scripts/paster.py serve universe_wsgi.ini $@ +python ./scripts/paster.py serve universe_wsgi.ini $@ diff -r a6c9862ece3d -r 0cb68935eede run_functional_tests.sh --- a/run_functional_tests.sh Mon Mar 01 13:30:37 2010 -0500 +++ b/run_functional_tests.sh Mon Mar 01 15:41:30 2010 -0500 @@ -4,10 +4,8 @@ rm -f run_functional_tests.log -CLEAN_PYTHON=`python ./scripts/check_python.py` - if [ ! $1 ]; then - $CLEAN_PYTHON ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html --exclude="^get" functional + python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html --exclude="^get" functional elif [ $1 = 'help' ]; then echo "'run_functional_tests.sh' for testing all the tools in functional directory" echo "'run_functional_tests.sh aaa' for testing one test case of 'aaa' ('aaa' is the file name with path)" @@ -15,16 +13,16 @@ echo "'run_functional_tests.sh -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::')" echo "'run_functional_tests.sh -list' for listing all the tool ids" elif [ $1 = '-id' ]; then - $CLEAN_PYTHON ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$2 --with-nosehtml --html-report-file run_functional_tests.html + python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$2 --with-nosehtml --html-report-file run_functional_tests.html elif [ $1 = '-sid' ]; then - $CLEAN_PYTHON ./scripts/functional_tests.py --with-nosehtml --html-report-file run_functional_tests.html -v `python tool_list.py $2` + python ./scripts/functional_tests.py --with-nosehtml --html-report-file run_functional_tests.html -v `python tool_list.py $2` elif [ $1 = '-list' ]; then python tool_list.py echo "===========================================================================================================================================" echo "'run_functional_tests.sh -id bbb' for testing one tool with id 'bbb' ('bbb' is the tool id)" echo "'run_functional_tests.sh -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::')" else - $CLEAN_PYTHON ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html $1 + python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html $1 fi echo "'run_functional_tests.sh help' for help" diff -r a6c9862ece3d -r 0cb68935eede run_reports.sh --- a/run_reports.sh Mon Mar 01 13:30:37 2010 -0500 +++ b/run_reports.sh Mon Mar 01 15:41:30 2010 -0500 @@ -1,5 +1,4 @@ #!/bin/sh cd `dirname $0` - -`python ./scripts/check_python.py` ./scripts/paster.py serve reports_wsgi.ini --pid-file=reports_webapp.pid --log-file=reports_webapp.log $@ +python ./scripts/paster.py serve reports_wsgi.ini --pid-file=reports_webapp.pid --log-file=reports_webapp.log $@ diff -r a6c9862ece3d -r 0cb68935eede run_unit_tests.sh --- a/run_unit_tests.sh Mon Mar 01 13:30:37 2010 -0500 +++ b/run_unit_tests.sh Mon Mar 01 15:41:30 2010 -0500 @@ -2,6 +2,6 @@ ## Excluding controllers due to the problematic genetrack dependency -`python ./scripts/check_python.py` ./scripts/nosetests.py -v -w lib \ +python ./scripts/nosetests.py -v -w lib \ --with-nosehtml --html-report-file run_unit_tests.html \ --with-doctest --exclude=functional --exclude="^get" --exclude=controllers diff -r a6c9862ece3d -r 0cb68935eede scripts/check_eggs.py --- a/scripts/check_eggs.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/check_eggs.py Mon Mar 01 15:41:30 2010 -0500 @@ -17,16 +17,10 @@ except: quiet = False -from galaxy.eggs import * +from galaxy.eggs import Crate c = Crate() -c.parse() -galaxy_config = GalaxyConfig() -ignore = [] -for name in c.get_names(): - if not galaxy_config.check_conditional( name ): - ignore.append( name ) -if not c.find( ignore=ignore ): +if c.config_missing: if not quiet: print "Some of your Galaxy eggs are out of date. Please update them" print "by running:" diff -r a6c9862ece3d -r 0cb68935eede scripts/check_python.py --- a/scripts/check_python.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/check_python.py Mon Mar 01 15:41:30 2010 -0500 @@ -6,13 +6,6 @@ supported version is installed but is not your default, getgalaxy.org contains instructions on how to force Galaxy to use a different version.""" % sys.version[:3] -def check_virtualenv(): - try: - assert sys.real_prefix - return 'python -E' - except: - return 'python -ES' - def check_python(): try: assert sys.version_info[:2] >= ( 2, 4 ) and sys.version_info[:2] <= ( 2, 6 ) @@ -21,9 +14,9 @@ raise if __name__ == '__main__': + rval = 0 try: check_python() - print check_virtualenv() - sys.exit( 0 ) except StandardError: - sys.exit( 1 ) + rval = 1 + sys.exit( rval ) diff -r a6c9862ece3d -r 0cb68935eede scripts/dist-scramble.py --- a/scripts/dist-scramble.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/dist-scramble.py Mon Mar 01 15:41:30 2010 -0500 @@ -11,28 +11,30 @@ root.setLevel( 10 ) root.addHandler( logging.StreamHandler( sys.stdout ) ) -lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) +lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) ) sys.path.append( lib ) -from galaxy.eggs import DistCrate +from galaxy.eggs.dist import DistScrambleCrate, ScrambleFailure if len( sys.argv ) > 3 or len( sys.argv ) < 2: print __doc__ sys.exit( 1 ) elif len( sys.argv ) == 3: - c = DistCrate( sys.argv[2] ) + c = DistScrambleCrate( sys.argv[2] ) else: - c = DistCrate() + c = DistScrambleCrate() -c.parse() -egg_list = c.get( sys.argv[1] ) -if egg_list is None: +try: + eggs = c[sys.argv[1]] +except: print "error: %s not in eggs.ini" % sys.argv[1] sys.exit( 1 ) failed = [] -for egg in egg_list: - if not egg.scramble( dist=True ): - failed.append( egg.platform['galaxy'] ) +for egg in eggs: + try: + egg.scramble() + except ScrambleFailure: + failed.append( egg.platform ) if len( failed ): print "" print "Scramble failed to build eggs on the following platforms (more details" diff -r a6c9862ece3d -r 0cb68935eede scripts/fetch_eggs.py --- a/scripts/fetch_eggs.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/fetch_eggs.py Mon Mar 01 15:41:30 2010 -0500 @@ -18,39 +18,43 @@ lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) sys.path.append( lib ) -from galaxy.eggs import * +from galaxy.eggs import Crate, EggNotFetchable +import pkg_resources c = Crate() -if len( sys.argv ) == 3: - c.platform = { 'peak' : sys.argv[2].rsplit('-',1)[0], 'galaxy' : sys.argv[2] } -c.parse() try: - galaxy_config = GalaxyConfig() - names = [] + c.platform = sys.argv[2] +except: + pass +try: if len( sys.argv ) == 1: - names = c.get_names() + c.resolve() # Only fetch eggs required by the config elif sys.argv[1] == 'all': - names = galaxy_config.always_conditional + c.resolve( all=True ) # Fetch everything else: # Fetch a specific egg - egg = c.get( sys.argv[1] ) - if egg is None: - print "error: %s not in eggs.ini" % sys.argv[1] + name = sys.argv[1] + try: + egg = c[name] + except: + print "error: %s not in eggs.ini" % name sys.exit( 1 ) - egg.fetch() - sys.exit( 0 ) - ignore = filter( lambda x: not galaxy_config.check_conditional( x ), list( names ) ) - c.fetch( ignore ) + dist = egg.resolve()[0] + print "%s %s is installed at %s" % ( dist.project_name, dist.version, dist.location ) except EggNotFetchable, e: - print "One or more of the python eggs necessary to run Galaxy couldn't be" - print "downloaded automatically. You may want to try building them by" - print "hand with:" - for egg in e.eggs: - print " python scripts/scramble.py %s" % egg + try: + assert sys.argv[1] != 'all' + egg = e.eggs[0] + print "%s %s couldn't be downloaded automatically. You can try" % ( egg.name, egg.version ) + print "building it by hand with:" + print " python scripts/scramble.py %s" + except ( AssertionError, IndexError ): + print "One or more of the python eggs necessary to run Galaxy couldn't be" + print "downloaded automatically. You can try building them by hand (all" + print "at once) with:" + print " python scripts/scramble.py" + print "Or individually:" + for egg in e.eggs: + print " python scripts/scramble.py %s" % egg.name sys.exit( 1 ) -except PlatformNotSupported, e: - print "Your platform (%s) is not supported." % e - print "Pre-built galaxy eggs are not available from the Galaxy developers for" - print "your platform. You may be able to build them by hand with:" - print " python scripts/scramble.py" - sys.exit( 1 ) +sys.exit( 0 ) diff -r a6c9862ece3d -r 0cb68935eede scripts/get_platforms.py --- a/scripts/get_platforms.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/get_platforms.py Mon Mar 01 15:41:30 2010 -0500 @@ -7,6 +7,6 @@ lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) sys.path.append( lib ) -from galaxy.eggs import get_platform -print get_platform() -print get_platform( True ) +import galaxy +import pkg_resources +print pkg_resources.get_platform() diff -r a6c9862ece3d -r 0cb68935eede scripts/paster.py --- a/scripts/paster.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/paster.py Mon Mar 01 15:41:30 2010 -0500 @@ -28,6 +28,7 @@ pkg_resources.require( "Paste" ) pkg_resources.require( "PasteScript" ) +pkg_resources.require( "PasteDeploy" ) from paste.script import command command.run() diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble.py --- a/scripts/scramble.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/scramble.py Mon Mar 01 15:41:30 2010 -0500 @@ -14,23 +14,29 @@ lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) sys.path.append( lib ) -from galaxy.eggs import Crate, GalaxyConfig +from galaxy.eggs.scramble import ScrambleCrate, ScrambleFailure -c = Crate() -c.parse() -galaxy_config = GalaxyConfig() -names = [] -if len( sys.argv ) == 1: - names = c.get_names() -elif sys.argv[1] == 'all': - names = galaxy_config.always_conditional -else: -# Scramble a specific egg - egg = c.get( sys.argv[1] ) - if egg is None: - print "error: %s not in eggs.ini" % sys.argv[1] - sys.exit( 1 ) - egg.scramble() - sys.exit( 0 ) -ignore = filter( lambda x: not galaxy_config.check_conditional( x ), list( names ) ) -c.scramble( ignore=ignore ) +c = ScrambleCrate() + +try: + if len( sys.argv ) == 1: + eggs = c.scramble() + elif sys.argv[1] == 'all': + c.scramble( all=True ) + else: + # Scramble a specific egg + name = sys.argv[1] + try: + egg = c[name] + except: + print "error: %s not in eggs.ini" % name + sys.exit( 1 ) + egg.scramble() + sys.exit( 0 ) +except ScrambleFailure, e: + if len( e.eggs ) == 1: + raise + else: + print 'Scrambling the following eggs failed:\n ', + print '\n '.join( [ egg.name for egg in e.eggs ] ) + sys.exit( 1 ) diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/lib/get_platform.py --- a/scripts/scramble/lib/get_platform.py Mon Mar 01 13:30:37 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,18 +0,0 @@ -""" -Monkeypatch get_platform since it's broken on OS X versions of Python 2.5 -""" -import os, sys -from distutils.sysconfig import get_config_vars -if sys.platform == 'darwin' and get_config_vars().get('UNIVERSALSDK', '').strip(): - # Has to be before anything imports pkg_resources - def _get_platform_monkeypatch(): - plat = distutils.util._get_platform() - if plat.startswith( 'macosx-' ): - plat = 'macosx-10.3-fat' - return plat - import distutils.util - try: - assert distutils.util._get_platform - except: - distutils.util._get_platform = distutils.util.get_platform - distutils.util.get_platform = _get_platform_monkeypatch diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/lib/scramble_lib.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/scramble/lib/scramble_lib.py Mon Mar 01 15:41:30 2010 -0500 @@ -0,0 +1,164 @@ +""" +Various utilities for scrambling. +""" +import os, sys, errno, re, distutils.util, glob, shutil, subprocess, tarfile, zipfile +from distutils.sysconfig import get_config_var, get_config_vars + +try: + import zlib +except: + raise Exception( 'Cannot import zlib, which must exist to build eggs. If your python interpreter is truly missing it, you will need to recompile or (on supported platforms) download a binary version from python.org.' ) + +def get_tag(): + try: + return open( '.galaxy_tag', 'r' ).read().strip() + except: + return None + +def get_ver(): + try: + return open( '.galaxy_ver', 'r' ).read().strip() + except: + return None + +def clean( extra_dirs=[] ): + for dir in [ 'build', 'dist' ] + extra_dirs: + try: + shutil.rmtree( dir ) + except OSError, e: + if e.errno != errno.ENOENT: + raise + +def apply_patches(): + name = os.path.basename( os.getcwd() ) + for file in glob.glob( os.path.join( patches, name, '*' ) ): + shutil.copy( file, os.path.basename( file ) ) + +def get_archive( base ): + for arctype in ( 'tar.gz', 'tgz', 'zip' ): + archive = '.'.join( ( base, arctype ) ) + if os.path.exists( archive ): + return archive + else: + raise Exception( "%s(): Couldn't find a suitable archive for %s in %s" % ( sys._getframe().f_code.co_name, os.path.basename( base ), archives ) ) + +def compress( path, *files ): + tarcf( path, *files ) + +def uncompress( path ): + if path.endswith( '.zip' ): + unzip( path ) + else: + tarxf( path ) + +def unzip( zipf ): + z = zipfile.ZipFile( zipf ) + try: + for info in z.infolist(): + name = info.filename + mode = (info.external_attr >> 16L) & 0777 + # don't extract absolute paths or ones with .. in them + if name.startswith('/') or '..' in name: + continue + if not name: + continue + if name.endswith('/'): + # directory + pkg_resources.ensure_directory(name) + else: + # file + pkg_resources.ensure_directory(name) + data = z.read(info.filename) + f = open(name,'wb') + try: + f.write(data) + finally: + f.close() + del data + try: + if not os.path.islink( name ): + os.chmod(name,mode) + except: + pass + finally: + z.close() + +def tarxf( ball ): + t = tarfile.open( ball, 'r' ) + for fn in t.getnames(): + t.extract( fn ) + t.close() + +def tarcf( ball, *files ): + if ball.endswith( '.gz' ): + t = tarfile.open( ball, 'w:gz' ) + elif ball.endswith( '.bz2' ): + t = tarfile.open( ball, 'w:bz2' ) + else: + t = tarfile.open( ball, 'w' ) + for file in files: + t.add( file ) + t.close() + +def run( cmd, d, txt ): + p = subprocess.Popen( args = cmd, shell = True, cwd = d ) + r = p.wait() + if r != 0: + print '%s(): %s failed' % ( sys._getframe().f_code.co_name, txt ) + sys.exit( 1 ) + +def unpack_dep( source, prepped, builder, args={} ): + if prepped is not None and os.path.exists( prepped ): + print "%s(): Prepared dependency already exists at the following path, remove to force re-prep:" % sys._getframe().f_code.co_name + print " ", prepped + uncompress( prepped ) + else: + print "%s(): Prepared dependency does not exist, preparing now from source:" % sys._getframe().f_code.co_name + print " ", source + uncompress( source ) + builder( prepped, args ) + +def get_solaris_compiler(): + p = subprocess.Popen( '%s -V 2>&1' % get_config_var('CC'), shell = True, stdout = subprocess.PIPE ) + out = p.stdout.read() + p.wait() + if 'Sun C' in out: + return 'cc' + else: + return 'gcc' + +# Monkeypatch pkg_resources for better ABI recognition +def _get_platform(): + plat = distutils.util._get_platform() + if sys.version_info[:2] == ( 2, 5 ) and \ + ( ( os.uname()[-1] in ( 'i386', 'ppc' ) and sys.platform == 'darwin' and os.path.abspath( sys.prefix ).startswith( '/System' ) ) or \ + ( sys.platform == 'darwin' and get_config_vars().get('UNIVERSALSDK', '').strip() ) ): + plat = 'macosx-10.3-fat' + if sys.platform == "sunos5" and not (plat.endswith('_32') or plat.endswith('_64')): + if sys.maxint > 2**31: + plat += '_64' + else: + plat += '_32' + if not (plat.endswith('-ucs2') or plat.endswith('-ucs4')): + if sys.maxunicode > 2**16: + plat += '-ucs4' + else: + plat += '-ucs2' + return plat +try: + assert distutil.util._get_platform +except: + distutils.util._get_platform = distutils.util.get_platform + distutils.util.get_platform = _get_platform + +# get setuptools +from ez_setup import use_setuptools +use_setuptools( download_delay=8, to_dir=os.path.dirname( __file__ ) ) +from setuptools import * +import pkg_resources + +# some constants +root = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..' ) ) +archives = os.path.abspath( os.path.join( root, 'archives' ) ) +patches = os.path.abspath( os.path.join( root, 'patches' ) ) +platform_noucs = pkg_resources.get_platform().rsplit( '-', 1 )[0] diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/patches/Cheetah/SetupConfig.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/scramble/patches/Cheetah/SetupConfig.py Mon Mar 01 15:41:30 2010 -0500 @@ -0,0 +1,103 @@ +#-------Main Package Settings-----------# +import sys + +name = 'Cheetah' +from cheetah.Version import Version as version +maintainer = "R. Tyler Ballance" +author = "Tavis Rudd" +author_email = "cheetahtemplate-discuss@lists.sf.net" +url = "http://www.cheetahtemplate.org/" +packages = ['Cheetah', + 'Cheetah.Macros', + 'Cheetah.Templates', + 'Cheetah.Tests', + 'Cheetah.Tools', + 'Cheetah.Utils', + ] +classifiers = [line.strip() for line in '''\ + #Development Status :: 4 - Beta + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Intended Audience :: System Administrators + License :: OSI Approved :: MIT License + Operating System :: OS Independent + Programming Language :: Python + Topic :: Internet :: WWW/HTTP + Topic :: Internet :: WWW/HTTP :: Dynamic Content + Topic :: Internet :: WWW/HTTP :: Site Management + Topic :: Software Development :: Code Generators + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Software Development :: User Interfaces + Topic :: Text Processing'''.splitlines() if not line.strip().startswith('#')] +del line + +package_dir = {'Cheetah':'cheetah'} + +import os +import os.path +from distutils.core import Extension + +ext_modules=[ + Extension("Cheetah._namemapper", + [os.path.join('cheetah', 'c', '_namemapper.c')]), + # Extension("Cheetah._verifytype", + # [os.path.join('cheetah', 'c', '_verifytype.c')]), + # Extension("Cheetah._filters", + # [os.path.join('cheetah', 'c', '_filters.c')]), + # Extension('Cheetah._template', + # [os.path.join('cheetah', 'c', '_template.c')]), + ] + +## Data Files and Scripts +scripts = ['bin/cheetah-compile', + 'bin/cheetah', + ] + +data_files = ['recursive: cheetah *.tmpl *.txt LICENSE README TODO CHANGES',] + +if not os.getenv('CHEETAH_INSTALL_WITHOUT_SETUPTOOLS'): + try: + from setuptools import setup +# install_requires = [ +# "Markdown >= 2.0.1", +# ] + if sys.platform == 'win32': + # use 'entry_points' instead of 'scripts' + del scripts + entry_points = { + 'console_scripts': [ + 'cheetah = Cheetah.CheetahWrapper:_cheetah', + 'cheetah-compile = Cheetah.CheetahWrapper:_cheetah_compile', + ] + } + except ImportError: + print 'Not using setuptools, so we cannot install the Markdown dependency' + + +description = "Cheetah is a template engine and code generation tool." + +long_description = '''Cheetah is an open source template engine and code generation tool. + +It can be used standalone or combined with other tools and frameworks. Web +development is its principle use, but Cheetah is very flexible and is also being +used to generate C++ game code, Java, sql, form emails and even Python code. + +Documentation +================================================================================ +For a high-level introduction to Cheetah please refer to the User\'s Guide +at http://www.cheetahtemplate.org/learn.html + +Mailing list +================================================================================ +cheetahtemplate-discuss@lists.sourceforge.net +Subscribe at http://lists.sourceforge.net/lists/listinfo/cheetahtemplate-discuss + +Credits +================================================================================ +http://www.cheetahtemplate.org/credits.html + +Recent Changes +================================================================================ +See http://www.cheetahtemplate.org/CHANGES.txt for full details + +''' diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/patches/GeneTrack/setup.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/scramble/patches/GeneTrack/setup.py Mon Mar 01 15:41:30 2010 -0500 @@ -0,0 +1,15 @@ +from os import walk +from os.path import join +from setuptools import setup, find_packages + +def walk_files( top ): + for dir, dirs, files in walk( top ): + yield( dir, [ join( dir, f ) for f in files ] ) + +setup( + name = "GeneTrack", + version = "2.0.0-beta-1", + packages = ['genetrack','genetrack.scripts'], + data_files = [ f for f in walk_files('tests') ], + zip_safe = False +) diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/patches/MySQL_python/site.cfg --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/scramble/patches/MySQL_python/site.cfg Mon Mar 01 15:41:30 2010 -0500 @@ -0,0 +1,18 @@ +[options] +# embedded: link against the embedded server library +# threadsafe: use the threadsafe client +# static: link against a static library (probably required for embedded) + +embedded = False +threadsafe = True +static = True + +# The path to mysql_config. +# Only use this if mysql_config is not on your PATH, or you have some weird +# setup that requires it. +mysql_config = mysql/bin/mysql_config + +# The Windows registry key for MySQL. +# This has to be set for Windows builds to work. +# Only change this if you have a different version. +registry_key = SOFTWARE\MySQL AB\MySQL Server 5.0 diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/patches/PasteScript/setup.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/scramble/patches/PasteScript/setup.py Mon Mar 01 15:41:30 2010 -0500 @@ -0,0 +1,145 @@ +import ez_setup +ez_setup.use_setuptools() +from setuptools import setup, find_packages +import re, os + +version = '1.7.3' + +news = os.path.join(os.path.dirname(__file__), 'docs', 'news.txt') +news = open(news).read() +parts = re.split(r'([0-9\.]+)\s*\n\r?-+\n\r?', news) +found_news = '' +for i in range(len(parts)-1): + if parts[i] == version: + found_news = parts[i+i] + break +if not found_news: + print 'Warning: no news for this version found' + +long_description="""\ +This is a pluggable command-line tool. + +It includes some built-in features; + +* Create file layouts for packages. For instance, ``paste create + --template=basic_package MyPackage`` will create a `setuptools + <http://peak.telecommunity.com/DevCenter/setuptools>`_-ready + file layout. + +* Serving up web applications, with configuration based on + `paste.deploy <http://pythonpaste.org/deploy/paste-deploy.html>`_. + +The latest version is available in a `Subversion repository +<http://svn.pythonpaste.org/Paste/Script/trunk#egg=PasteScript-dev>`_. + +For the latest changes see the `news file +<http://pythonpaste.org/script/news.html>`_. +""" + +if found_news: + title = 'Changes in %s' % version + long_description += "\n%s\n%s\n" % (title, '-'*len(title)) + long_description += found_news + +setup( + name="PasteScript", + version=version, + description="A pluggable command-line frontend, including commands to setup package file layouts", + long_description=long_description, + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "Topic :: Software Development :: Libraries :: Python Modules", + "Framework :: Paste", + ], + keywords='web wsgi setuptools framework command-line setup', + author="Ian Bicking", + author_email="ianb@colorstudy.com", + url="http://pythonpaste.org/script/", + namespace_packages=['paste'], + license='MIT', + packages=find_packages(exclude='tests'), + package_data={ + 'paste.script': ['paster-templates/basic_package/setup.*', + 'paster-templates/basic_package/tests/*.py', + # @@: docs/ doesn't have any files :( + 'paster-templates/basic_package/+package+/*.py'], + }, + zip_safe=False, + scripts=['scripts/paster'], + extras_require={ + 'Templating': [], + 'Cheetah': ['Cheetah'], + 'Config': ['PasteDeploy'], + 'WSGIUtils': ['WSGIUtils'], + 'Flup': ['Flup'], + # the Paste feature means the complete set of features; + # (other features are truly optional) + 'Paste': ['PasteDeploy', 'Cheetah'], + }, + entry_points=""" + [paste.global_paster_command] + help=paste.script.help:HelpCommand + create=paste.script.create_distro:CreateDistroCommand [Templating] + serve=paste.script.serve:ServeCommand [Config] + request=paste.script.request:RequestCommand [Config] + post=paste.script.request:RequestCommand [Config] + exe=paste.script.exe:ExeCommand + points=paste.script.entrypoints:EntryPointCommand + make-config=paste.script.appinstall:MakeConfigCommand + setup-app=paste.script.appinstall:SetupCommand + + [paste.paster_command] + grep = paste.script.grep:GrepCommand + + [paste.paster_create_template] + basic_package=paste.script.templates:BasicPackage + + [paste.server_runner] + wsgiutils=paste.script.wsgiutils_server:run_server [WSGIUtils] + flup_ajp_thread=paste.script.flup_server:run_ajp_thread [Flup] + flup_ajp_fork=paste.script.flup_server:run_ajp_fork [Flup] + flup_fcgi_thread=paste.script.flup_server:run_fcgi_thread [Flup] + flup_fcgi_fork=paste.script.flup_server:run_fcgi_fork [Flup] + flup_scgi_thread=paste.script.flup_server:run_scgi_thread [Flup] + flup_scgi_fork=paste.script.flup_server:run_scgi_fork [Flup] + cgi=paste.script.cgi_server:paste_run_cgi + cherrypy=paste.script.cherrypy_server:cpwsgi_server + twisted=paste.script.twisted_web2_server:run_twisted + + [paste.app_factory] + test=paste.script.testapp:make_test_application + + [paste.entry_point_description] + paste.entry_point_description = paste.script.epdesc:MetaEntryPointDescription + paste.paster_create_template = paste.script.epdesc:CreateTemplateDescription + paste.paster_command = paste.script.epdesc:PasterCommandDescription + paste.global_paster_command = paste.script.epdesc:GlobalPasterCommandDescription + paste.app_install = paste.script.epdesc:AppInstallDescription + + # These aren't part of Paste Script particularly, but + # we'll document them here + console_scripts = paste.script.epdesc:ConsoleScriptsDescription + # @@: Need non-console scripts... + distutils.commands = paste.script.epdesc:DistutilsCommandsDescription + distutils.setup_keywords = paste.script.epdesc:SetupKeywordsDescription + egg_info.writers = paste.script.epdesc:EggInfoWriters + # @@: Not sure what this does: + #setuptools.file_finders = paste.script.epdesc:SetuptoolsFileFinders + + [console_scripts] + paster=paste.script.command:run + + [distutils.setup_keywords] + paster_plugins = setuptools.dist:assert_string_list + + [egg_info.writers] + paster_plugins.txt = setuptools.command.egg_info:write_arg + """, + install_requires=[ + ], + ) diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/patches/psycopg2/setup.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/scramble/patches/psycopg2/setup.py Mon Mar 01 15:41:30 2010 -0500 @@ -0,0 +1,252 @@ +# setup.py - distutils packaging +# +# Copyright (C) 2003-2004 Federico Di Gregorio <fog@debian.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTIBILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +"""Python-PostgreSQL Database Adapter + +psycopg is a PostgreSQL database adapter for the Python programming +language. This is version 2, a complete rewrite of the original code to +provide new-style classes for connection and cursor objects and other sweet +candies. Like the original, psycopg 2 was written with the aim of being +very small and fast, and stable as a rock. + +psycopg is different from the other database adapter because it was +designed for heavily multi-threaded applications that create and destroy +lots of cursors and make a conspicuous number of concurrent INSERTs or +UPDATEs. psycopg 2 also provide full asycronous operations for the really +brave programmer. +""" + +classifiers = """\ +Development Status :: 4 - Beta +Intended Audience :: Developers +License :: OSI Approved :: GNU General Public License (GPL) +License :: OSI Approved :: Zope Public License +Programming Language :: Python +Programming Language :: C +Programming Language :: SQL +Topic :: Database +Topic :: Database :: Front-Ends +Topic :: Software Development +Topic :: Software Development :: Libraries :: Python Modules +Operating System :: Microsoft :: Windows +Operating System :: Unix +""" + +import os +import os.path +import sys +import popen2 +import ConfigParser +from glob import glob +from distutils.core import setup, Extension +from distutils.errors import DistutilsFileError +from distutils.command.build_ext import build_ext +from distutils.sysconfig import get_python_inc +from distutils.ccompiler import get_default_compiler + +assert sys.version_info[:2] >= ( 2, 4 ) + +PSYCOPG_VERSION = '2.0.6' +version_flags = [] + +PG_VERSION = '8.2.6' + +class psycopg_build_ext(build_ext): + """Conditionally complement the setup.cfg options file. + + This class configures the include_dirs, libray_dirs, libraries + options as required by the system. Most of the configuration happens + in finalize_options() method. + + If you want to set up the build step for a peculiar platform, add a + method finalize_PLAT(), where PLAT matches your sys.platform. + """ + user_options = build_ext.user_options[:] + user_options.extend([ + ('use-pydatetime', None, + "Use Python datatime objects for date and time representation."), + ('use-decimal', None, + "Use Decimal type even on Python 2.3 if the module is provided."), + ]) + + boolean_options = build_ext.boolean_options[:] + boolean_options.extend(('use-pydatetime', 'use-decimal', 'have-ssl')) + + def initialize_options(self): + build_ext.initialize_options(self) + self.use_pg_dll = 1 + self.pgdir = None + self.mx_include_dir = None + + def get_compiler(self): + """Return the name of the C compiler used to compile extensions. + + If a compiler was not explicitely set (on the command line, for + example), fall back on the default compiler. + """ + if self.compiler: + # distutils doesn't keep the type of self.compiler uniform; we + # compensate: + if isinstance(self.compiler, str): + name = self.compiler + else: + name = self.compiler.compiler_type + else: + name = get_default_compiler() + return name + + def finalize_darwin(self): + """Finalize build system configuration on darwin platform.""" + self.libraries.append('ssl') + self.libraries.append('crypto') + + def finalize_options(self): + """Complete the build system configuation.""" + build_ext.finalize_options(self) + + self.include_dirs.append(".") + self.include_dirs.append("postgresql-8.2.6/src/interfaces/libpq") + self.include_dirs.append("postgresql-8.2.6/src/include") + self.include_dirs.append("postgresql-8.2.6/src/port") + pgmajor, pgminor, pgpatch = PG_VERSION.split('.') + define_macros.append(("PG_MAJOR_VERSION", pgmajor)) + define_macros.append(("PG_MINOR_VERSION", pgminor)) + define_macros.append(("PG_PATCH_VERSION", pgpatch)) + + if hasattr(self, "finalize_" + sys.platform): + getattr(self, "finalize_" + sys.platform)() + +# let's start with macro definitions (the ones not already in setup.cfg) +define_macros = [] +include_dirs = [] + +# python version +define_macros.append(('PY_MAJOR_VERSION', str(sys.version_info[0]))) +define_macros.append(('PY_MINOR_VERSION', str(sys.version_info[1]))) + +# some macros related to python versions and features +if sys.version_info[0] >= 2 and sys.version_info[1] >= 3: + define_macros.append(('HAVE_PYBOOL','1')) + +# gather information to build the extension module +ext = [] ; data_files = [] + +# sources + +sources = [ + 'psycopgmodule.c', 'pqpath.c', 'typecast.c', + 'microprotocols.c', 'microprotocols_proto.c', + 'connection_type.c', 'connection_int.c', 'cursor_type.c', 'cursor_int.c', + 'adapter_qstring.c', 'adapter_pboolean.c', 'adapter_binary.c', + 'adapter_asis.c', 'adapter_list.c'] + +parser = ConfigParser.ConfigParser() +parser.read('setup.cfg') + +# Choose if to use Decimal type +use_decimal = int(parser.get('build_ext', 'use_decimal')) +if sys.version_info[0] >= 2 and ( + sys.version_info[1] >= 4 or (sys.version_info[1] == 3 and use_decimal)): + define_macros.append(('HAVE_DECIMAL','1')) + version_flags.append('dec') + +# Choose a datetime module +have_pydatetime = False +have_mxdatetime = False +use_pydatetime = int(parser.get('build_ext', 'use_pydatetime')) + +# check for mx package +if parser.has_option('build_ext', 'mx_include_dir'): + mxincludedir = parser.get('build_ext', 'mx_include_dir') +else: + mxincludedir = os.path.join(get_python_inc(plat_specific=1), "mx") +if os.path.exists(mxincludedir): + include_dirs.append(mxincludedir) + define_macros.append(('HAVE_MXDATETIME','1')) + sources.append('adapter_mxdatetime.c') + have_mxdatetime = True + version_flags.append('mx') + +# check for python datetime package +if os.path.exists(os.path.join(get_python_inc(plat_specific=1),"datetime.h")): + define_macros.append(('HAVE_PYDATETIME','1')) + sources.append('adapter_datetime.c') + have_pydatetime = True + version_flags.append('dt') + +# now decide which package will be the default for date/time typecasts +if have_pydatetime and use_pydatetime \ + or have_pydatetime and not have_mxdatetime: + define_macros.append(('PSYCOPG_DEFAULT_PYDATETIME','1')) +elif have_mxdatetime: + define_macros.append(('PSYCOPG_DEFAULT_MXDATETIME','1')) +else: + def e(msg): + sys.stderr.write("error: " + msg + "\n") + e("psycopg requires a datetime module:") + e(" mx.DateTime module not found") + e(" python datetime module not found") + e("Note that psycopg needs the module headers and not just the module") + e("itself. If you installed Python or mx.DateTime from a binary package") + e("you probably need to install its companion -dev or -devel package.") + sys.exit(1) + +# generate a nice version string to avoid confusion when users report bugs +for have in parser.get('build_ext', 'define').split(','): + if have == 'PSYCOPG_EXTENSIONS': + version_flags.append('ext') + elif have == 'HAVE_PQPROTOCOL3': + version_flags.append('pq3') +if version_flags: + PSYCOPG_VERSION_EX = PSYCOPG_VERSION + " (%s)" % ' '.join(version_flags) +else: + PSYCOPG_VERSION_EX = PSYCOPG_VERSION + +define_macros.append(('PSYCOPG_VERSION', '"'+PSYCOPG_VERSION_EX+'"')) + +if parser.has_option('build_ext', 'have_ssl'): + have_ssl = int(parser.get('build_ext', 'have_ssl')) +else: + have_ssl = 0 + +# build the extension + +sources = map(lambda x: os.path.join('psycopg', x), sources) +sources.append( 'postgresql-%s/src/port/pgstrcasecmp.c' % PG_VERSION ) +sources.append( 'postgresql-%s/src/backend/libpq/md5.c' % PG_VERSION ) +sources.extend( glob('postgresql-%s/src/interfaces/libpq/*.c' % PG_VERSION) ) + +ext.append(Extension("psycopg2._psycopg", sources, + define_macros=define_macros, + include_dirs=include_dirs, + undef_macros=[])) +setup(name="psycopg2", + version=PSYCOPG_VERSION, + maintainer="Federico Di Gregorio", + maintainer_email="fog@initd.org", + author="Federico Di Gregorio", + author_email="fog@initd.org", + url="http://initd.org/tracker/psycopg", + download_url = "http://initd.org/pub/software/psycopg2", + license="GPL with exceptions or ZPL", + platforms = ["any"], + description=__doc__.split("\n")[0], + long_description="\n".join(__doc__.split("\n")[2:]), + classifiers=filter(None, classifiers.split("\n")), + data_files=data_files, + package_dir={'psycopg2':'lib'}, + packages=['psycopg2'], + cmdclass={ 'build_ext': psycopg_build_ext }, + ext_modules=ext) + diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/scripts/Cheetah.py --- a/scripts/scramble/scripts/Cheetah.py Mon Mar 01 13:30:37 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,62 +0,0 @@ -import os, sys, shutil - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -scramble_lib = os.path.join( "..", "..", "..", "lib" ) -sys.path.append( scramble_lib ) -import get_platform # fixes fat python 2.5 -from ez_setup import use_setuptools -use_setuptools( download_delay=8, to_dir=scramble_lib ) -from setuptools import * - -# get the tag -if os.access( ".galaxy_tag", os.F_OK ): - tagfile = open( ".galaxy_tag", "r" ) - tag = tagfile.readline().strip() -else: - tag = None - -# in case you're running this by hand from a dirty module source dir -for dir in [ "build", "dist" ]: - if os.access( dir, os.F_OK ): - print "scramble_it.py: removing dir:", dir - shutil.rmtree( dir ) - -# patch -file = "SetupConfig.py" -if not os.access( "%s.orig" %file, os.F_OK ): - print "scramble.py(): Patching", file - shutil.copyfile( file, "%s.orig" %file ) - i = open( "%s.orig" %file, "r" ) - o = open( file, "w" ) - comment = False - for line in i.readlines(): - if line == " install_requires = [\n": - comment = True - print >>o, "#" + line, - elif comment and line == " ]\n": - comment = False - print >>o, "#" + line, - elif comment: - print >>o, "#" + line, - else: - print >>o, line, - i.close() - o.close() - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "egg_info" ) -if tag is not None: - #sys.argv.append( "egg_info" ) - sys.argv.append( "--tag-build=%s" %tag ) -# svn revision (if any) is handled directly in tag-build -sys.argv.append( "--no-svn-revision" ) -sys.argv.append( "bdist_egg" ) - -# do it -execfile( "setup.py", globals(), locals() ) diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/scripts/DRMAA_python-macosx.py --- a/scripts/scramble/scripts/DRMAA_python-macosx.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/scramble/scripts/DRMAA_python-macosx.py Mon Mar 01 15:41:30 2010 -0500 @@ -1,8 +1,8 @@ import os, sys, shutil if "SGE_ROOT" not in os.environ: - print "scramble(): Please set SGE_ROOT to the path of your SGE installation" - print "scramble(): before scrambling DRMAA_python" + print "main(): Please set SGE_ROOT to the path of your SGE installation" + print "main(): before scrambling DRMAA_python" sys.exit(1) # change back to the build dir @@ -10,27 +10,15 @@ os.chdir( os.path.dirname( sys.argv[0] ) ) # find setuptools -scramble_lib = os.path.join( "..", "..", "..", "lib" ) -sys.path.append( scramble_lib ) -import get_platform # fixes fat python 2.5 -try: - from setuptools import * - import pkg_resources -except: - from ez_setup import use_setuptools - use_setuptools( download_delay=8, to_dir=scramble_lib ) - from setuptools import * - import pkg_resources +sys.path.append( os.path.join( '..', '..', '..', 'lib' ) ) +from scramble_lib import * -# clean, in case you're running this by hand from a dirty module source dir -for dir in [ "build", "dist", "gridengine" ]: - if os.access( dir, os.F_OK ): - print "scramble_it.py: removing dir:", dir - shutil.rmtree( dir ) +tag = get_tag() # get the tag +clean() # clean up any existing stuff (could happen if you run scramble.py by hand) # patch file = "setup.py" -print "scramble(): Patching", file +print "main(): Patching", file if not os.access( "%s.orig" %file, os.F_OK ): shutil.copyfile( file, "%s.orig" %file ) i = open( "%s.orig" %file, "r" ) @@ -44,7 +32,7 @@ i.close() o.close() -# go +# build me = sys.argv[0] sys.argv = [ me ] sys.argv.append( "build" ) @@ -55,6 +43,7 @@ libdrmaa = os.path.join(SGE6_ROOT, "lib", SGE6_ARCH, "libdrmaa.dylib.1.0" ) os.system( "install_name_tool -change libdrmaa.dylib.1.0 %s %s" % ( libdrmaa, so ) ) +# package sys.argv = [ me ] sys.argv.append( "bdist_egg" ) execfile( "setup.py", globals(), locals() ) diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/scripts/DRMAA_python-solaris.py --- a/scripts/scramble/scripts/DRMAA_python-solaris.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/scramble/scripts/DRMAA_python-solaris.py Mon Mar 01 15:41:30 2010 -0500 @@ -1,8 +1,8 @@ import os, sys, shutil if "SGE_ROOT" not in os.environ: - print "scramble(): Please set SGE_ROOT to the path of your SGE installation" - print "scramble(): before scrambling DRMAA_python" + print "main(): Please set SGE_ROOT to the path of your SGE installation" + print "main(): before scrambling DRMAA_python" sys.exit(1) # change back to the build dir @@ -10,22 +10,11 @@ os.chdir( os.path.dirname( sys.argv[0] ) ) # find setuptools -scramble_lib = os.path.join( "..", "..", "..", "lib" ) -sys.path.append( scramble_lib ) -try: - from setuptools import * - import pkg_resources -except: - from ez_setup import use_setuptools - use_setuptools( download_delay=8, to_dir=scramble_lib ) - from setuptools import * - import pkg_resources +sys.path.append( os.path.join( '..', '..', '..', 'lib' ) ) +from scramble_lib import * -# clean, in case you're running this by hand from a dirty module source dir -for dir in [ "build", "dist", "gridengine" ]: - if os.access( dir, os.F_OK ): - print "scramble_it.py: removing dir:", dir - shutil.rmtree( dir ) +tag = get_tag() # get the tag +clean() # clean up any existing stuff (could happen if you run scramble.py by hand) # if our python is 64 bit, use 64 bit sge... if sys.maxint < ( 2048 * 1024 * 1024 ): @@ -38,24 +27,13 @@ arch = "sparc64" else: arch = "amd64" - if not "CFLAGS" in os.environ: - os.environ["CFLAGS"] = "" - os.environ["CFLAGS"] += " -m64" # if we're using sun cc, drop the gcc -Wno-unused option -import distutils.sysconfig -cc = distutils.sysconfig.get_config_var('CC') -if os.popen( cc + ' --version 2>&1' ).read().strip().split('\n')[0].startswith('gcc'): - compiler = 'gcc' -elif os.popen( cc + ' -V 2>&1' ).read().strip().split('\n')[0].startswith('cc: Sun C'): - compiler = 'sun' -else: - print "scramble(): Unable to determine compiler" - sys.exit(1) +cc = get_solaris_compiler() # patch file = "setup.py" -print "scramble(): Patching", file +print "main(): Patching", file if not os.access( "%s.orig" %file, os.F_OK ): shutil.copyfile( file, "%s.orig" %file ) i = open( "%s.orig" %file, "r" ) @@ -68,14 +46,16 @@ elif line.startswith('link_args ='): line = 'link_args = [ "-L%s" % os.path.join(SGE6_ROOT, "lib", SGE6_ARCH), "-Wl,-R%s" % os.path.join(SGE6_ROOT, "lib", SGE6_ARCH), "-ldrmaa" ]\n' if line == " + [ '-Wno-unused' ]\n": - if compiler == 'sun': + if cc == 'cc': line = " #+ [ '-Wno-unused' ]\n" print >>o, line, i.close() o.close() -# go +# reset args for distutils me = sys.argv[0] sys.argv = [ me ] sys.argv.append( "bdist_egg" ) + +# do it execfile( "setup.py", globals(), locals() ) diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/scripts/DRMAA_python.py --- a/scripts/scramble/scripts/DRMAA_python.py Mon Mar 01 13:30:37 2010 -0500 +++ b/scripts/scramble/scripts/DRMAA_python.py Mon Mar 01 15:41:30 2010 -0500 @@ -1,8 +1,8 @@ import os, sys, shutil if "SGE_ROOT" not in os.environ: - print "scramble(): Please set SGE_ROOT to the path of your SGE installation" - print "scramble(): before scrambling DRMAA_python" + print "main(): Please set SGE_ROOT to the path of your SGE installation" + print "main(): before scrambling DRMAA_python" sys.exit(1) # change back to the build dir @@ -10,27 +10,15 @@ os.chdir( os.path.dirname( sys.argv[0] ) ) # find setuptools -scramble_lib = os.path.join( "..", "..", "..", "lib" ) -sys.path.append( scramble_lib ) -import get_platform # fixes fat python 2.5 -try: - from setuptools import * - import pkg_resources -except: - from ez_setup import use_setuptools - use_setuptools( download_delay=8, to_dir=scramble_lib ) - from setuptools import * - import pkg_resources +sys.path.append( os.path.join( '..', '..', '..', 'lib' ) ) +from scramble_lib import * -# clean, in case you're running this by hand from a dirty module source dir -for dir in [ "build", "dist", "gridengine" ]: - if os.access( dir, os.F_OK ): - print "scramble_it.py: removing dir:", dir - shutil.rmtree( dir ) +tag = get_tag() # get the tag +clean() # clean up any existing stuff (could happen if you run scramble.py by hand) # patch file = "setup.py" -print "scramble(): Patching", file +print "main(): Patching", file if not os.access( "%s.orig" %file, os.F_OK ): shutil.copyfile( file, "%s.orig" %file ) i = open( "%s.orig" %file, "r" ) @@ -44,8 +32,10 @@ i.close() o.close() -# go +# reset args for distutils me = sys.argv[0] sys.argv = [ me ] sys.argv.append( "bdist_egg" ) + +# do it execfile( "setup.py", globals(), locals() ) diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/scripts/GeneTrack.py --- a/scripts/scramble/scripts/GeneTrack.py Mon Mar 01 13:30:37 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,58 +0,0 @@ -import os, sys, shutil - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -scramble_lib = os.path.join( "..", "..", "..", "lib" ) -sys.path.append( scramble_lib ) -from ez_setup import use_setuptools -use_setuptools( download_delay=8, to_dir=scramble_lib ) -from setuptools import * - -# get the tag -if os.access( ".galaxy_tag", os.F_OK ): - tagfile = open( ".galaxy_tag", "r" ) - tag = tagfile.readline().strip() -else: - tag = None - -# in case you're running this by hand from a dirty module source dir -for dir in [ "build", "dist" ]: - if os.access( dir, os.F_OK ): - print "scramble.py: removing dir:", dir - shutil.rmtree( dir ) - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "egg_info" ) -if tag is not None: - #sys.argv.append( "egg_info" ) - sys.argv.append( "--tag-build=%s" %tag ) -# svn revision (if any) is handled directly in tag-build -sys.argv.append( "--no-svn-revision" ) -sys.argv.append( "bdist_egg" ) - -if not os.access( 'setup.py', os.F_OK ): - print "scramble.py: Creating setup.py for GeneTrack" - setup_py = """from os import walk -from os.path import join -from setuptools import setup, find_packages -def walk_files( top ): - for dir, dirs, files in walk( top ): - yield( dir, [ join( dir, f ) for f in files ] ) -setup( - name = "GeneTrack", - version = "2.0.0-beta-1", - packages = ['genetrack','genetrack.scripts'], - data_files = [ f for f in walk_files('tests') ], - zip_safe = False -) -""" - open( 'setup.py', 'w' ).write( setup_py ) - - -# do it -execfile( "setup.py", globals(), locals() ) diff -r a6c9862ece3d -r 0cb68935eede scripts/scramble/scripts/MySQL_python-solaris.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/scramble/scripts/MySQL_python-solaris.py Mon Mar 01 15:41:30 2010 -0500 @@ -0,0 +1,81 @@ +import os, sys, shutil +from distutils.sysconfig import get_config_var + +def prep_mysql( prepped, args ): + + my_version = args['version'] + my_srcdir = os.path.join( os.getcwd(), "mysql-%s" % my_version ) + + # set up environment + os.environ['CC'] = get_config_var('CC') + os.environ['CFLAGS'] = get_config_var('CFLAGS') + os.environ['LDFLAGS'] = get_config_var('LDFLAGS') + + cc = get_solaris_compiler() + if cc == 'cc': + os.environ['CFLAGS'] += ' -KPIC' + elif cc == 'gcc': + os.environ['CFLAGS'] += ' -fPIC -DPIC' + + # run configure + run( "./configure --prefix=%s/mysql --disable-dependency-tracking --enable-static --disable-shared --without-server --without-uca " %os.getcwd() + \ + "--without-libwrap --without-ssl --without-docs --without-man --enable-thread-safe-client --with-named-curses-libs=''", + my_srcdir, "Configuring mysql (./configure)" ) + + # compile + run( "make", os.path.join( my_srcdir, 'include' ), "Preparing mysql includes (cd include; make)" ) + run( "make link_sources", os.path.join( my_srcdir, 'libmysql' ), "Preparing libmysql (cd libmysql; make link_sources)" ) + run( "make link_sources", os.path.join( my_srcdir, 'libmysql_r' ), "Preparing libmysql_r (cd libmysql_r; make link_sources)" ) + run( "make", os.path.join( my_srcdir, 'libmysql_r' ), "Compiling libmysql_r (cd libmysql_r; make)" ) + run( "make", os.path.join( my_srcdir, 'scripts' ), "Preparing scripts (cd scripts; make)" ) + + # install + run( "make install", os.path.join( my_srcdir, 'include' ), "Installing mysql includes (cd include; make install)" ) + run( "make install", os.path.join( my_srcdir, 'libmysql_r' ), "Installing libmysql_r (cd libmysql_r; make install)" ) + run( "make install", os.path.join( my_srcdir, 'scripts' ), "Installing mysql scripts (cd scripts; make install)" ) + shutil.copy( os.path.join( my_srcdir, 'include', 'mysqld_error.h' ), os.path.join( 'mysql', 'include', 'mysql' ) ) + + # create prepped archive + print "%s(): Creating prepped archive for future builds at:" % sys._getframe().f_code.co_name + print " ", prepped + compress( prepped, + 'mysql/bin/mysql_config', + 'mysql/include', + 'mysql/lib' ) + +if __name__ == '__main__': + + # change back to the build dir + if os.path.dirname( sys.argv[0] ) != "": + os.chdir( os.path.dirname( sys.argv[0] ) ) + + # find setuptools + sys.path.append( os.path.abspath( os.path.join( '..', '..', '..', 'lib' ) ) ) + from scramble_lib import * + + tag = get_tag() + + my_version = ( tag.split( '_' ) )[1] + my_archive_base = os.path.join( archives, 'mysql-%s' % my_version ) + my_archive = get_archive( my_archive_base ) + my_archive_prepped = os.path.join( archives, 'mysql-%s-%s.tar.gz' % ( my_version, platform_noucs ) ) + + # clean up any existing stuff (could happen if you run scramble.py by hand) + clean( [ 'mysql-%s' % my_version, 'mysql' ] ) + + # unpack mysql + unpack_dep( my_archive, my_archive_prepped, prep_mysql, dict( version=my_version ) ) + + # get site.cfg + shutil.copy( os.path.join( patches, 'MySQL_python', 'site.cfg' ), 'site.cfg' ) + + # tag + me = sys.argv[0] + sys.argv = [ me ] + if tag is not None: + sys.argv.append( "egg_info" ) + sys.argv.append( "--tag-build=%s" %tag ) + sys.argv.append( "bdist_egg" ) + + # go + execfile( "setup.py", globals(), locals() )