details: http://www.bx.psu.edu/hg/galaxy/rev/fdf07565eedf changeset: 2963:fdf07565eedf user: Nate Coraor nate@bx.psu.edu date: Wed Nov 04 12:58:48 2009 -0500 description: Add the GeneTrack egg. Also includes some enhancements and cleanup of the eggs/fetch/scramble code.
diffstat:
dist-eggs.ini | 4 + eggs.ini | 5 +- lib/galaxy/config.py | 13 +++ lib/galaxy/eggs/__init__.py | 149 ++++++++++++++---------------------- scripts/fetch_eggs.py | 36 ++++---- scripts/scramble.py | 28 +++--- scripts/scramble/scripts/GeneTrack.py | 52 +++++++++++++ 7 files changed, 164 insertions(+), 123 deletions(-)
diffs (490 lines):
diff -r 92d395c8614b -r fdf07565eedf dist-eggs.ini --- a/dist-eggs.ini Wed Nov 04 12:30:36 2009 -0500 +++ b/dist-eggs.ini Wed Nov 04 12:58:48 2009 -0500 @@ -57,3 +57,7 @@ all = py2.4-all py2.5-all py2.6-all ; default hosts for platform-inspecific eggs noplatform = py2.4-linux-i686-ucs4 py2.5-linux-i686-ucs4 py2.6-linux-i686-ucs4 + +[ignore] +; Don't build these eggs on these platforms: +GeneTrack = py2.4-noplatform diff -r 92d395c8614b -r fdf07565eedf eggs.ini --- a/eggs.ini Wed Nov 04 12:30:36 2009 -0500 +++ b/eggs.ini Wed Nov 04 12:58:48 2009 -0500 @@ -9,7 +9,7 @@ [general] repository = http://eggs.g2.bx.psu.edu ; these eggs must be scrambled for your local environment -no_download = pbs_python DRMAA_python +no_auto = pbs_python DRMAA_python
[eggs:platform] bx_python = 0.5.0 @@ -31,6 +31,7 @@ decorator = 3.1.2 docutils = 0.4 elementtree = 1.2.6_20050316 +GeneTrack = 2.0.0_beta_1 lrucache = 0.2 ;lsprof - james Mako = 0.2.5 @@ -61,6 +62,7 @@ python_lzo = _static flup = .dev_r2311 bx_python = _dev_r4bf1f32e6b76 +GeneTrack = _dev_raa786e9fc131d998e532a1aef39d108850c9e93d ; nose = .dev_r7156749efc58
; source location, necessary for scrambling @@ -81,6 +83,7 @@ decorator = http://pypi.python.org/packages/source/d/decorator/decorator-3.1.2.tar.gz docutils = http://downloads.sourceforge.net/docutils/docutils-0.4.tar.gz elementtree = http://effbot.org/downloads/elementtree-1.2.6-20050316.tar.gz +GeneTrack = http://github.com/ialbert/genetrack-central/tarball/aa786e9fc131d998e532a1ae... lrucache = http://evan.prodromou.name/lrucache/lrucache-0.2.tar.gz Mako = http://www.makotemplates.org/downloads/Mako-0.2.5.tar.gz MyghtyUtils = http://cheeseshop.python.org/packages/source/M/MyghtyUtils/MyghtyUtils-0.52.... diff -r 92d395c8614b -r fdf07565eedf lib/galaxy/config.py --- a/lib/galaxy/config.py Wed Nov 04 12:30:36 2009 -0500 +++ b/lib/galaxy/config.py Wed Nov 04 12:58:48 2009 -0500 @@ -8,6 +8,9 @@ import ConfigParser from galaxy.util import string_as_bool
+from galaxy import eggs +import pkg_resources + log = logging.getLogger( __name__ )
def resolve_path( path, root ): @@ -126,6 +129,16 @@ for path in self.tool_config, self.datatypes_config: if not os.path.isfile(path): raise ConfigurationError("File not found: %s" % path ) + # Check job runners so the admin can scramble dependent egg. + if self.start_job_runners is not None: + runner_to_egg = dict( pbs = 'pbs_python', sge = 'DRMAA_python' ) + for runner in self.start_job_runners.split( ',' ): + try: + pkg_resources.require( runner_to_egg[runner] ) + except eggs.EggNotFetchable, e: + raise eggs.EggNotFetchable( 'You must scramble the %s egg to use the %s job runner. Instructions are available at:\n http://bitbucket.org/galaxy/galaxy-central/wiki/Config/Cluster' % ( runner_to_egg[runner], runner ) ) + except KeyError: + raise Exception( 'No such job runner: %s. Please double-check the value of start_job_runners in universe_wsgi.ini' % runner )
def is_admin_user( self,user ): """ diff -r 92d395c8614b -r fdf07565eedf lib/galaxy/eggs/__init__.py --- a/lib/galaxy/eggs/__init__.py Wed Nov 04 12:30:36 2009 -0500 +++ b/lib/galaxy/eggs/__init__.py Wed Nov 04 12:58:48 2009 -0500 @@ -14,11 +14,12 @@ # within tools. i don't know of any way around this. -ndc galaxy_dir = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "..", ".." ) )
-class NewEgg( Exception ): - pass - class EggNotFetchable( Exception ): - pass + def __init__( self, eggs ): + if type( eggs ) in ( list, tuple ): + self.eggs = eggs + else: + self.eggs = [ eggs ]
class PlatformNotSupported( Exception ): pass @@ -157,6 +158,14 @@ # scramble helper methods def get_archive_path( self, url ): return os.path.join( Egg.archive_dir, (url.rsplit( '/', 1 ))[1] ) + def get_tld( self, names ): + tld = names[0].split( os.path.sep, 1 )[0] + for name in names: + try: + assert tld == name.split( os.path.sep, 1 )[0] + except: + raise Exception( "get_tld(): Archive contains multiple top-level directories!" ) + return tld def fetch_source( self ): if not os.access( Egg.archive_dir, os.F_OK ): os.makedirs( Egg.archive_dir ) @@ -195,7 +204,7 @@ log.warning( " %s" % self.buildpath ) def unpack_zip( self, source_path, unpack_path ): z = zipfile.ZipFile( source_path, "r" ) - tld = ( z.namelist()[0].split( os.path.sep, 1 ) )[0] + tld = self.get_tld( z.namelist() ) cur = os.getcwd() os.chdir( unpack_path ) for fn in z.namelist(): @@ -211,12 +220,12 @@ os.chdir( cur ) def unpack_tar( self, source_path, unpack_path ): t = tarfile.open( source_path, "r" ) - tld = ( t.getnames()[0].split( os.path.sep, 1 ) )[0] + members = filter( lambda x: "ez_setup" not in x.name and "pax_global_header" != x.name, t.getmembers() ) + tld = self.get_tld( [ x.name for x in members ] ) cur = os.getcwd() os.chdir( unpack_path ) - for member in t.getmembers(): - if "ez_setup" not in member.name: - t.extract( member ) + for member in members: + t.extract( member ) t.close() os.rename( tld, self.name ) os.chdir( cur ) @@ -265,7 +274,7 @@ self.eggs = {} self.config = CSConfigParser() self.repo = None - self.no_download = [] + self.no_auto = [] self.platform = { 'peak' : get_platform( platform=True, peak=True ), 'galaxy' : get_platform( platform=True, peak=False ) } self.noplatform = { 'peak' : get_platform( platform=False, peak=True ), 'galaxy' : get_platform( platform=False, peak=False ) } def parse( self ): @@ -273,7 +282,7 @@ raise Exception( "unable to read egg config from %s" % Crate.config_file ) try: self.repo = self.config.get( "general", "repository" ) - self.no_download = self.config.get( "general", "no_download" ).split() + self.no_auto = self.config.get( "general", "no_auto" ).split() except ConfigParser.NoSectionError: raise Exception( "eggs.ini is missing required section [general]" ) #except ConfigParser.NoOptionError: @@ -316,19 +325,19 @@ return True def fetch( self, ignore=[] ): """ - Fetch all eggs in the crate (ignoring any that you want to - ignore). If your platform isn't available, it'll attempt to - download all the noplatform eggs before failing. + Fetch all eggs in the crate (ignoring any that you want to + ignore). If your platform isn't available, it'll attempt to + download all the noplatform eggs before failing. """ skip_platform = False - ignore.extend( self.no_download ) + ignore.extend( self.no_auto ) + missing = [] try: f = urllib2.urlopen( "%s/%s" % ( self.repo, self.platform['galaxy'] ) ) f.close() except urllib2.HTTPError, e: if e.code == 404: skip_platform = True - missing = [] for egg in self.eggs.itervalues(): if ignore is not None: if egg.name in ignore: @@ -336,11 +345,18 @@ if skip_platform and egg.platform['galaxy'] == self.platform['galaxy']: missing.append( egg.name ) continue - egg.fetch() + try: + egg.fetch() + except EggNotFetchable: + missing.append( egg.name ) if skip_platform: raise PlatformNotSupported( self.platform['galaxy'] ) + if missing: + raise EggNotFetchable( missing ) return True def scramble( self, ignore=None ): + # Crate-scrambling the no_auto eggs makes no sense + ignore.extend( self.no_auto ) for egg in self.eggs.itervalues(): if ignore is not None: if egg.name in ignore: @@ -379,21 +395,14 @@ if self.config.read( DistCrate.dist_config_file ) == []: raise Exception( "unable to read dist egg config from %s" % DistCrate.dist_config_file ) try: - self.hosts = self.dictize_list_of_tuples( self.config.items( "hosts" ) ) - self.groups = self.dictize_list_of_tuples( self.config.items( "groups" ) ) + self.hosts = dict( self.config.items( "hosts" ) ) + self.groups = dict( self.config.items( "groups" ) ) + self.ignore = dict( self.config.items( "ignore" ) ) except ConfigParser.NoSectionError, e: raise Exception( "eggs.ini is missing required section: %s" % e ) self.platforms = self.get_platforms( self.build_on ) self.noplatforms = self.get_platforms( 'noplatform' ) Crate.parse( self ) - def dictize_list_of_tuples( self, lot ): - """ - Makes a list of 2-value tuples into a dict. - """ - d = {} - for k, v in lot: - d[k] = v - return d def get_platforms( self, wanted ): # find all the members of a group and process them if self.groups.has_key( wanted ): @@ -409,8 +418,8 @@ raise Exception( "unknown platform: %s" % wanted ) def parse_egg_section( self, eggs, type ): """ - Overrides the base class's method. Here we use the third arg - to find out what type of egg we'll be building. + Overrides the base class's method. Here we use the third arg + to find out what type of egg we'll be building. """ if type == "platform": platforms = self.platforms @@ -418,14 +427,16 @@ platforms = self.noplatforms for name, version in eggs: for platform in platforms: - # can't use the regular methods here because we're not - # actually ON the target platform + # can't use the regular methods here because we're not + # actually ON the target platform if type == "platform": gplat = platform pplat = platform.rsplit('-', 1)[0] elif type == "noplatform": gplat = "%s-noplatform" % platform.split('-', 1)[0] pplat = platform.split('-', 1)[0] + if name in self.ignore and gplat in self.ignore[name].split(): + continue egg = Egg() try: egg.tag = self.config.get( "tags", name ) @@ -448,75 +459,31 @@
class GalaxyConfig: config_file = os.path.join( galaxy_dir, "universe_wsgi.ini" ) + always_conditional = ( 'GeneTrack', ) def __init__( self ): self.config = ConfigParser.ConfigParser() if self.config.read( GalaxyConfig.config_file ) == []: raise Exception( "error: unable to read Galaxy config from %s" % GalaxyConfig.config_file ) # TODO: conditionals should really be handled better than this def check_conditional( self, egg_name ): - if egg_name == "psycopg2": + if egg_name == "pysqlite": + # SQLite is different since it can be specified in two config vars and defaults to True try: - if self.config.get( "app:main", "database_connection" ).startswith( "postgres://" ): - return True - else: - return False + return self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" ) + except: + return True + else: + try: + return { "psycopg2": lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres://" ), + "MySQL_python": lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ), + "DRMAA_python": lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","), + "pbs_python": lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), + "threadframe": lambda: self.config.get( "app:main", "use_heartbeat" ), + "guppy": lambda: self.config.get( "app:main", "use_memdump" ), + "GeneTrack": lambda: sys.version_info[:2] >= ( 2, 5 ), + }.get( egg_name, lambda: True )() except: return False - elif egg_name == "pysqlite": - try: - # database connection is the sqlite alchemy dialect (not really - # a documented usage in Galaxy, but it would work) - if self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" ): - return True - else: - return False - # database connection is unset, so sqlite is the default - except: - return True - elif egg_name == "DRMAA_python": - try: - runners = self.config.get( "app:main", "start_job_runners" ).split(",") - if "sge" in runners: - return True - else: - return False - except: - return False - elif egg_name == "pbs_python": - try: - runners = self.config.get( "app:main", "start_job_runners" ).split(",") - if "pbs" in runners: - return True - else: - return False - except: - return False - elif egg_name == "threadframe": - try: - if self.config.get( "app:main", "use_heartbeat" ): - return True - else: - return False - except: - return False - elif egg_name == "guppy": - try: - if self.config.get( "app:main", "use_memdump" ): - return True - else: - return False - except: - return False - elif egg_name == "MySQL_python": - try: - if self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ): - return True - else: - return False - except: - return False - else: - return True
def require( pkg ): # add the egg dirs to sys.path if they're not already there diff -r 92d395c8614b -r fdf07565eedf scripts/fetch_eggs.py --- a/scripts/fetch_eggs.py Wed Nov 04 12:30:36 2009 -0500 +++ b/scripts/fetch_eggs.py Wed Nov 04 12:58:48 2009 -0500 @@ -25,26 +25,28 @@ c.platform = { 'peak' : sys.argv[2].rsplit('-',1)[0], 'galaxy' : sys.argv[2] } c.parse() try: + galaxy_config = GalaxyConfig() + names = [] if len( sys.argv ) == 1: - galaxy_config = GalaxyConfig() - ignore = [] - for name in c.get_names(): - if not galaxy_config.check_conditional( name ): - ignore.append( name ) - c.fetch( ignore=ignore ) + names = c.get_names() + elif sys.argv[1] == 'all': + names = galaxy_config.always_conditional else: - if sys.argv[1] == 'all': - c.fetch() - else: - egg = c.get( sys.argv[1] ) - if egg is None: - print "error: %s not in eggs.ini" % sys.argv[1] - sys.exit( 1 ) - egg.fetch() + # Fetch a specific egg + egg = c.get( sys.argv[1] ) + if egg is None: + print "error: %s not in eggs.ini" % sys.argv[1] + sys.exit( 1 ) + egg.fetch() + sys.exit( 0 ) + ignore = filter( lambda x: not galaxy_config.check_conditional( x ), list( names ) ) + c.fetch( ignore ) except EggNotFetchable, e: - print "One of the python eggs necessary to run Galaxy couldn't be downloaded" - print "automatically. You may want to try building it by hand with:" - print " python scripts/scramble.py %s" % e + print "One or more of the python eggs necessary to run Galaxy couldn't be" + print "downloaded automatically. You may want to try building them by" + print "hand with:" + for egg in e.eggs: + print " python scripts/scramble.py %s" % egg sys.exit( 1 ) except PlatformNotSupported, e: print "Your platform (%s) is not supported." % e diff -r 92d395c8614b -r fdf07565eedf scripts/scramble.py --- a/scripts/scramble.py Wed Nov 04 12:30:36 2009 -0500 +++ b/scripts/scramble.py Wed Nov 04 12:58:48 2009 -0500 @@ -18,19 +18,19 @@
c = Crate() c.parse() +galaxy_config = GalaxyConfig() +names = [] if len( sys.argv ) == 1: - galaxy_config = GalaxyConfig() - ignore = [] - for name in c.get_names(): - if not galaxy_config.check_conditional( name ): - ignore.append( name ) - c.scramble( ignore=ignore ) + names = c.get_names() +elif sys.argv[1] == 'all': + names = galaxy_config.always_conditional else: - if sys.argv[1] == 'all': - c.scramble() - else: - egg = c.get( sys.argv[1] ) - if egg is None: - print "error: %s not in eggs.ini" % sys.argv[1] - sys.exit( 1 ) - egg.scramble() +# Scramble a specific egg + egg = c.get( sys.argv[1] ) + if egg is None: + print "error: %s not in eggs.ini" % sys.argv[1] + sys.exit( 1 ) + egg.scramble() + sys.exit( 0 ) +ignore = filter( lambda x: not galaxy_config.check_conditional( x ), list( names ) ) +c.scramble( ignore=ignore ) diff -r 92d395c8614b -r fdf07565eedf scripts/scramble/scripts/GeneTrack.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/scramble/scripts/GeneTrack.py Wed Nov 04 12:58:48 2009 -0500 @@ -0,0 +1,52 @@ +import os, sys, shutil + +# change back to the build dir +if os.path.dirname( sys.argv[0] ) != "": + os.chdir( os.path.dirname( sys.argv[0] ) ) + +# find setuptools +scramble_lib = os.path.join( "..", "..", "..", "lib" ) +sys.path.append( scramble_lib ) +from ez_setup import use_setuptools +use_setuptools( download_delay=8, to_dir=scramble_lib ) +from setuptools import * + +# get the tag +if os.access( ".galaxy_tag", os.F_OK ): + tagfile = open( ".galaxy_tag", "r" ) + tag = tagfile.readline().strip() +else: + tag = None + +# in case you're running this by hand from a dirty module source dir +for dir in [ "build", "dist" ]: + if os.access( dir, os.F_OK ): + print "scramble.py: removing dir:", dir + shutil.rmtree( dir ) + +# reset args for distutils +me = sys.argv[0] +sys.argv = [ me ] +sys.argv.append( "egg_info" ) +if tag is not None: + #sys.argv.append( "egg_info" ) + sys.argv.append( "--tag-build=%s" %tag ) +# svn revision (if any) is handled directly in tag-build +sys.argv.append( "--no-svn-revision" ) +sys.argv.append( "bdist_egg" ) + +print "scramble.py: Creating setup.py for GeneTrack" +setup_py = """from setuptools import setup, find_packages +setup( + name = "GeneTrack", + version = "2.0.0-beta-1", + package_data = {'':["*.*"]}, + packages = find_packages(), + zip_safe = True, +) +""" +open( 'setup.py', 'w' ).write( setup_py ) + + +# do it +execfile( "setup.py", globals(), locals() )