# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate@bx.psu.edu> # Date 1283182318 14400 # Node ID ecf203e9d70295c9850cad575c63c1760cd83c73 # Parent 00336a54c6b1f1202c14737f7cc85bd04fa66247 Fix the platform argument to fetch_eggs.py and create an egg packager for easing the installation/updating of Galaxy on systems without a direct connection to the internet. --- /dev/null +++ b/scripts/egg_packager_template.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python + +# Configure stdout logging + +import os, sys, logging, glob, zipfile, shutil + +log = logging.getLogger() +log.setLevel( 10 ) +log.addHandler( logging.StreamHandler( sys.stdout ) ) + +# Fake pkg_resources + +import re + +macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)") +darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)") +solarisVersionString = re.compile(r"solaris-(\d)\.(\d+)-(.*)") + +def compatible_platforms(provided,required): + """Can code for the `provided` platform run on the `required` platform? + + Returns true if either platform is ``None``, or the platforms are equal. + + XXX Needs compatibility checks for Linux and other unixy OSes. + """ + if provided is None or required is None or provided==required: + return True # easy case + + # Mac OS X special cases + reqMac = macosVersionString.match(required) + if reqMac: + provMac = macosVersionString.match(provided) + + # is this a Mac package? + if not provMac: + # this is backwards compatibility for packages built before + # setuptools 0.6. All packages built after this point will + # use the new macosx designation. + provDarwin = darwinVersionString.match(provided) + if provDarwin: + dversion = int(provDarwin.group(1)) + macosversion = "%s.%s" % (reqMac.group(1), reqMac.group(2)) + if dversion == 7 and macosversion >= "10.3" or \ + dversion == 8 and macosversion >= "10.4": + + #import warnings + #warnings.warn("Mac eggs should be rebuilt to " + # "use the macosx designation instead of darwin.", + # category=DeprecationWarning) + return True + return False # egg isn't macosx or legacy darwin + + # are they the same major version and machine type? + if provMac.group(1) != reqMac.group(1) or \ + provMac.group(3) != reqMac.group(3): + return False + + + + # is the required OS major update >= the provided one? + if int(provMac.group(2)) > int(reqMac.group(2)): + return False + + return True + + # Solaris' special cases + reqSol = solarisVersionString.match(required) + if reqSol: + provSol = solarisVersionString.match(provided) + if not provSol: + return False + if provSol.group(1) != reqSol.group(1) or \ + provSol.group(3) != reqSol.group(3): + return False + if int(provSol.group(2)) > int(reqSol.group(2)): + return False + return True + + # XXX Linux and other platforms' special cases should go here + return False + +EGG_NAME = re.compile( + r"(?P<name>[^-]+)" + r"( -(?P<ver>[^-]+) (-py(?P<pyver>[^-]+) (-(?P<plat>.+))? )? )?", + re.VERBOSE | re.IGNORECASE +).match + +class Distribution( object ): + def __init__( self, egg_name, project_name, version, py_version, platform ): + self._egg_name = egg_name + self.project_name = project_name + if project_name is not None: + self.project_name = project_name.replace( '-', '_' ) + self.version = version + if version is not None: + self.version = version.replace( '-', '_' ) + self.py_version = py_version + self.platform = platform + self.location = os.path.join( tmpd, egg_name ) + '.egg' + def egg_name( self ): + return self._egg_name + @classmethod + def from_filename( cls, basename ): + project_name, version, py_version, platform = [None]*4 + basename, ext = os.path.splitext(basename) + if ext.lower() == '.egg': + match = EGG_NAME( basename ) + if match: + project_name, version, py_version, platform = match.group( 'name','ver','pyver','plat' ) + return cls( basename, project_name, version, py_version, platform ) + +class pkg_resources( object ): + pass + +pkg_resources.Distribution = Distribution + +# Fake galaxy.eggs + +env = None +def get_env(): + return None + +import urllib, urllib2, HTMLParser +class URLRetriever( urllib.FancyURLopener ): + def http_error_default( *args ): + urllib.URLopener.http_error_default( *args ) + +class Egg( object ): + def __init__( self, distribution ): + self.url = url + '/' + distribution.project_name.replace( '-', '_' ) + self.dir = tmpd + self.distribution = distribution + def set_distribution( self ): + pass + def unpack_if_needed( self ): + pass + def remove_doppelgangers( self ): + pass + def fetch( self, requirement ): + """ + fetch() serves as the install method to pkg_resources.working_set.resolve() + """ + def find_alternative(): + """ + Some platforms (e.g. Solaris) support eggs compiled on older platforms + """ + class LinkParser( HTMLParser.HTMLParser ): + """ + Finds links in what should be an Apache-style directory index + """ + def __init__( self ): + HTMLParser.HTMLParser.__init__( self ) + self.links = [] + def handle_starttag( self, tag, attrs ): + if tag == 'a' and 'href' in dict( attrs ): + self.links.append( dict( attrs )['href'] ) + parser = LinkParser() + try: + parser.feed( urllib2.urlopen( self.url + '/' ).read() ) + except urllib2.HTTPError, e: + if e.code == 404: + return None + parser.close() + for link in parser.links: + file = urllib.unquote( link ).rsplit( '/', 1 )[-1] + tmp_dist = pkg_resources.Distribution.from_filename( file ) + if tmp_dist.platform is not None and \ + self.distribution.project_name == tmp_dist.project_name and \ + self.distribution.version == tmp_dist.version and \ + self.distribution.py_version == tmp_dist.py_version and \ + compatible_platforms( tmp_dist.platform, self.distribution.platform ): + return file + return None + if self.url is None: + return None + alternative = None + try: + url = self.url + '/' + self.distribution.egg_name() + '.egg' + URLRetriever().retrieve( url, self.distribution.location ) + log.debug( "Fetched %s" % url ) + except IOError, e: + if e[1] == 404 and self.distribution.platform != py: + alternative = find_alternative() + if alternative is None: + return None + else: + return None + if alternative is not None: + try: + url = '/'.join( ( self.url, alternative ) ) + URLRetriever().retrieve( url, os.path.join( self.dir, alternative ) ) + log.debug( "Fetched %s" % url ) + except IOError, e: + return None + self.platform = alternative.split( '-', 2 )[-1].rsplit( '.egg', 1 )[0] + self.set_distribution() + self.unpack_if_needed() + self.remove_doppelgangers() + global env + env = get_env() # reset the global Environment object now that we've obtained a new egg + return self.distribution + +def create_zip(): + fname = 'galaxy_eggs-%s.zip' % platform + z = zipfile.ZipFile( fname, 'w', zipfile.ZIP_STORED ) + for egg in glob.glob( os.path.join( tmpd, '*.egg' ) ): + z.write( egg, 'eggs/' + os.path.basename( egg ) ) + z.close() + print 'Egg package is in %s' % fname + print "To install the eggs, please copy this file to your Galaxy installation's root" + print "directory and unpack with:" + print " unzip %s" % fname + +def clean(): + shutil.rmtree( tmpd ) + +import tempfile +tmpd = tempfile.mkdtemp() + +failures = [] + +# Automatically generated egg definitions follow + --- a/scripts/fetch_eggs.py +++ b/scripts/fetch_eggs.py @@ -21,11 +21,10 @@ sys.path.append( lib ) from galaxy.eggs import Crate, EggNotFetchable import pkg_resources -c = Crate() try: - c.platform = sys.argv[2] + c = Crate( platform = sys.argv[2] ) except: - pass + c = Crate() try: if len( sys.argv ) == 1: c.resolve() # Only fetch eggs required by the config --- a/lib/galaxy/eggs/__init__.py +++ b/lib/galaxy/eggs/__init__.py @@ -190,11 +190,15 @@ class Crate( object ): Reads the eggs.ini file for use with checking and fetching. """ config_file = os.path.join( galaxy_dir, 'eggs.ini' ) - def __init__( self ): + def __init__( self, platform=None ): self.eggs = {} self.config = CaseSensitiveConfigParser() self.repo = None self.no_auto = [] + self.platform = platform + self.py_platform = None + if platform is not None: + self.py_platform = platform.split( '-' )[0] self.galaxy_config = GalaxyConfig() self.parse() def parse( self ): @@ -208,9 +212,9 @@ class Crate( object ): tag = dict( tags ).get( name, '' ) url = '/'.join( ( self.repo, name ) ) if full_platform: - platform = '-'.join( ( py, pkg_resources.get_platform() ) ) + platform = self.platform or '-'.join( ( py, pkg_resources.get_platform() ) ) else: - platform = py + platform = self.py_platform or py egg = egg_class( name, version, tag, url, platform ) self.eggs[name] = egg @property @@ -261,8 +265,6 @@ class Crate( object ): for egg in self.eggs.values(): if egg.name not in self.galaxy_config.always_conditional: rval.append( egg ) - elif self.galaxy_config.check_conditional( egg.name ): - rval.append( egg ) return rval def __getitem__( self, name ): """ --- /dev/null +++ b/scripts/make_egg_packager.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +import os, sys, logging, shutil + +root = logging.getLogger() +root.setLevel( 10 ) +root.addHandler( logging.StreamHandler( sys.stdout ) ) + +lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) +sys.path.append( lib ) + +from galaxy.eggs import Crate, EggNotFetchable, py +import pkg_resources + +try: + platform = sys.argv[1] + c = Crate( platform = platform ) + print "Platform forced to '%s'" % platform +except: + platform = '-'.join( ( py, pkg_resources.get_platform() ) ) + c = Crate() + print "Using Python interpreter at %s, Version %s" % ( sys.executable, sys.version ) + print "This platform is '%s'" % platform + print "Override with:" + print " make_egg_packager.py <forced-platform>" + +shutil.copy( os.path.join( os.path.dirname( __file__ ), 'egg_packager_template.py' ), 'egg_packager-%s.py' % platform ) + +packager = open( 'egg_packager-%s.py' % platform, 'a' ) +packager.write( "py = '%s'\n" % py ) +packager.write( "url = '%s'\n" % c.repo ) +packager.write( "platform = '%s'\n" % platform ) +packager.write( "dists = [\n" ) + +for egg in c.all_eggs: + if egg.name in c.no_auto: + continue + packager.write( " Distribution( '%s', '%s', '%s', '%s', '%s' ),\n" % ( egg.distribution.egg_name(), egg.distribution.project_name, egg.distribution.version, egg.distribution.py_version, egg.distribution.platform ) ) + +packager.write( """] + +for d in dists: + e = Egg( d ) + if not e.fetch( None ): + failures.append( e ) + +if failures: + print "" + print "Failed:" + for e in failures: + print e.distribution.project_name +else: + create_zip() +clean() +""" ) + +print "Completed packager is 'egg_packager-%s.py'. To" % platform +print "fetch eggs, please copy this file to a system with internet access and run" +print "with:" +print " python egg_packager-%s.py" % platform