3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e89af5d76371/ Changeset: e89af5d76371 User: natefoo Date: 2015-02-12 17:13:24+00:00 Summary: Make it possible to disable egg fetching or the use of eggs entirely and provide a requirements.txt. Affected #: 3 files diff -r 93d2916499f4ddc3093adc65cb5718a25841aa15 -r e89af5d7637192c58d4ebde8986e627a9e07fb24 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -125,6 +125,34 @@ # but prefixed with install_ are also available). #install_database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE +# -- Galaxy framework dependencies (eggs) + +# Options in this section can also be provided as environment variables in the +# form `GALAXY_<OPTION_NAME>`, e.g. `GALAXY_ENABLE_EGGS=False`, etc. +# Environment variables override the settings in this config. + +# For more details, see the `requirements.txt` file at the root of the Galaxy +# source distribution. + +# Galaxy manages all of its dependent Python modules by downloading Python eggs +# from the Galaxy eggs distribution server at https://eggs.galaxyproject.org. +# You may choose to disable this and attempt to install dependencies by hand +# (e.g. with pip), but beware that this may cause problems due to version +# incompatibilities between Galaxy and its dependencies. Do not do this unless +# you know what you're doing. +#enable_eggs = True + +# You may choose to keep eggs enabled but disable fetching them automatically. +# (hint: To do a one-time fetch, you can disable here and then use +# `GALAXY_ENABLE_EGG_FETCH=True python ./scripts/fetch_eggs.py` from the root +# of the Galaxy source. +#enable_egg_fetch = True + +# If you'd like to prefer any dependencies found in your local environment +# before (fetching and) using Galaxy's egg of that dependency, you can enable +# the following option: +#try_dependencies_from_env = False + # -- Files and directories # Dataset files are stored in this directory. diff -r 93d2916499f4ddc3093adc65cb5718a25841aa15 -r e89af5d7637192c58d4ebde8986e627a9e07fb24 lib/galaxy/eggs/__init__.py --- a/lib/galaxy/eggs/__init__.py +++ b/lib/galaxy/eggs/__init__.py @@ -61,6 +61,7 @@ self.distribution = None self.dir = None self.removed_location = None + self.enable_fetch = crate.galaxy_config.enable_egg_fetch if self.name is not None and self.version is not None: self.set_distribution() @@ -191,7 +192,10 @@ try: rval = [] # resolve this egg and its dependencies - dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) + if self.enable_fetch: + dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) + else: + dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, lambda x: None ) for dist in dists: # if any of the resolved dependencies should be managed eggs but are being pulled from the wrong path, fix them if dist.project_name in self.crate.all_names and not os.path.realpath( dist.location ).startswith( os.path.realpath( self.dir ) ): @@ -250,7 +254,10 @@ if egg: # Store the removed path so the fetch method can use it egg.removed_location = location - r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch ) + if self.enable_fetch: + r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch ) + else: + r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, lambda x: None ) egg.removed_location = None else: r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env ) @@ -374,31 +381,48 @@ """ Try to resolve (e.g. fetch) all eggs in the crate. """ - if all: - eggs = self.all_eggs + if self.galaxy_config.enable_eggs and not self.galaxy_config.try_dependencies_from_env: + if all: + eggs = self.all_eggs + else: + eggs = self.config_eggs + eggs = filter( lambda x: x.name not in self.no_auto, eggs ) + missing = [] + for egg in eggs: + try: + egg.resolve() + except EggNotFetchable: + missing.append( egg ) + if missing: + raise EggNotFetchable( missing ) else: - eggs = self.config_eggs - eggs = filter( lambda x: x.name not in self.no_auto, eggs ) - missing = [] - for egg in eggs: - try: - egg.resolve() - except EggNotFetchable: - missing.append( egg ) - if missing: - raise EggNotFetchable( missing ) + log.info('Dependencies will attempt to be loaded from the environment') class GalaxyConfig( object ): always_conditional = ( 'pysam', 'ctypes', 'python_daemon' ) def __init__( self, config_file ): + self.enable_egg_fetch = False + self.enable_eggs = False + self.try_dependencies_from_env = False if config_file is None: self.config = None else: self.config = ConfigParser.ConfigParser() if self.config.read( config_file ) == []: raise Exception( "error: unable to read Galaxy config from %s" % config_file ) + self.enable_egg_fetch = True + self.enable_eggs = True + for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'): + try: + setattr(self, opt, string_as_bool(self.config.get('app:main', opt))) + except: + pass # use default + for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'): + var = 'GALAXY_' + opt.upper() + if var in os.environ: + setattr(self, opt, string_as_bool(os.environ[var])) def check_conditional( self, egg_name ): def check_pysam(): @@ -435,6 +459,13 @@ return False + +def string_as_bool( string ): + if str( string ).lower() in ( 'true', 'yes', 'on' ): + return True + else: + return False + def get_env(): env = pkg_resources.Environment( search_path='', platform=pkg_resources.get_platform() ) for dist in pkg_resources.find_distributions( eggs_dir, False ): @@ -446,6 +477,13 @@ def require( req_str ): c = Crate( None ) req = pkg_resources.Requirement.parse( req_str ) + if c.galaxy_config.try_dependencies_from_env or not c.galaxy_config.enable_eggs: + try: + return pkg_resources.working_set.require( req_str ) + except Exception as exc: + if not c.galaxy_config.enable_eggs: + raise + log.info("%s not found in local environment, will try Galaxy egg: %s", (req_str, exc)) # TODO: This breaks egg version requirements. Not currently a problem, but # it could become one. try: diff -r 93d2916499f4ddc3093adc65cb5718a25841aa15 -r e89af5d7637192c58d4ebde8986e627a9e07fb24 requirements.txt --- /dev/null +++ b/requirements.txt @@ -0,0 +1,66 @@ +# +# Galaxy has traditionally managed all of its own dependencies (in the form of +# precompiled Python eggs) for a few reasons: +# +# 1. We can ensure that Galaxy works with the versions of the dependencies that +# we require. +# 2. We can precompile them for numerous platforms, meaning that developers or +# system administrators attempting to use Galaxy do not need to wait while +# dependencies compile/install, nor do they need to have compilers or +# various development libraries installed on their system. +# +# However, you may want to manage these dependencies on your own. If so, this +# file can be used with pip to install all of the bare minimum dependencies for +# starting a Galaxy server. Use this in conjunction with the +# `enable_eggs = False` galaxy.ini option or `GALAXY_ENABLE_EGGS=False` +# environment variable +# +# Note that except in the case of SQLAlchemy, all dependencies will be +# installed at their newest versions. Galaxy has not been tested with these and +# you may (likely?) run into problems! The Galaxy Team does not support using +# Galaxy with dependency versions other than those in eggs.ini, however, if you +# do find an incompatibility between Galaxy and the latest version, we would +# appreciate if you would investigate and provide a code fix for the issue. We +# are trying to move away from our strict version dependency policy and this +# would greatly help in our efforts. + +# not available in PyPI, must be installed by hand: +#SVGFig + +Paste +PasteDeploy +docutils +wchartype +pexpect +amqp +repoze.lru +Routes +WebOb +SQLAlchemy<0.8 +Cheetah +pycrypto +MarkupSafe +WebHelpers +anyjson +kombu +numpy +bx-python # bx must be installed after numpy! +pysam +Mako +PyYAML +mercurial +pytz +Babel +decorator +Tempita +sqlalchemy-migrate +Beaker +Whoosh +paramiko +ssh +Fabric +boto +simplejson +bioblend +Parsley +WebError https://bitbucket.org/galaxy/galaxy-central/commits/6b03101f9eb4/ Changeset: 6b03101f9eb4 User: natefoo Date: 2015-02-12 17:31:58+00:00 Summary: Use GALAXY_CONFIG_ environment variables instead of GALAXY_ to fall in line with convention. Affected #: 3 files diff -r e89af5d7637192c58d4ebde8986e627a9e07fb24 -r 6b03101f9eb482477276e0c1b719964f02f51469 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -128,8 +128,8 @@ # -- Galaxy framework dependencies (eggs) # Options in this section can also be provided as environment variables in the -# form `GALAXY_<OPTION_NAME>`, e.g. `GALAXY_ENABLE_EGGS=False`, etc. -# Environment variables override the settings in this config. +# form `GALAXY_CONFIG_<OPTION_NAME>`, e.g. `GALAXY_CONFIG_ENABLE_EGGS=False`, +# etc. Environment variables override the settings in this config. # For more details, see the `requirements.txt` file at the root of the Galaxy # source distribution. @@ -144,8 +144,8 @@ # You may choose to keep eggs enabled but disable fetching them automatically. # (hint: To do a one-time fetch, you can disable here and then use -# `GALAXY_ENABLE_EGG_FETCH=True python ./scripts/fetch_eggs.py` from the root -# of the Galaxy source. +# `GALAXY_CONFIG_ENABLE_EGG_FETCH=True python ./scripts/fetch_eggs.py` from the +# root of the Galaxy source. #enable_egg_fetch = True # If you'd like to prefer any dependencies found in your local environment diff -r e89af5d7637192c58d4ebde8986e627a9e07fb24 -r 6b03101f9eb482477276e0c1b719964f02f51469 lib/galaxy/eggs/__init__.py --- a/lib/galaxy/eggs/__init__.py +++ b/lib/galaxy/eggs/__init__.py @@ -420,7 +420,7 @@ except: pass # use default for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'): - var = 'GALAXY_' + opt.upper() + var = 'GALAXY_CONFIG_' + opt.upper() if var in os.environ: setattr(self, opt, string_as_bool(os.environ[var])) diff -r e89af5d7637192c58d4ebde8986e627a9e07fb24 -r 6b03101f9eb482477276e0c1b719964f02f51469 requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ # However, you may want to manage these dependencies on your own. If so, this # file can be used with pip to install all of the bare minimum dependencies for # starting a Galaxy server. Use this in conjunction with the -# `enable_eggs = False` galaxy.ini option or `GALAXY_ENABLE_EGGS=False` +# `enable_eggs = False` galaxy.ini option or `GALAXY_CONFIG_ENABLE_EGGS=False` # environment variable # # Note that except in the case of SQLAlchemy, all dependencies will be https://bitbucket.org/galaxy/galaxy-central/commits/b0452677a40f/ Changeset: b0452677a40f User: natefoo Date: 2015-02-12 17:39:27+00:00 Summary: Require Whoosh<2.5 as per Martin. Affected #: 1 file diff -r 6b03101f9eb482477276e0c1b719964f02f51469 -r b0452677a40f85d4b92cbcb1ebe4e137dbbfbec1 requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -55,7 +55,7 @@ Tempita sqlalchemy-migrate Beaker -Whoosh +Whoosh<2.5 paramiko ssh Fabric Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.