1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/d540227165d5/
Changeset: d540227165d5
User: natefoo
Date: 2015-02-12 20:10:19+00:00
Summary: There's no reliable way to use the Galaxy config file to control egg use/fetching since so many things use galaxy.eggs from outside of Galaxy, so drop the support for the config file options and rely on the environment variables only. Reverse the defaults so that eggs and fetching are enabled by default. I'll make sure that they properly get passed to the tool execution environments in the next commit.
Affected #: 4 files
diff -r fd753316154e8ad2a92bebffda9441c8614a06f5 -r d540227165d54950208d3948e2bdecd4babc70ca config/galaxy.ini.sample
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -125,34 +125,6 @@
# but prefixed with install_ are also available).
#install_database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
-# -- Galaxy framework dependencies (eggs)
-
-# Options in this section can also be provided as environment variables in the
-# form `GALAXY_CONFIG_<OPTION_NAME>`, e.g. `GALAXY_CONFIG_ENABLE_EGGS=False`,
-# etc. Environment variables override the settings in this config.
-
-# For more details, see the `requirements.txt` file at the root of the Galaxy
-# source distribution.
-
-# Galaxy manages all of its dependent Python modules by downloading Python eggs
-# from the Galaxy eggs distribution server at https://eggs.galaxyproject.org.
-# You may choose to disable this and attempt to install dependencies by hand
-# (e.g. with pip), but beware that this may cause problems due to version
-# incompatibilities between Galaxy and its dependencies. Do not do this unless
-# you know what you're doing.
-#enable_eggs = True
-
-# You may choose to keep eggs enabled but disable fetching them automatically.
-# (hint: To do a one-time fetch, you can disable here and then use
-# `GALAXY_CONFIG_ENABLE_EGG_FETCH=True python ./scripts/fetch_eggs.py` from the
-# root of the Galaxy source.
-#enable_egg_fetch = True
-
-# If you'd like to prefer any dependencies found in your local environment
-# before (fetching and) using Galaxy's egg of that dependency, you can enable
-# the following option:
-#try_dependencies_from_env = False
-
# -- Files and directories
# Dataset files are stored in this directory.
diff -r fd753316154e8ad2a92bebffda9441c8614a06f5 -r d540227165d54950208d3948e2bdecd4babc70ca lib/galaxy/eggs/__init__.py
--- a/lib/galaxy/eggs/__init__.py
+++ b/lib/galaxy/eggs/__init__.py
@@ -403,8 +403,8 @@
always_conditional = ( 'pysam', 'ctypes', 'python_daemon' )
def __init__( self, config_file ):
- self.enable_egg_fetch = False
- self.enable_eggs = False
+ self.enable_egg_fetch = True
+ self.enable_eggs = True
self.try_dependencies_from_env = False
if config_file is None:
self.config = None
@@ -412,13 +412,6 @@
self.config = ConfigParser.ConfigParser()
if self.config.read( config_file ) == []:
raise Exception( "error: unable to read Galaxy config from %s" % config_file )
- self.enable_egg_fetch = True
- self.enable_eggs = True
- for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'):
- try:
- setattr(self, opt, string_as_bool(self.config.get('app:main', opt)))
- except:
- pass # use default
for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'):
var = 'GALAXY_CONFIG_' + opt.upper()
if var in os.environ:
diff -r fd753316154e8ad2a92bebffda9441c8614a06f5 -r d540227165d54950208d3948e2bdecd4babc70ca requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,8 +12,7 @@
# However, you may want to manage these dependencies on your own. If so, this
# file can be used with pip to install all of the bare minimum dependencies for
# starting a Galaxy server. Use this in conjunction with the
-# `enable_eggs = False` galaxy.ini option or `GALAXY_CONFIG_ENABLE_EGGS=False`
-# environment variable
+# `GALAXY_CONFIG_ENABLE_EGGS=False` environment variable.
#
# Note that except in the case of SQLAlchemy, all dependencies will be
# installed at their newest versions. Galaxy has not been tested with these and
@@ -62,4 +61,3 @@
Parsley
WebError
http://svgfig.googlecode.com/files/svgfig-1.1.6.tgz
-
diff -r fd753316154e8ad2a92bebffda9441c8614a06f5 -r d540227165d54950208d3948e2bdecd4babc70ca scripts/paster.py
--- a/scripts/paster.py
+++ b/scripts/paster.py
@@ -30,36 +30,6 @@
_log_tempfile = TempFile()
import tempfile
-try:
- serve = sys.argv.index('serve')
-except:
- print >>sys.stderr, "Galaxy does not use the real Paste Script, the only supported command is 'serve'"
- sys.exit(1)
-
-# eggs.require() can be called inside the app without access to the Galaxy
-# config, so we need to push the egg options into the environment so they are
-# available to Crate instantiated in require()
-
-# locate the arg containing the path to the config file
-config = None
-p = configparser.ConfigParser()
-for arg in sys.argv:
- try:
- p.read(arg)
- assert 'app:main' in p.sections()
- config = arg
- break
- except (configparser.Error, AssertionError):
- pass
-
-# find any egg options set in the config
-crate = eggs.Crate(config)
-for opt in ('enable_eggs', 'enable_egg_fetch', 'try_dependencies_from_env'):
- env = 'GALAXY_CONFIG_' + opt.upper()
- # don't overwrite existing env vars configured by the user
- if env not in os.environ:
- os.environ[env] = str(getattr(crate.galaxy_config, opt))
-
eggs.require( "Paste" )
eggs.require( "PasteDeploy" )
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/45072dc79c5c/
Changeset: 45072dc79c5c
User: natefoo
Date: 2015-02-12 19:27:37+00:00
Summary: Place the new egg config options into the environment at Galaxy startup since
eggs.require() has no access to the config file.
Affected #: 1 file
diff -r c981094a4d50f40e4dae3f0fd49089b46a0e2897 -r 45072dc79c5c8a5d6f1bbc9883e25bb3cb8b6319 scripts/paster.py
--- a/scripts/paster.py
+++ b/scripts/paster.py
@@ -7,6 +7,11 @@
import os, sys
+try:
+ import configparser
+except:
+ import ConfigParser as configparser
+
# ensure supported version
from check_python import check_python
try:
@@ -19,15 +24,44 @@
sys.path = new_path
from galaxy import eggs
-import pkg_resources
if 'LOG_TEMPFILES' in os.environ:
from log_tempfile import TempFile
_log_tempfile = TempFile()
import tempfile
-pkg_resources.require( "Paste" )
-pkg_resources.require( "PasteDeploy" )
+try:
+ serve = sys.argv.index('serve')
+except:
+ print >>sys.stderr, "Galaxy does not use the real Paste Script, the only supported command is 'serve'"
+ sys.exit(1)
+
+# eggs.require() can be called inside the app without access to the Galaxy
+# config, so we need to push the egg options into the environment so they are
+# available to Crate instantiated in require()
+
+# locate the arg containing the path to the config file
+config = None
+p = configparser.ConfigParser()
+for arg in sys.argv:
+ try:
+ p.read(arg)
+ assert 'app:main' in p.sections()
+ config = arg
+ break
+ except (configparser.Error, AssertionError):
+ pass
+
+# find any egg options set in the config
+crate = eggs.Crate(config)
+for opt in ('enable_eggs', 'enable_egg_fetch', 'try_dependencies_from_env'):
+ env = 'GALAXY_CONFIG_' + opt.upper()
+ # don't overwrite existing env vars configured by the user
+ if env not in os.environ:
+ os.environ[env] = str(getattr(crate.galaxy_config, opt))
+
+eggs.require( "Paste" )
+eggs.require( "PasteDeploy" )
from galaxy.util.pastescript import serve
serve.run()
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/82644ba2a73d/
Changeset: 82644ba2a73d
User: natefoo
Date: 2015-02-12 18:18:15+00:00
Summary: So, you can put module source URLs right into a requirements.txt. Thanks John (and thanks for nothing, pip documentation).
Affected #: 1 file
diff -r d86c3cbc6d9016e8511f36a2a1dd48efe4e0fe2c -r 82644ba2a73d5e2b2f07975b39b6e70aefedff9b requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,9 +24,6 @@
# are trying to move away from our strict version dependency policy and this
# would greatly help in our efforts.
-# not available in PyPI, must be installed by hand:
-#SVGFig
-
Paste
PasteDeploy
docutils
@@ -64,3 +61,5 @@
bioblend
Parsley
WebError
+http://svgfig.googlecode.com/files/svgfig-1.1.6.tgz
+
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
3 new commits in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/e89af5d76371/
Changeset: e89af5d76371
User: natefoo
Date: 2015-02-12 17:13:24+00:00
Summary: Make it possible to disable egg fetching or the use of eggs entirely and provide a requirements.txt.
Affected #: 3 files
diff -r 93d2916499f4ddc3093adc65cb5718a25841aa15 -r e89af5d7637192c58d4ebde8986e627a9e07fb24 config/galaxy.ini.sample
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -125,6 +125,34 @@
# but prefixed with install_ are also available).
#install_database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE
+# -- Galaxy framework dependencies (eggs)
+
+# Options in this section can also be provided as environment variables in the
+# form `GALAXY_<OPTION_NAME>`, e.g. `GALAXY_ENABLE_EGGS=False`, etc.
+# Environment variables override the settings in this config.
+
+# For more details, see the `requirements.txt` file at the root of the Galaxy
+# source distribution.
+
+# Galaxy manages all of its dependent Python modules by downloading Python eggs
+# from the Galaxy eggs distribution server at https://eggs.galaxyproject.org.
+# You may choose to disable this and attempt to install dependencies by hand
+# (e.g. with pip), but beware that this may cause problems due to version
+# incompatibilities between Galaxy and its dependencies. Do not do this unless
+# you know what you're doing.
+#enable_eggs = True
+
+# You may choose to keep eggs enabled but disable fetching them automatically.
+# (hint: To do a one-time fetch, you can disable here and then use
+# `GALAXY_ENABLE_EGG_FETCH=True python ./scripts/fetch_eggs.py` from the root
+# of the Galaxy source.
+#enable_egg_fetch = True
+
+# If you'd like to prefer any dependencies found in your local environment
+# before (fetching and) using Galaxy's egg of that dependency, you can enable
+# the following option:
+#try_dependencies_from_env = False
+
# -- Files and directories
# Dataset files are stored in this directory.
diff -r 93d2916499f4ddc3093adc65cb5718a25841aa15 -r e89af5d7637192c58d4ebde8986e627a9e07fb24 lib/galaxy/eggs/__init__.py
--- a/lib/galaxy/eggs/__init__.py
+++ b/lib/galaxy/eggs/__init__.py
@@ -61,6 +61,7 @@
self.distribution = None
self.dir = None
self.removed_location = None
+ self.enable_fetch = crate.galaxy_config.enable_egg_fetch
if self.name is not None and self.version is not None:
self.set_distribution()
@@ -191,7 +192,10 @@
try:
rval = []
# resolve this egg and its dependencies
- dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch )
+ if self.enable_fetch:
+ dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch )
+ else:
+ dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, lambda x: None )
for dist in dists:
# if any of the resolved dependencies should be managed eggs but are being pulled from the wrong path, fix them
if dist.project_name in self.crate.all_names and not os.path.realpath( dist.location ).startswith( os.path.realpath( self.dir ) ):
@@ -250,7 +254,10 @@
if egg:
# Store the removed path so the fetch method can use it
egg.removed_location = location
- r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch )
+ if self.enable_fetch:
+ r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch )
+ else:
+ r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, lambda x: None )
egg.removed_location = None
else:
r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env )
@@ -374,31 +381,48 @@
"""
Try to resolve (e.g. fetch) all eggs in the crate.
"""
- if all:
- eggs = self.all_eggs
+ if self.galaxy_config.enable_eggs and not self.galaxy_config.try_dependencies_from_env:
+ if all:
+ eggs = self.all_eggs
+ else:
+ eggs = self.config_eggs
+ eggs = filter( lambda x: x.name not in self.no_auto, eggs )
+ missing = []
+ for egg in eggs:
+ try:
+ egg.resolve()
+ except EggNotFetchable:
+ missing.append( egg )
+ if missing:
+ raise EggNotFetchable( missing )
else:
- eggs = self.config_eggs
- eggs = filter( lambda x: x.name not in self.no_auto, eggs )
- missing = []
- for egg in eggs:
- try:
- egg.resolve()
- except EggNotFetchable:
- missing.append( egg )
- if missing:
- raise EggNotFetchable( missing )
+ log.info('Dependencies will attempt to be loaded from the environment')
class GalaxyConfig( object ):
always_conditional = ( 'pysam', 'ctypes', 'python_daemon' )
def __init__( self, config_file ):
+ self.enable_egg_fetch = False
+ self.enable_eggs = False
+ self.try_dependencies_from_env = False
if config_file is None:
self.config = None
else:
self.config = ConfigParser.ConfigParser()
if self.config.read( config_file ) == []:
raise Exception( "error: unable to read Galaxy config from %s" % config_file )
+ self.enable_egg_fetch = True
+ self.enable_eggs = True
+ for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'):
+ try:
+ setattr(self, opt, string_as_bool(self.config.get('app:main', opt)))
+ except:
+ pass # use default
+ for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'):
+ var = 'GALAXY_' + opt.upper()
+ if var in os.environ:
+ setattr(self, opt, string_as_bool(os.environ[var]))
def check_conditional( self, egg_name ):
def check_pysam():
@@ -435,6 +459,13 @@
return False
+
+def string_as_bool( string ):
+ if str( string ).lower() in ( 'true', 'yes', 'on' ):
+ return True
+ else:
+ return False
+
def get_env():
env = pkg_resources.Environment( search_path='', platform=pkg_resources.get_platform() )
for dist in pkg_resources.find_distributions( eggs_dir, False ):
@@ -446,6 +477,13 @@
def require( req_str ):
c = Crate( None )
req = pkg_resources.Requirement.parse( req_str )
+ if c.galaxy_config.try_dependencies_from_env or not c.galaxy_config.enable_eggs:
+ try:
+ return pkg_resources.working_set.require( req_str )
+ except Exception as exc:
+ if not c.galaxy_config.enable_eggs:
+ raise
+ log.info("%s not found in local environment, will try Galaxy egg: %s", (req_str, exc))
# TODO: This breaks egg version requirements. Not currently a problem, but
# it could become one.
try:
diff -r 93d2916499f4ddc3093adc65cb5718a25841aa15 -r e89af5d7637192c58d4ebde8986e627a9e07fb24 requirements.txt
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,66 @@
+#
+# Galaxy has traditionally managed all of its own dependencies (in the form of
+# precompiled Python eggs) for a few reasons:
+#
+# 1. We can ensure that Galaxy works with the versions of the dependencies that
+# we require.
+# 2. We can precompile them for numerous platforms, meaning that developers or
+# system administrators attempting to use Galaxy do not need to wait while
+# dependencies compile/install, nor do they need to have compilers or
+# various development libraries installed on their system.
+#
+# However, you may want to manage these dependencies on your own. If so, this
+# file can be used with pip to install all of the bare minimum dependencies for
+# starting a Galaxy server. Use this in conjunction with the
+# `enable_eggs = False` galaxy.ini option or `GALAXY_ENABLE_EGGS=False`
+# environment variable
+#
+# Note that except in the case of SQLAlchemy, all dependencies will be
+# installed at their newest versions. Galaxy has not been tested with these and
+# you may (likely?) run into problems! The Galaxy Team does not support using
+# Galaxy with dependency versions other than those in eggs.ini, however, if you
+# do find an incompatibility between Galaxy and the latest version, we would
+# appreciate if you would investigate and provide a code fix for the issue. We
+# are trying to move away from our strict version dependency policy and this
+# would greatly help in our efforts.
+
+# not available in PyPI, must be installed by hand:
+#SVGFig
+
+Paste
+PasteDeploy
+docutils
+wchartype
+pexpect
+amqp
+repoze.lru
+Routes
+WebOb
+SQLAlchemy<0.8
+Cheetah
+pycrypto
+MarkupSafe
+WebHelpers
+anyjson
+kombu
+numpy
+bx-python # bx must be installed after numpy!
+pysam
+Mako
+PyYAML
+mercurial
+pytz
+Babel
+decorator
+Tempita
+sqlalchemy-migrate
+Beaker
+Whoosh
+paramiko
+ssh
+Fabric
+boto
+simplejson
+bioblend
+Parsley
+WebError
https://bitbucket.org/galaxy/galaxy-central/commits/6b03101f9eb4/
Changeset: 6b03101f9eb4
User: natefoo
Date: 2015-02-12 17:31:58+00:00
Summary: Use GALAXY_CONFIG_ environment variables instead of GALAXY_ to fall in line with convention.
Affected #: 3 files
diff -r e89af5d7637192c58d4ebde8986e627a9e07fb24 -r 6b03101f9eb482477276e0c1b719964f02f51469 config/galaxy.ini.sample
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -128,8 +128,8 @@
# -- Galaxy framework dependencies (eggs)
# Options in this section can also be provided as environment variables in the
-# form `GALAXY_<OPTION_NAME>`, e.g. `GALAXY_ENABLE_EGGS=False`, etc.
-# Environment variables override the settings in this config.
+# form `GALAXY_CONFIG_<OPTION_NAME>`, e.g. `GALAXY_CONFIG_ENABLE_EGGS=False`,
+# etc. Environment variables override the settings in this config.
# For more details, see the `requirements.txt` file at the root of the Galaxy
# source distribution.
@@ -144,8 +144,8 @@
# You may choose to keep eggs enabled but disable fetching them automatically.
# (hint: To do a one-time fetch, you can disable here and then use
-# `GALAXY_ENABLE_EGG_FETCH=True python ./scripts/fetch_eggs.py` from the root
-# of the Galaxy source.
+# `GALAXY_CONFIG_ENABLE_EGG_FETCH=True python ./scripts/fetch_eggs.py` from the
+# root of the Galaxy source.
#enable_egg_fetch = True
# If you'd like to prefer any dependencies found in your local environment
diff -r e89af5d7637192c58d4ebde8986e627a9e07fb24 -r 6b03101f9eb482477276e0c1b719964f02f51469 lib/galaxy/eggs/__init__.py
--- a/lib/galaxy/eggs/__init__.py
+++ b/lib/galaxy/eggs/__init__.py
@@ -420,7 +420,7 @@
except:
pass # use default
for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'):
- var = 'GALAXY_' + opt.upper()
+ var = 'GALAXY_CONFIG_' + opt.upper()
if var in os.environ:
setattr(self, opt, string_as_bool(os.environ[var]))
diff -r e89af5d7637192c58d4ebde8986e627a9e07fb24 -r 6b03101f9eb482477276e0c1b719964f02f51469 requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,7 +12,7 @@
# However, you may want to manage these dependencies on your own. If so, this
# file can be used with pip to install all of the bare minimum dependencies for
# starting a Galaxy server. Use this in conjunction with the
-# `enable_eggs = False` galaxy.ini option or `GALAXY_ENABLE_EGGS=False`
+# `enable_eggs = False` galaxy.ini option or `GALAXY_CONFIG_ENABLE_EGGS=False`
# environment variable
#
# Note that except in the case of SQLAlchemy, all dependencies will be
https://bitbucket.org/galaxy/galaxy-central/commits/b0452677a40f/
Changeset: b0452677a40f
User: natefoo
Date: 2015-02-12 17:39:27+00:00
Summary: Require Whoosh<2.5 as per Martin.
Affected #: 1 file
diff -r 6b03101f9eb482477276e0c1b719964f02f51469 -r b0452677a40f85d4b92cbcb1ebe4e137dbbfbec1 requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -55,7 +55,7 @@
Tempita
sqlalchemy-migrate
Beaker
-Whoosh
+Whoosh<2.5
paramiko
ssh
Fabric
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.