3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8eaabf513c44/ Changeset: 8eaabf513c44 User: jmchilton Date: 2013-12-17 18:19:41 Summary: Reuse code between scripts/manage_tools.py and lib/galaxy/model/orm/scripts.py Affected #: 2 files diff -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa -r 8eaabf513c4494a3990ab93bebc7d16ad9276e3d lib/galaxy/model/orm/scripts.py --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -43,6 +43,31 @@ } +def require_dialect_egg( db_url ): + dialect = ( db_url.split( ':', 1 ) )[0] + try: + egg = dialect_to_egg[dialect] + try: + pkg_resources.require( egg ) + log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module is in the path elsewhere (i.e. non-egg), it'll still load. + log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) + except KeyError: + # Let this go, it could possibly work with db's we don't support + log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + + +def read_config_file_arg( argv, default ): + if '-c' in argv: + pos = argv.index( '-c' ) + argv.pop(pos) + config_file = argv.pop( pos ) + else: + config_file = default + return config_file + + def get_config( argv, cwd=None ): """ Read sys.argv and parse out repository of migrations and database url. @@ -73,12 +98,7 @@ database = 'galaxy' database_defaults = DATABASE[ database ] - if '-c' in argv: - pos = argv.index( '-c' ) - argv.pop(pos) - config_file = argv.pop( pos ) - else: - config_file = database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE ) + config_file = read_config_file_arg( argv, database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE ) ) repo = database_defaults[ 'repo' ] config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX ) default_sqlite_file = database_defaults[ 'default_sqlite_file' ] @@ -95,16 +115,5 @@ else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % default_sqlite_file - dialect = ( db_url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[dialect] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + require_dialect_egg( db_url ) return dict(db_url=db_url, repo=repo, config_file=config_file, database=database) diff -r a2bad69f03f4786309ba80ff305ebdd7b68f93fa -r 8eaabf513c4494a3990ab93bebc7d16ad9276e3d scripts/manage_tools.py --- a/scripts/manage_tools.py +++ b/scripts/manage_tools.py @@ -1,7 +1,9 @@ -import sys, os.path, logging +import sys +import os.path +import logging new_path = [ os.path.join( os.getcwd(), "lib" ) ] -new_path.extend( sys.path[1:] ) # remove scripts/ from the path +new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy import eggs @@ -15,15 +17,12 @@ from migrate.versioning.shell import main from ConfigParser import SafeConfigParser -from galaxy.model.orm import dialect_to_egg +from galaxy.model.orm.scripts import require_dialect_egg +from galaxy.model.orm.scripts import read_config_file_arg log = logging.getLogger( __name__ ) -config_file = 'universe_wsgi.ini' -if '-c' in sys.argv: - pos = sys.argv.index( '-c' ) - sys.argv.pop( pos ) - config_file = sys.argv.pop( pos ) +config_file = read_config_file_arg( sys.argv, 'universe_wsgi.ini' ) if not os.path.exists( config_file ): print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % config_file sys.exit( 1 ) @@ -42,17 +41,6 @@ else: db_url = "sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE" -dialect = ( db_url.split( ':', 1 ) )[0] -try: - egg = dialect_to_egg[dialect] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) -except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) +require_dialect_egg( db_url ) main( repository=repo, url=db_url ) https://bitbucket.org/galaxy/galaxy-central/commits/723431aa61fb/ Changeset: 723431aa61fb User: jmchilton Date: 2013-12-17 18:19:41 Summary: Tools now properly migrated into install_database_connection database... ... this second database is specified. Affected #: 1 file diff -r 8eaabf513c4494a3990ab93bebc7d16ad9276e3d -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 scripts/manage_tools.py --- a/scripts/manage_tools.py +++ b/scripts/manage_tools.py @@ -34,6 +34,8 @@ if config_file == 'universe_wsgi.ini.sample' and 'GALAXY_TEST_DBURI' in os.environ: # Running functional tests. db_url = os.environ[ 'GALAXY_TEST_DBURI' ] +elif cp.has_option( "app:main", "install_database_connection" ): + db_url = cp.get( "app:main", "install_database_connection" ) elif cp.has_option( "app:main", "database_connection" ): db_url = cp.get( "app:main", "database_connection" ) elif cp.has_option( "app:main", "database_file" ): https://bitbucket.org/galaxy/galaxy-central/commits/b1e82182414d/ Changeset: b1e82182414d User: jmchilton Date: 2013-12-17 18:47:48 Summary: Merge latest galaxy-central. Affected #: 7 files diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -69,7 +69,7 @@ default_sort_key = "-update_time" columns = [ HistoryListNameColumn( "Name", key="name", attach_popup=True, filterable="advanced" ), - DatasetsByStateColumn( "Datasets", key="datasets_by_state", sortable=False ), + DatasetsByStateColumn( "Datasets", key="datasets_by_state", sortable=False, nowrap=True), grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.HistoryTagAssociation, \ filterable="advanced", grid_name="HistoryListGrid" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False ), diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe lib/tool_shed/scripts/clean_up_tool_dependency_directory.py --- /dev/null +++ b/lib/tool_shed/scripts/clean_up_tool_dependency_directory.py @@ -0,0 +1,58 @@ +import argparse +import os +import sys +import shutil + +def main( args ): + if not os.path.exists( args.basepath ): + print 'Tool dependency path %s does not exist.' % str( args.basepath ) + return 1 + if args.delete: + print 'Deleting contents of tool dependency path %s.' % args.basepath + for node in os.listdir( args.basepath ): + path = os.path.join( args.basepath, node ) + if os.path.isdir( path ): + try: + shutil.rmtree( path ) + print 'Deleted directory %s and all its contents.' % path + except Exception, e: + print 'Error deleting directory %s: %s' % ( path, str( e ) ) + pass + elif os.path.isfile( path ): + try: + os.remove( path ) + print 'Deleted file %s.' % path + except Exception, e: + print 'Error deleting file %s: %s' % ( path, str( e ) ) + pass + elif os.path.islink( path ): + print 'Deleting symlink %s with target %s.' % ( path, os.path.realpath( path ) ) + try: + os.remove( path ) + except Exception, e: + print 'Error deleting symlink %s: %s' % ( path, str( e ) ) + pass + else: + print 'Tool dependency path %s contains the following files and directories:' % args.basepath + for element in os.listdir( args.basepath ): + print element + return 0 + +if __name__ == '__main__': + description = 'Clean out or list the contents of the provided tool dependency path. Remove if ' + description += 'the --delete command line argument is provided.' + parser = argparse.ArgumentParser( description=description ) + parser.add_argument( '--delete', + dest='delete', + required=False, + action='store_true', + default=False, + help='Whether to delete all folders and files or list them on exit.' ) + parser.add_argument( '--basepath', + dest='basepath', + required=True, + action='store', + metavar='name', + help='The base path where tool dependencies are installed.' ) + args = parser.parse_args() + sys.exit( main( args ) ) diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/base/tool_shed_util.py --- a/test/base/tool_shed_util.py +++ b/test/base/tool_shed_util.py @@ -15,6 +15,9 @@ log = logging.getLogger(__name__) +# Set a 10 minute timeout for repository installation. +repository_installation_timeout = 600 + def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision, tool_path ): """ Return the GALAXY_TEST_FILE_DIR, the containing repository name and the change set revision for the tool elem. diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1,39 +1,38 @@ -import pkg_resources -pkg_resources.require( "twill==0.9" ) - +import difflib +import filecmp +import logging +import os +import re +import pprint +import shutil import StringIO -import os -import filecmp +import subprocess +import tarfile +import tempfile import time import unittest import urllib -import logging -import difflib -import tarfile import zipfile -import tempfile -import re -import shutil -import subprocess -import pprint -import twill -import twill.commands as tc -from twill.other_packages._mechanize_dist import ClientForm -pkg_resources.require( "elementtree" ) -pkg_resources.require( "MarkupSafe" ) -from markupsafe import escape -from elementtree import ElementTree from galaxy.web import security from galaxy.web.framework.helpers import iff from galaxy.util.json import from_json_string from base.asserts import verify_assertions -buffer = StringIO.StringIO() +from galaxy import eggs +eggs.require( "elementtree" ) +eggs.require( 'twill' ) + +from elementtree import ElementTree + +import twill +import twill.commands as tc +from twill.other_packages._mechanize_dist import ClientForm #Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? -twill.set_output(buffer) -tc.config('use_tidy', 0) +buffer = StringIO.StringIO() +twill.set_output( buffer ) +tc.config( 'use_tidy', 0 ) # Dial ClientCookie logging down (very noisy) logging.getLogger( "ClientCookie.cookies" ).setLevel( logging.WARNING ) diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/install_and_test_tool_shed_repositories/base/twilltestcase.py --- a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py +++ b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py @@ -1,9 +1,22 @@ +import logging +import os +import re +import test_db_util +import time +import urllib + import galaxy.model as model import galaxy.model.tool_shed_install as install_model -import common, string, os, re, test_db_util, simplejson, logging, time, sys -import galaxy.util as util -from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib -from tool_shed.util.encoding_util import tool_shed_encode, tool_shed_decode +import galaxy.util + +from galaxy.web import security +from base.twilltestcase import TwillTestCase +from base.tool_shed_util import repository_installation_timeout + +from galaxy import eggs +eggs.require( 'twill' ) + +import twill.commands as tc log = logging.getLogger( __name__ ) @@ -52,7 +65,7 @@ if install_parameters: iri_ids = install_parameters.group(1) # In some cases, the returned iri_ids are of the form: "[u'<encoded id>', u'<encoded id>']" - # This regex ensures that non-hex characters are stripped out of the list, so that util.listify/decode_id + # This regex ensures that non-hex characters are stripped out of the list, so that galaxy.util.listify/decode_id # will handle them correctly. It's safe to pass the cleaned list to manage_repositories, because it can parse # comma-separated values. repository_ids = str( iri_ids ) @@ -60,9 +73,9 @@ encoded_kwd = install_parameters.group(2) reinstalling = install_parameters.group(3) url = '/admin_toolshed/manage_repositories?operation=install&tool_shed_repository_ids=%s&encoded_kwd=%s&reinstalling=%s' % \ - ( ','.join( util.listify( repository_ids ) ), encoded_kwd, reinstalling ) + ( ','.join( galaxy.util.listify( repository_ids ) ), encoded_kwd, reinstalling ) self.visit_url( url ) - return util.listify( repository_ids ) + return galaxy.util.listify( repository_ids ) def install_repository( self, repository_info_dict, install_tool_dependencies=True, install_repository_dependencies=True, strings_displayed=[], strings_not_displayed=[], preview_strings_displayed=[], @@ -161,9 +174,9 @@ timeout_counter = timeout_counter + 1 if timeout_counter % 10 == 0: log.debug( 'Waited %d seconds for repository %s.' % ( timeout_counter, str( galaxy_repository.name ) ) ) - # This timeout currently defaults to 180 seconds, or 3 minutes. - if timeout_counter > common.repository_installation_timeout: - raise AssertionError( 'Repository installation timed out, %d seconds elapsed, repository state is %s.' % \ - ( timeout_counter, repository.status ) ) + # This timeout currently defaults to 10 minutes. + if timeout_counter > repository_installation_timeout: + raise AssertionError( 'Repository installation timed out after %d seconds, repository state is %s.' % \ + ( timeout_counter, repository.status ) ) break time.sleep( 1 ) diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/tool_shed/base/common.py --- a/test/tool_shed/base/common.py +++ b/test/tool_shed/base/common.py @@ -33,6 +33,3 @@ ''' new_repository_dependencies_line = ''' <repository toolshed="${toolshed_url}" name="${repository_name}" owner="${owner}" changeset_revision="${changeset_revision}"${prior_installation_required} />''' - -# Set a 3 minute timeout for repository installation. This should be sufficient, since we're not installing tool dependencies. -repository_installation_timeout = 180 \ No newline at end of file diff -r 723431aa61fb4c7d5cd1f73fe34e16af29368ef4 -r b1e82182414d359f6f43e117e92ec6d6638878fe test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -12,16 +12,27 @@ import urllib import galaxy.webapps.tool_shed.util.hgweb_config import galaxy.model.tool_shed_install as galaxy_model -import galaxy.util as util +import galaxy.util + +from base.tool_shed_util import repository_installation_timeout +from base.twilltestcase import TwillTestCase +from galaxy.util.json import from_json_string +from galaxy.web import security +from tool_shed.util.encoding_util import tool_shed_encode from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util -from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib -from tool_shed.util.encoding_util import tool_shed_encode, tool_shed_decode from galaxy import eggs -eggs.require('mercurial') -from mercurial import hg, ui, commands +eggs.require( 'mercurial' ) +eggs.require( 'twill' ) + from mercurial.util import Abort +from mercurial import commands +from mercurial import hg +from mercurial import ui + +import twill.commands as tc + log = logging.getLogger( __name__ ) @@ -848,7 +859,7 @@ if install_parameters: iri_ids = install_parameters.group(1) # In some cases, the returned iri_ids are of the form: "[u'<encoded id>', u'<encoded id>']" - # This regex ensures that non-hex characters are stripped out of the list, so that util.listify/decode_id + # This regex ensures that non-hex characters are stripped out of the list, so that galaxy.util.listify/decode_id # will handle them correctly. It's safe to pass the cleaned list to manage_repositories, because it can parse # comma-separated values. repository_ids = str( iri_ids ) @@ -856,9 +867,9 @@ encoded_kwd = install_parameters.group(2) reinstalling = install_parameters.group(3) url = '/admin_toolshed/manage_repositories?operation=install&tool_shed_repository_ids=%s&encoded_kwd=%s&reinstalling=%s' % \ - ( ','.join( util.listify( repository_ids ) ), encoded_kwd, reinstalling ) + ( ','.join( galaxy.util.listify( repository_ids ) ), encoded_kwd, reinstalling ) self.visit_galaxy_url( url ) - return util.listify( repository_ids ) + return galaxy.util.listify( repository_ids ) def install_repositories_from_search_results( self, repositories, install_tool_dependencies=False, strings_displayed=[], strings_not_displayed=[], **kwd ): @@ -1327,7 +1338,7 @@ def verify_installed_repository_data_table_entries( self, required_data_table_entries ): # The value of the received required_data_table_entries will be something like: [ 'sam_fa_indexes' ] - data_tables = util.parse_xml( self.shed_tool_data_table_conf ) + data_tables = xml_util.parse_xml( self.shed_tool_data_table_conf ) found = False # With the tool shed, the "path" attribute that is hard-coded into the tool_data_tble_conf.xml # file is ignored. This is because the tool shed requires the directory location to which this @@ -1445,8 +1456,8 @@ while galaxy_repository.status not in final_states: test_db_util.ga_refresh( galaxy_repository ) timeout_counter = timeout_counter + 1 - # This timeout currently defaults to 180 seconds, or 3 minutes. - if timeout_counter > common.repository_installation_timeout: + # This timeout currently defaults to 10 minutes. + if timeout_counter > repository_installation_timeout: raise AssertionError( 'Repository installation timed out, %d seconds elapsed, repository state is %s.' % \ ( timeout_counter, galaxy_repository.status ) ) break Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.