- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 0 new changesets
by commits-noreply＠bitbucket.org 18 Mar '13

18 Mar '13

0 new commits in galaxy-central: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Rename community_wsgi.ini.sample to be tool_shed_wsgi.ini.sample, run_community.sh to run_tool_shed.sh, and fix references to these files in the code.
by commits-noreply＠bitbucket.org 18 Mar '13

18 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/da3aece419ca/ changeset: da3aece419ca user: greg date: 2013-03-18 15:24:03 summary: Rename community_wsgi.ini.sample to be tool_shed_wsgi.ini.sample, run_community.sh to run_tool_shed.sh, and fix references to these files in the code. affected #: 8 files diff -r 6be1fed44bf65402f11d2bd4b576fa76941f1393 -r da3aece419ca2940c4a8a3a059e3bfe25a98098b community_wsgi.ini.sample --- a/community_wsgi.ini.sample +++ /dev/null @@ -1,98 +0,0 @@ -# ---- HTTP Server ---------------------------------------------------------- - -[server:main] - -use = egg:Paste#http -port = 9009 - -# The address on which to listen. By default, only listen to localhost (the tool shed will not -# be accessible over the network). Use '0.0.0.0' to listen on all available network interfaces. -#host = 0.0.0.0 -host = 127.0.0.1 - -use_threadpool = true -threadpool_workers = 10 - -# ---- Galaxy Webapps Community Interface ------------------------------------------------- - -[app:main] - -# Specifies the factory for the universe WSGI application -paste.app_factory = galaxy.webapps.tool_shed.buildapp:app_factory -log_level = DEBUG - -# Database connection -database_file = database/community.sqlite -# You may use a SQLAlchemy connection string to specify an external database instead -#database_connection = postgres:///community_test?host=/var/run/postgresql - -# Where the hgweb.config file is stored. The default is the Galaxy installation directory. -#hgweb_config_dir = None - -# Where dataset files are saved -file_path = database/community_files -# Temporary storage for additional datasets, this should be shared through the cluster -new_file_path = database/tmp - -# Session support (beaker) -use_beaker_session = True -session_type = memory -session_data_dir = %(here)s/database/beaker_sessions -session_key = galaxysessions -session_secret = changethisinproduction - -# -- Users and Security - -# Galaxy encodes various internal values when these values will be output in -# some format (for example, in a URL or cookie). You should set a key to be -# used by the algorithm that encodes and decodes these values. It can be any -# string. If left unchanged, anyone could construct a cookie that would grant -# them access to others' sessions. -id_secret = changethisinproductiontoo - -# User authentication can be delegated to an upstream proxy server (usually -# Apache). The upstream proxy should set a REMOTE_USER header in the request. -# Enabling remote user disables regular logins. For more information, see: -# http://wiki.g2.bx.psu.edu/Admin/Config/Apache%20Proxy -#use_remote_user = False - -# Configuration for debugging middleware -debug = true -use_lint = false - -# NEVER enable this on a public site (even test or QA) -# use_interactive = true - -# this should be a comma-separated list of valid Galaxy users -#admin_users = user1@example.org,user2@example.org - -# Force everyone to log in (disable anonymous access) -require_login = False - -# path to sendmail -sendmail_path = /usr/sbin/sendmail - -# For use by email messages sent from the tool shed -#smtp_server = smtp.your_tool_shed_server -#email_from = your_tool_shed_email@server - -# The URL linked by the "Support" link in the "Help" menu. -#support_url = http://wiki.g2.bx.psu.edu/Support - -# Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!) -## use_heartbeat = True - -# Profiling middleware (cProfile based) -## use_profile = True - -# Use the new iframe / javascript based layout -use_new_layout = true - -# Serving static files (needed if running standalone) -static_enabled = True -static_cache_time = 360 -static_dir = %(here)s/static/ -static_images_dir = %(here)s/static/images -static_favicon_dir = %(here)s/static/favicon.ico -static_scripts_dir = %(here)s/static/scripts/ -static_style_dir = %(here)s/static/june_2007_style/blue diff -r 6be1fed44bf65402f11d2bd4b576fa76941f1393 -r da3aece419ca2940c4a8a3a059e3bfe25a98098b run_community.sh --- a/run_community.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -cd `dirname $0` - -SAMPLES=" - community_wsgi.ini.sample - datatypes_conf.xml.sample - external_service_types_conf.xml.sample - migrated_tools_conf.xml.sample - reports_wsgi.ini.sample - shed_tool_conf.xml.sample - tool_conf.xml.sample - shed_tool_data_table_conf.xml.sample - tool_data_table_conf.xml.sample - tool_sheds_conf.xml.sample - openid_conf.xml.sample - universe_wsgi.ini.sample - tool-data/shared/ncbi/builds.txt.sample - tool-data/shared/ensembl/builds.txt.sample - tool-data/shared/ucsc/builds.txt.sample - tool-data/shared/ucsc/publicbuilds.txt.sample - tool-data/shared/igv/igv_build_sites.txt.sample - tool-data/shared/rviewer/rviewer_build_sites.txt.sample - tool-data/*.sample - static/welcome.html.sample -" - -# Create any missing config/location files -for sample in $SAMPLES; do - file=`echo $sample | sed -e 's/\.sample$//'` - if [ ! -f "$file" -a -f "$sample" ]; then - echo "Initializing $file from `basename $sample`" - cp $sample $file - fi -done - -python ./scripts/paster.py serve community_wsgi.ini --pid-file=community_webapp.pid --log-file=community_webapp.log $@ diff -r 6be1fed44bf65402f11d2bd4b576fa76941f1393 -r da3aece419ca2940c4a8a3a059e3bfe25a98098b run_tool_shed.sh --- /dev/null +++ b/run_tool_shed.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +cd `dirname $0` + +SAMPLES=" + tool_shed_wsgi.ini.sample + datatypes_conf.xml.sample + external_service_types_conf.xml.sample + migrated_tools_conf.xml.sample + reports_wsgi.ini.sample + shed_tool_conf.xml.sample + tool_conf.xml.sample + shed_tool_data_table_conf.xml.sample + tool_data_table_conf.xml.sample + tool_sheds_conf.xml.sample + openid_conf.xml.sample + universe_wsgi.ini.sample + tool-data/shared/ncbi/builds.txt.sample + tool-data/shared/ensembl/builds.txt.sample + tool-data/shared/ucsc/builds.txt.sample + tool-data/shared/ucsc/publicbuilds.txt.sample + tool-data/shared/igv/igv_build_sites.txt.sample + tool-data/shared/rviewer/rviewer_build_sites.txt.sample + tool-data/*.sample + static/welcome.html.sample +" + +# Create any missing config/location files +for sample in $SAMPLES; do + file=`echo $sample | sed -e 's/\.sample$//'` + if [ ! -f "$file" -a -f "$sample" ]; then + echo "Initializing $file from `basename $sample`" + cp $sample $file + fi +done + +python ./scripts/paster.py serve tool_shed_wsgi.ini --pid-file=tool_shed_webapp.pid --log-file=tool_shed_webapp.log $@ diff -r 6be1fed44bf65402f11d2bd4b576fa76941f1393 -r da3aece419ca2940c4a8a3a059e3bfe25a98098b scripts/db_shell.py --- a/scripts/db_shell.py +++ b/scripts/db_shell.py @@ -30,7 +30,7 @@ # Need to pop the last arg so the command line args will be correct # for sqlalchemy-migrate webapp = sys.argv.pop() - config_file = 'community_wsgi.ini' + config_file = 'tool_shed_wsgi.ini' repo = 'lib/galaxy/webapps/tool_shed/model/migrate' else: # Poor man's optparse diff -r 6be1fed44bf65402f11d2bd4b576fa76941f1393 -r da3aece419ca2940c4a8a3a059e3bfe25a98098b scripts/manage_db.py --- a/scripts/manage_db.py +++ b/scripts/manage_db.py @@ -18,7 +18,7 @@ # Need to pop the last arg so the command line args will be correct # for sqlalchemy-migrate webapp = sys.argv.pop() - config_file = 'community_wsgi.ini' + config_file = 'tool_shed_wsgi.ini' repo = 'lib/galaxy/webapps/tool_shed/model/migrate' else: # Poor man's optparse diff -r 6be1fed44bf65402f11d2bd4b576fa76941f1393 -r da3aece419ca2940c4a8a3a059e3bfe25a98098b scripts/tool_shed/migrate_tools_to_repositories.py --- a/scripts/tool_shed/migrate_tools_to_repositories.py +++ b/scripts/tool_shed/migrate_tools_to_repositories.py @@ -10,7 +10,7 @@ 0. This script must be run on a repo updated to changeset: 5621:4618be57481b -1. Before running this script, make sure the following config setting is set in community_wsgi.ini +1. Before running this script, make sure the following config setting is set in tool_shed_wsgi.ini # Enable next-gen tool shed features enable_next_gen_tool_shed = True @@ -262,7 +262,7 @@ print " " print "##########################################" print "%s - Migrating current tool archives to new tool repositories" % now - # community_wsgi.ini file + # tool_shed_wsgi.ini file ini_file = sys.argv[1] conf_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} ) conf_parser.read( ini_file ) diff -r 6be1fed44bf65402f11d2bd4b576fa76941f1393 -r da3aece419ca2940c4a8a3a059e3bfe25a98098b test/tool_shed/functional_tests.py --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -175,7 +175,7 @@ os.environ[ 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file # ---- Build Tool Shed Application -------------------------------------------------- toolshedapp = None - global_conf = { '__file__' : 'community_wsgi.ini.sample' } + global_conf = { '__file__' : 'tool_shed_wsgi.ini.sample' } # if not toolshed_database_connection.startswith( 'sqlite://' ): # kwargs[ 'database_engine_option_max_overflow' ] = '20' if tool_dependency_dir is not None: diff -r 6be1fed44bf65402f11d2bd4b576fa76941f1393 -r da3aece419ca2940c4a8a3a059e3bfe25a98098b tool_shed_wsgi.ini.sample --- /dev/null +++ b/tool_shed_wsgi.ini.sample @@ -0,0 +1,98 @@ +# ---- HTTP Server ---------------------------------------------------------- + +[server:main] + +use = egg:Paste#http +port = 9009 + +# The address on which to listen. By default, only listen to localhost (the tool shed will not +# be accessible over the network). Use '0.0.0.0' to listen on all available network interfaces. +#host = 0.0.0.0 +host = 127.0.0.1 + +use_threadpool = true +threadpool_workers = 10 + +# ---- Galaxy Webapps Community Interface ------------------------------------------------- + +[app:main] + +# Specifies the factory for the universe WSGI application +paste.app_factory = galaxy.webapps.tool_shed.buildapp:app_factory +log_level = DEBUG + +# Database connection +database_file = database/community.sqlite +# You may use a SQLAlchemy connection string to specify an external database instead +#database_connection = postgres:///community_test?host=/var/run/postgresql + +# Where the hgweb.config file is stored. The default is the Galaxy installation directory. +#hgweb_config_dir = None + +# Where dataset files are saved +file_path = database/community_files +# Temporary storage for additional datasets, this should be shared through the cluster +new_file_path = database/tmp + +# Session support (beaker) +use_beaker_session = True +session_type = memory +session_data_dir = %(here)s/database/beaker_sessions +session_key = galaxysessions +session_secret = changethisinproduction + +# -- Users and Security + +# Galaxy encodes various internal values when these values will be output in +# some format (for example, in a URL or cookie). You should set a key to be +# used by the algorithm that encodes and decodes these values. It can be any +# string. If left unchanged, anyone could construct a cookie that would grant +# them access to others' sessions. +id_secret = changethisinproductiontoo + +# User authentication can be delegated to an upstream proxy server (usually +# Apache). The upstream proxy should set a REMOTE_USER header in the request. +# Enabling remote user disables regular logins. For more information, see: +# http://wiki.g2.bx.psu.edu/Admin/Config/Apache%20Proxy +#use_remote_user = False + +# Configuration for debugging middleware +debug = true +use_lint = false + +# NEVER enable this on a public site (even test or QA) +# use_interactive = true + +# this should be a comma-separated list of valid Galaxy users +#admin_users = user1@example.org,user2@example.org + +# Force everyone to log in (disable anonymous access) +require_login = False + +# path to sendmail +sendmail_path = /usr/sbin/sendmail + +# For use by email messages sent from the tool shed +#smtp_server = smtp.your_tool_shed_server +#email_from = your_tool_shed_email@server + +# The URL linked by the "Support" link in the "Help" menu. +#support_url = http://wiki.g2.bx.psu.edu/Support + +# Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!) +## use_heartbeat = True + +# Profiling middleware (cProfile based) +## use_profile = True + +# Use the new iframe / javascript based layout +use_new_layout = true + +# Serving static files (needed if running standalone) +static_enabled = True +static_cache_time = 360 +static_dir = %(here)s/static/ +static_images_dir = %(here)s/static/images +static_favicon_dir = %(here)s/static/favicon.ico +static_scripts_dir = %(here)s/static/scripts/ +static_style_dir = %(here)s/static/june_2007_style/blue Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Import clean up and documentation for compute tool.
by commits-noreply＠bitbucket.org 18 Mar '13

18 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6be1fed44bf6/ changeset: 6be1fed44bf6 user: jgoecks date: 2013-03-18 15:09:43 summary: Import clean up and documentation for compute tool. affected #: 1 file diff -r 0ddc03509b24d4d4e9024902546025df185c6a18 -r 6be1fed44bf65402f11d2bd4b576fa76941f1393 tools/stats/column_maker.py --- a/tools/stats/column_maker.py +++ b/tools/stats/column_maker.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # This tool takes a tab-delimited textfile as input and creates another column in the file which is the result of # a computation performed on every row in the original file. The tool will skip over invalid lines within the file, -# informing the user about the number of lines skipped. -import sys, re, os.path -from galaxy import eggs -from galaxy.tools import validation -from galaxy.datatypes import metadata +# informing the user about the number of lines skipped. + +import sys, re +# These functions may be used in compute expression: from math import log,exp,sqrt,ceil,floor + assert sys.version_info[:2] >= ( 2, 4 ) def stop_err( msg ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Eliminate a duplicate method from shed_util_common and add missing documentation to several utility methods.
by commits-noreply＠bitbucket.org 18 Mar '13

18 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0ddc03509b24/ changeset: 0ddc03509b24 user: greg date: 2013-03-18 14:55:28 summary: Eliminate a duplicate method from shed_util_common and add missing documentation to several utility methods. affected #: 6 files diff -r 92275794f6efe2f2e17c2d3785061e88f2f23f84 -r 0ddc03509b24d4d4e9024902546025df185c6a18 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -184,7 +184,7 @@ params = util.Params( kwd ) repository_id = params.get( 'id', None ) repository = suc.get_installed_tool_shed_repository( trans, repository_id ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) url = suc.url_join( tool_shed_url, 'repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s' % \ ( web.url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) ) @@ -314,7 +314,7 @@ need it so that we can derive the tool shed from which it was installed. """ repository = suc.get_installed_tool_shed_repository( trans, repository_id ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) url = suc.url_join( tool_shed_url, 'repository/get_repository_dependencies?name=%s&owner=%s&changeset_revision=%s' % \ ( repository_name, repository_owner, changeset_revision ) ) @@ -337,7 +337,7 @@ it so that we can derive the tool shed from which it was installed. """ repository = suc.get_installed_tool_shed_repository( trans, repository_id ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) url = suc.url_join( tool_shed_url, 'repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s' % \ ( repository_name, repository_owner, changeset_revision ) ) @@ -359,7 +359,7 @@ uninstalled tool shed repository. """ repository = suc.get_installed_tool_shed_repository( trans, repository_id ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) url = suc.url_join( tool_shed_url, 'repository/get_updated_repository_information?name=%s&owner=%s&changeset_revision=%s' % \ ( repository_name, repository_owner, changeset_revision ) ) @@ -654,7 +654,7 @@ suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, tool_shed_repository.tool_shed ) url = suc.url_join( tool_shed_url, '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) ) @@ -711,7 +711,7 @@ repository = suc.get_installed_tool_shed_repository( trans, repository_id ) if not repository: return trans.show_error_message( 'Invalid repository specified.' ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) if repository.status in [ trans.model.ToolShedRepository.installation_status.CLONING ]: tool_shed_repository_ids = [ repository_id ] return trans.response.send_redirect( web.url_for( controller='admin_toolshed', @@ -1172,7 +1172,7 @@ clone_dir = os.path.join( tool_path, suc.generate_tool_shed_repository_install_dir( repository_clone_url, tool_shed_repository.installed_changeset_revision ) ) relative_install_dir = os.path.join( clone_dir, tool_shed_repository.name ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, tool_shed_repository.tool_shed ) tool_section = None tool_panel_section = kwd.get( 'tool_panel_section', '' ) new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' ) @@ -1317,7 +1317,7 @@ tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) repository_clone_url = suc.generate_clone_url_for_installed_repository( trans.app, tool_shed_repository ) metadata = tool_shed_repository.metadata - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, tool_shed_repository.tool_shed ) tool_path, relative_install_dir = tool_shed_repository.get_tool_relative_path( trans.app ) if latest_changeset_revision and latest_ctx_rev: # There are updates available in the tool shed for the repository, so use the receieved dependency information which was retrieved from @@ -1481,7 +1481,7 @@ def reset_repository_metadata( self, trans, id ): """Reset all metadata on a single installed tool shed repository.""" repository = suc.get_installed_tool_shed_repository( trans, id ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) repository_clone_url = suc.generate_clone_url_for_installed_repository( trans.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: @@ -1560,7 +1560,7 @@ metadata for the repository's revision in the Galaxy database. """ repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) url = suc.url_join( tool_shed_url, 'repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ ( repository.name, repository.owner, repository.changeset_revision ) ) diff -r 92275794f6efe2f2e17c2d3785061e88f2f23f84 -r 0ddc03509b24d4d4e9024902546025df185c6a18 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -279,7 +279,7 @@ if self.__isinstalled( clone_dir ): print "Skipping automatic install of repository '", name, "' because it has already been installed in location ", clone_dir else: - tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed ) + tool_shed_url = suc.get_url_from_tool_shed( self.app, self.tool_shed ) repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name ) relative_install_dir = os.path.join( relative_clone_dir, name ) install_dir = os.path.join( clone_dir, name ) @@ -351,16 +351,6 @@ @property def non_shed_tool_panel_configs( self ): return common_util.get_non_shed_tool_panel_configs( self.app ) - def __get_url_from_tool_shed( self, tool_shed ): - # The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is something like: - # http://toolshed.g2.bx.psu.edu/ - for shed_name, shed_url in self.app.tool_shed_registry.tool_sheds.items(): - if shed_url.find( tool_shed ) >= 0: - if shed_url.endswith( '/' ): - shed_url = shed_url.rstrip( '/' ) - return shed_url - # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. - return None def __isinstalled( self, clone_dir ): full_path = os.path.abspath( clone_dir ) if os.path.exists( full_path ): diff -r 92275794f6efe2f2e17c2d3785061e88f2f23f84 -r 0ddc03509b24d4d4e9024902546025df185c6a18 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -77,7 +77,7 @@ def get_update_to_changeset_revision_and_ctx_rev( trans, repository ): """Return the changeset revision hash to which the repository can be updated.""" changeset_revision_dict = {} - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) url = suc.url_join( tool_shed_url, 'repository/get_changeset_revision_and_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % \ ( repository.name, repository.owner, repository.installed_changeset_revision ) ) try: diff -r 92275794f6efe2f2e17c2d3785061e88f2f23f84 -r 0ddc03509b24d4d4e9024902546025df185c6a18 lib/tool_shed/galaxy_install/update_manager.py --- a/lib/tool_shed/galaxy_install/update_manager.py +++ b/lib/tool_shed/galaxy_install/update_manager.py @@ -34,7 +34,7 @@ self.sleeper.sleep( self.seconds_to_sleep ) log.info( 'Transfer job restarter shutting down...' ) def check_for_update( self, repository ): - tool_shed_url = suc.get_url_from_repository_tool_shed( self.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( self.app, repository.tool_shed ) url = '%s/repository/check_for_updates?name=%s&owner=%s&changeset_revision=%s&from_update_manager=True' % \ ( tool_shed_url, repository.name, repository.owner, repository.changeset_revision ) try: diff -r 92275794f6efe2f2e17c2d3785061e88f2f23f84 -r 0ddc03509b24d4d4e9024902546025df185c6a18 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1432,7 +1432,7 @@ def reset_all_metadata_on_installed_repository( trans, id ): """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" repository = suc.get_installed_tool_shed_repository( trans, id ) - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) repository_clone_url = suc.generate_clone_url_for_installed_repository( trans.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: diff -r 92275794f6efe2f2e17c2d3785061e88f2f23f84 -r 0ddc03509b24d4d4e9024902546025df185c6a18 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -160,7 +160,7 @@ return False, error_message def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): - # Persist the current in-memory list of config_elems to a file named by the value of config_filename. + """Persist the current in-memory list of config_elems to a file named by the value of config_filename.""" fd, filename = tempfile.mkstemp() os.write( fd, '<?xml version="1.0"?>\n' ) os.write( fd, '<toolbox tool_path="%s">\n' % str( tool_path ) ) @@ -242,7 +242,7 @@ def generate_clone_url_for_installed_repository( app, repository ): """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" - tool_shed_url = get_url_from_repository_tool_shed( app, repository ) + tool_shed_url = get_url_from_tool_shed( app, repository.tool_shed ) return url_join( tool_shed_url, 'repos', repository.owner, repository.name ) def generate_clone_url_for_repository_in_tool_shed( trans, repository ): @@ -469,7 +469,7 @@ return None def get_installed_tool_shed_repository( trans, id ): - """Get a repository on the Galaxy side from the database via id""" + """Get a tool shed repository record from the Galaxy database defined by the id.""" return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) def get_named_tmpfile_from_ctx( ctx, filename, dir ): @@ -516,6 +516,11 @@ return None def get_or_create_tool_shed_repository( trans, tool_shed, name, owner, changeset_revision ): + """ + Return a tool shed repository database record defined by the combination of tool shed, repository name, repository owner and changeset_revision + or installed_changeset_revision. A new tool shed repository record will be created if one is not located. + """ + # This method is used only in Galaxy, not the tool shed. repository = get_repository_for_dependency_relationship( trans.app, tool_shed, name, owner, changeset_revision ) if not repository: tool_shed_url = get_url_from_tool_shed( trans.app, tool_shed ) @@ -572,7 +577,7 @@ previous_changeset_revision = changeset_revision def get_repo_info_tuple_contents( repo_info_tuple ): - # Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced. + """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple repository_dependencies = None @@ -613,6 +618,8 @@ return None def get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ): + """Return a tool shed repository database record that is defined by either the current changeset revision or the installed_changeset_revision.""" + # This method is used only in Galaxy, not the tool shed. repository = get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( app=app, tool_shed=tool_shed, name=name, @@ -627,6 +634,7 @@ return repository def get_repository_file_contents( file_path ): + """Return the display-safe contents of a repository file.""" if checkers.is_gzip( file_path ): safe_str = to_safe_string( '\ngzip compressed file\n' ) elif checkers.is_bz2( file_path ): @@ -646,6 +654,7 @@ return safe_str def get_repository_files( trans, folder_path ): + """Return the file hierarchy of a tool shed repository.""" contents = [] for item in os.listdir( folder_path ): # Skip .hg directories @@ -684,6 +693,7 @@ return None def get_repository_owner( cleaned_repository_url ): + """Gvien a "cleaned" repository clone URL, return the owner of the repository.""" items = cleaned_repository_url.split( '/repos/' ) repo_path = items[ 1 ] if repo_path.startswith( '/' ): @@ -691,11 +701,13 @@ return repo_path.lstrip( '/' ).split( '/' )[ 0 ] def get_repository_owner_from_clone_url( repository_clone_url ): + """Given a repository clone URL, return the owner of the repository.""" tmp_url = clean_repository_clone_url( repository_clone_url ) tool_shed = tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) return get_repository_owner( tmp_url ) def get_repository_tools_tups( app, metadata_dict ): + """Return a list of tuples of the form (relative_path, guid, tool) for each tool defined in the received tool shed repository metadata.""" repository_tools_tups = [] index, shed_conf_dict = get_shed_tool_conf_dict( app, metadata_dict.get( 'shed_config_filename' ) ) if 'tools' in metadata_dict: @@ -713,6 +725,7 @@ return repository_tools_tups def get_reversed_changelog_changesets( repo ): + """Return a list of changesets in reverse order from that provided by the repository manifest.""" reversed_changelog = [] for changeset in repo.changelog: reversed_changelog.insert( 0, changeset ) @@ -728,6 +741,7 @@ return "-1:%s" % changeset_revision def get_rev_label_from_changeset_revision( repo, changeset_revision ): + """Given a changeset revision hash, return two strings, the changeset rev and the changeset revision hash.""" ctx = get_changectx_for_changeset( repo, changeset_revision ) if ctx: rev = '%04d' % ctx.rev() @@ -748,8 +762,10 @@ return index, shed_tool_conf_dict def get_tool_panel_config_tool_path_install_dir( app, repository ): - # Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the - # repository is installed. This method assumes all repository tools are defined in a single shed-related tool panel config. + """ + Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the repository is installed. + This method assumes all repository tools are defined in a single shed-related tool panel config. + """ tool_shed = clean_tool_shed_url( repository.tool_shed ) partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) # Get the relative tool installation paths from each of the shed tool configs. @@ -782,11 +798,14 @@ return None def get_tool_shed_repository_by_id( trans, repository_id ): + """Return a tool shed repository database record defined by the id.""" + # This method is used only in Galaxy, not the tool shed. return trans.sa_session.query( trans.model.ToolShedRepository ) \ .filter( trans.model.ToolShedRepository.table.c.id == trans.security.decode_id( repository_id ) ) \ .first() def get_tool_shed_repository_by_shed_name_owner_changeset_revision( app, tool_shed, name, owner, changeset_revision ): + """Return a tool shed repository database record defined by the combination of a tool_shed, repository name, repository owner and current changeet_revision.""" # This method is used only in Galaxy, not the tool shed. sa_session = app.model.context.current if tool_shed.find( '//' ) > 0: @@ -800,6 +819,7 @@ .first() def get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( app, tool_shed, name, owner, installed_changeset_revision ): + """Return a tool shed repository database record defined by the combination of a tool_shed, repository name, repository owner and installed_changeet_revision.""" # This method is used only in Galaxy, not the tool shed. sa_session = app.model.context.current if tool_shed.find( '//' ) > 0: @@ -816,23 +836,12 @@ tmp_url = clean_repository_clone_url( repository_clone_url ) return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) -def get_url_from_repository_tool_shed( app, repository ): +def get_url_from_tool_shed( app, tool_shed ): """ - The stored value of repository.tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is - something like: http://toolshed.g2.bx.psu.edu/. + The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is something like: + http://toolshed.g2.bx.psu.edu/ """ for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items(): - if shed_url.find( repository.tool_shed ) >= 0: - if shed_url.endswith( '/' ): - shed_url = shed_url.rstrip( '/' ) - return shed_url - # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. - return None - -def get_url_from_tool_shed( app, tool_shed ): - # The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is something like: - # http://toolshed.g2.bx.psu.edu/ - for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items(): if shed_url.find( tool_shed ) >= 0: if shed_url.endswith( '/' ): shed_url = shed_url.rstrip( '/' ) @@ -856,26 +865,28 @@ return None def handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=False, admin_only=False ): - # There are 2 complementary features that enable a tool shed user to receive email notification: - # 1. Within User Preferences, they can elect to receive email when the first (or first valid) - # change set is produced for a new repository. - # 2. When viewing or managing a repository, they can check the box labeled "Receive email alerts" - # which caused them to receive email alerts when updates to the repository occur. This same feature - # is available on a per-repository basis on the repository grid within the tool shed. - # - # There are currently 4 scenarios for sending email notification when a change is made to a repository: - # 1. An admin user elects to receive email when the first change set is produced for a new repository - # from User Preferences. The change set does not have to include any valid content. This allows for - # the capture of inappropriate content being uploaded to new repositories. - # 2. A regular user elects to receive email when the first valid change set is produced for a new repository - # from User Preferences. This differs from 1 above in that the user will not receive email until a - # change set tha tincludes valid content is produced. - # 3. An admin user checks the "Receive email alerts" check box on the manage repository page. Since the - # user is an admin user, the email will include information about both HTML and image content that was - # included in the change set. - # 4. A regular user checks the "Receive email alerts" check box on the manage repository page. Since the - # user is not an admin user, the email will not include any information about both HTML and image content - # that was included in the change set. + """ + There are 2 complementary features that enable a tool shed user to receive email notification: + 1. Within User Preferences, they can elect to receive email when the first (or first valid) + change set is produced for a new repository. + 2. When viewing or managing a repository, they can check the box labeled "Receive email alerts" + which caused them to receive email alerts when updates to the repository occur. This same feature + is available on a per-repository basis on the repository grid within the tool shed. + + There are currently 4 scenarios for sending email notification when a change is made to a repository: + 1. An admin user elects to receive email when the first change set is produced for a new repository + from User Preferences. The change set does not have to include any valid content. This allows for + the capture of inappropriate content being uploaded to new repositories. + 2. A regular user elects to receive email when the first valid change set is produced for a new repository + from User Preferences. This differs from 1 above in that the user will not receive email until a + change set tha tincludes valid content is produced. + 3. An admin user checks the "Receive email alerts" check box on the manage repository page. Since the + user is an admin user, the email will include information about both HTML and image content that was + included in the change set. + 4. A regular user checks the "Receive email alerts" check box on the manage repository page. Since the + user is not an admin user, the email will not include any information about both HTML and image content + that was included in the change set. + """ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( get_configured_ui(), repo_dir ) sharable_link = generate_sharable_link_for_repository_in_tool_shed( trans, repository, changeset_revision=None ) @@ -948,6 +959,7 @@ log.exception( "An error occurred sending a tool shed repository update alert by email." ) def open_repository_files_folder( trans, folder_path ): + """Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy.""" try: files_list = get_repository_files( trans, folder_path ) except OSError, e: @@ -957,19 +969,20 @@ folder_contents = [] for filename in files_list: is_folder = False - if filename and filename[-1] == os.sep: + if filename and filename[ -1 ] == os.sep: is_folder = True if filename: full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } + node = { "title" : filename, + "isFolder" : is_folder, + "isLazy" : is_folder, + "tooltip" : full_path, + "key" : full_path } folder_contents.append( node ) return folder_contents def remove_dir( dir ): + """Attempt to remove a directory from disk.""" if os.path.exists( dir ): try: shutil.rmtree( dir ) @@ -1029,9 +1042,11 @@ return reversed_changelog def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revision ): + """Return a reversed list of changesets in the repository changelog up to and including the included_upper_bounds_changeset_revision.""" return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) def strip_path( fpath ): + """Attempt to strip the path from a file name.""" if not fpath: return fpath try: @@ -1072,12 +1087,15 @@ return text def tool_shed_from_repository_clone_url( repository_clone_url ): + """Given a repository clone URL, return the tool shed that contains the repository.""" return clean_repository_clone_url( repository_clone_url ).split( '/repos/' )[ 0 ].rstrip( '/' ) def tool_shed_is_this_tool_shed( toolshed_base_url ): + """Determine if a tool shed is the current tool shed.""" return toolshed_base_url.rstrip( '/' ) == str( url_for( '/', qualified=True ) ).rstrip( '/' ) def translate_string( raw_text, to_html=True ): + """Return a subset of a string (up to MAX_CONTENT_SIZE) translated to a safe string for display in a browser.""" if raw_text: if len( raw_text ) <= MAX_CONTENT_SIZE: translated_string = to_safe_string( raw_text, to_html=to_html ) @@ -1141,12 +1159,14 @@ commands.update( get_configured_ui(), repo, rev=ctx_rev ) def update_tool_shed_repository_status( app, tool_shed_repository, status ): + """Update the status of a tool shed repository in the process of being installed into Galaxy.""" sa_session = app.model.context.current tool_shed_repository.status = status sa_session.add( tool_shed_repository ) sa_session.flush() def url_join( *args ): + """Return a valid URL produced by appending a base URL and a set of request parameters.""" parts = [] for arg in args: parts.append( arg.strip( '/' ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 18 Mar '13

18 Mar '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8070d6f638fe/ changeset: 8070d6f638fe user: jgoecks date: 2013-03-17 15:58:43 summary: Import and documentation fixes for web helpers. affected #: 1 file diff -r 1aa6224b32c2e1c05eafbf1f507d2f7a754ded89 -r 8070d6f638fea37aff3ac3e081b7bd9fba0402b7 lib/galaxy/web/framework/helpers/__init__.py --- a/lib/galaxy/web/framework/helpers/__init__.py +++ b/lib/galaxy/web/framework/helpers/__init__.py @@ -1,7 +1,7 @@ import pkg_resources pkg_resources.require( "WebHelpers" ) -from webhelpers import * +from webhelpers import date, stylesheet_link_tag, javascript_include_tag, url_for from galaxy.util.json import to_json_string from galaxy.util import hash_util @@ -12,10 +12,13 @@ server_starttime = int(time.time()) -# If the date is more than one week ago, then display the actual date instead of in words def time_ago( x ): + """ + Convert a datetime to a string. + """ delta = timedelta(weeks=1) + # If the date is more than one week ago, then display the actual date instead of in words if (datetime.utcnow() - x) > delta: # Greater than a week difference return x.strftime("%b %d, %Y") else: @@ -27,8 +30,10 @@ else: return c -# Smart string truncation def truncate(content, length=100, suffix='...'): + """ + Smart string truncation + """ if len(content) <= length: return content else: https://bitbucket.org/galaxy/galaxy-central/commits/92275794f6ef/ changeset: 92275794f6ef user: jgoecks date: 2013-03-17 16:27:36 summary: Add method to encode ids in an dictionary and use to encode dataset id in tabular display. affected #: 2 files diff -r 8070d6f638fea37aff3ac3e081b7bd9fba0402b7 -r 92275794f6efe2f2e17c2d3785061e88f2f23f84 lib/galaxy/web/security/__init__.py --- a/lib/galaxy/web/security/__init__.py +++ b/lib/galaxy/web/security/__init__.py @@ -30,10 +30,13 @@ random_pool.stir() return str( number.getRandomNumber( nbits, random_pool.get_bytes ) ) + class SecurityHelper( object ): + def __init__( self, **config ): self.id_secret = config['id_secret'] self.id_cipher = Blowfish.new( self.id_secret ) + def encode_id( self, obj_id ): # Convert to string s = str( obj_id ) @@ -41,17 +44,32 @@ s = ( "!" * ( 8 - len(s) % 8 ) ) + s # Encrypt return self.id_cipher.encrypt( s ).encode( 'hex' ) + + def encode_dict_ids( self, a_dict ): + """ + Encode all ids in dictionary. Ids are identified by (a) an 'id' key or + (b) a key that ends with '_id' + """ + for key, val in a_dict.items(): + if key == 'id' or key.endswith('_id'): + a_dict[ key ] = self.encode_id( val ) + + return a_dict + def decode_id( self, obj_id ): return int( self.id_cipher.decrypt( obj_id.decode( 'hex' ) ).lstrip( "!" ) ) + def encode_guid( self, session_key ): # Session keys are strings # Pad to a multiple of 8 with leading "!" s = ( "!" * ( 8 - len( session_key ) % 8 ) ) + session_key # Encrypt return self.id_cipher.encrypt( s ).encode( 'hex' ) + def decode_guid( self, session_key ): # Session keys are strings return self.id_cipher.decrypt( session_key.decode( 'hex' ) ).lstrip( "!" ) + def get_new_guid( self ): # Generate a unique, high entropy 128 bit random number return get_random_bytes( 16 ) diff -r 8070d6f638fea37aff3ac3e081b7bd9fba0402b7 -r 92275794f6efe2f2e17c2d3785061e88f2f23f84 templates/webapps/galaxy/dataset/tabular_chunked.mako --- a/templates/webapps/galaxy/dataset/tabular_chunked.mako +++ b/templates/webapps/galaxy/dataset/tabular_chunked.mako @@ -17,8 +17,7 @@ require(['mvc/data'], function(data) { data.createTabularDatasetChunkedView( - // Dataset config. TODO: encode id. - _.extend( ${h.to_json_string( dataset.get_api_value() )}, + _.extend( ${h.to_json_string( trans.security.encode_dict_ids( dataset.get_api_value() ) )}, { chunk_url: "${h.url_for( controller='/dataset', action='display', dataset_id=trans.security.encode_id( dataset.id ))}", Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Import fix for admin grids in the tool shed.
by commits-noreply＠bitbucket.org 16 Mar '13

16 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1aa6224b32c2/ changeset: 1aa6224b32c2 user: greg date: 2013-03-16 20:18:16 summary: Import fix for admin grids in the tool shed. affected #: 1 file diff -r 4c7d89fe3fc9e9e4e8cd69aa0ff1504b842d668a -r 1aa6224b32c2e1c05eafbf1f507d2f7a754ded89 lib/tool_shed/grids/admin_grids.py --- a/lib/tool_shed/grids/admin_grids.py +++ b/lib/tool_shed/grids/admin_grids.py @@ -5,6 +5,12 @@ import tool_shed.util.shed_util_common as suc from tool_shed.grids.repository_grids import RepositoryGrid, CategoryGrid +from galaxy import eggs +import pkg_resources + +pkg_resources.require( 'mercurial' ) +from mercurial import hg, ui, commands + log = logging.getLogger( __name__ ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for viewing tools in a repository.
by commits-noreply＠bitbucket.org 16 Mar '13

16 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4c7d89fe3fc9/ changeset: 4c7d89fe3fc9 user: greg date: 2013-03-16 20:08:14 summary: Fix for viewing tools in a repository. affected #: 1 file diff -r a5c906f0e8a03b9ec884b224393c5cb767dcf37d -r 4c7d89fe3fc9e9e4e8cd69aa0ff1504b842d668a lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -535,7 +535,7 @@ def handle_sample_files_and_load_tool_from_tmp_config( trans, repo, changeset_revision, tool_config_filename, work_dir ): tool = None message = '' - ctx = get_changectx_for_changeset( repo, changeset_revision ) + ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) # We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in # the manifest, but have been deleted from disk. sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) @@ -548,7 +548,7 @@ error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) if error: log.debug( message ) - manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) + manifest_ctx, ctx_file = suc.get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) if manifest_ctx and ctx_file: tool, message2 = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir ) message = concat_messages( message, message2 ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Relocate the tool shed's workflow utility component.
by commits-noreply＠bitbucket.org 16 Mar '13

16 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a5c906f0e8a0/ changeset: a5c906f0e8a0 user: greg date: 2013-03-16 17:43:20 summary: Relocate the tool shed's workflow utility component. affected #: 4 files diff -r 3f2e8f5f0d7de7131aa878e867c12c352f1f79b5 -r a5c906f0e8a03b9ec884b224393c5cb767dcf37d lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -7,9 +7,8 @@ from galaxy.model.orm import or_ import tool_shed.util.shed_util_common as suc from tool_shed.util import common_install_util, data_manager_util, datatype_util, encoding_util, metadata_util -from tool_shed.util import readme_util, repository_dependency_util, tool_dependency_util, tool_util +from tool_shed.util import readme_util, repository_dependency_util, tool_dependency_util, tool_util, workflow_util from tool_shed.galaxy_install import repository_util -from galaxy.webapps.tool_shed.util import workflow_util import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids import pkg_resources diff -r 3f2e8f5f0d7de7131aa878e867c12c352f1f79b5 -r a5c906f0e8a03b9ec884b224393c5cb767dcf37d lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -11,9 +11,9 @@ from galaxy.util import json from galaxy.model.orm import and_, or_ import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, metadata_util, readme_util, repository_dependency_util, review_util, tool_dependency_util, tool_util +from tool_shed.util import encoding_util, metadata_util, readme_util, repository_dependency_util, review_util, tool_dependency_util, tool_util, workflow_util from tool_shed.galaxy_install import repository_util -from galaxy.webapps.tool_shed.util import common_util, container_util, workflow_util +from galaxy.webapps.tool_shed.util import common_util, container_util import galaxy.tools import tool_shed.grids.repository_grids as repository_grids import tool_shed.grids.util as grids_util diff -r 3f2e8f5f0d7de7131aa878e867c12c352f1f79b5 -r a5c906f0e8a03b9ec884b224393c5cb767dcf37d lib/galaxy/webapps/tool_shed/util/workflow_util.py --- a/lib/galaxy/webapps/tool_shed/util/workflow_util.py +++ /dev/null @@ -1,411 +0,0 @@ -from galaxy import eggs -import pkg_resources - -pkg_resources.require( "SVGFig" ) - -import logging, svgfig -from galaxy.util import json -import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, metadata_util, tool_util -from galaxy.workflow.modules import InputDataModule, ToolModule, WorkflowModuleFactory -import galaxy.webapps.galaxy.controllers.workflow -import galaxy.tools -import galaxy.tools.parameters - -log = logging.getLogger( __name__ ) - -class RepoInputDataModule( InputDataModule ): - - type = "data_input" - name = "Input dataset" - - @classmethod - def new( Class, trans, tools_metadata=None, tool_id=None ): - module = Class( trans ) - module.state = dict( name="Input Dataset" ) - return module - @classmethod - def from_dict( Class, trans, repository_id, changeset_revision, step_dict, tools_metadata=None, secure=True ): - module = Class( trans ) - state = json.from_json_string( step_dict[ "tool_state" ] ) - module.state = dict( name=state.get( "name", "Input Dataset" ) ) - return module - @classmethod - def from_workflow_step( Class, trans, repository_id, changeset_revision, tools_metadata, step ): - module = Class( trans ) - module.state = dict( name="Input Dataset" ) - if step.tool_inputs and "name" in step.tool_inputs: - module.state[ 'name' ] = step.tool_inputs[ 'name' ] - return module - -class RepoToolModule( ToolModule ): - - type = "tool" - - def __init__( self, trans, repository_id, changeset_revision, tools_metadata, tool_id ): - self.trans = trans - self.tools_metadata = tools_metadata - self.tool_id = tool_id - self.tool = None - self.errors = None - for tool_dict in tools_metadata: - if self.tool_id in [ tool_dict[ 'id' ], tool_dict[ 'guid' ] ]: - if trans.webapp.name == 'tool_shed': - # We're in the tool shed. - repository, self.tool, message = tool_util.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_dict[ 'tool_config' ] ) - if message and self.tool is None: - self.errors = 'unavailable' - break - else: - # We're in Galaxy. - self.tool = trans.app.toolbox.tools_by_id.get( self.tool_id, None ) - if self.tool is None: - self.errors = 'unavailable' - self.post_job_actions = {} - self.workflow_outputs = [] - self.state = None - @classmethod - def new( Class, trans, repository_id, changeset_revision, tools_metadata, tool_id=None ): - module = Class( trans, repository_id, changeset_revision, tools_metadata, tool_id ) - module.state = module.tool.new_state( trans, all_pages=True ) - return module - @classmethod - def from_dict( Class, trans, repository_id, changeset_revision, step_dict, tools_metadata, secure=True ): - tool_id = step_dict[ 'tool_id' ] - module = Class( trans, repository_id, changeset_revision, tools_metadata, tool_id ) - module.state = galaxy.tools.DefaultToolState() - if module.tool is not None: - module.state.decode( step_dict[ "tool_state" ], module.tool, module.trans.app, secure=secure ) - module.errors = step_dict.get( "tool_errors", None ) - return module - @classmethod - def from_workflow_step( Class, trans, repository_id, changeset_revision, tools_metadata, step ): - module = Class( trans, repository_id, changeset_revision, tools_metadata, step.tool_id ) - module.state = galaxy.tools.DefaultToolState() - if module.tool: - module.state.inputs = module.tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True ) - else: - module.state.inputs = {} - module.errors = step.tool_errors - return module - def get_data_inputs( self ): - data_inputs = [] - def callback( input, value, prefixed_name, prefixed_label ): - if isinstance( input, galaxy.tools.parameters.DataToolParameter ): - data_inputs.append( dict( name=prefixed_name, - label=prefixed_label, - extensions=input.extensions ) ) - if self.tool: - galaxy.tools.parameters.visit_input_values( self.tool.inputs, self.state.inputs, callback ) - return data_inputs - def get_data_outputs( self ): - data_outputs = [] - if self.tool: - data_inputs = None - for name, tool_output in self.tool.outputs.iteritems(): - if tool_output.format_source != None: - # Default to special name "input" which remove restrictions on connections - formats = [ 'input' ] - if data_inputs == None: - data_inputs = self.get_data_inputs() - # Find the input parameter referenced by format_source - for di in data_inputs: - # Input names come prefixed with conditional and repeat names separated by '|', - # so remove prefixes when comparing with format_source. - if di[ 'name' ] != None and di[ 'name' ].split( '|' )[ -1 ] == tool_output.format_source: - formats = di[ 'extensions' ] - else: - formats = [ tool_output.format ] - for change_elem in tool_output.change_format: - for when_elem in change_elem.findall( 'when' ): - format = when_elem.get( 'format', None ) - if format and format not in formats: - formats.append( format ) - data_outputs.append( dict( name=name, extensions=formats ) ) - return data_outputs - -class RepoWorkflowModuleFactory( WorkflowModuleFactory ): - def __init__( self, module_types ): - self.module_types = module_types - def new( self, trans, type, tools_metadata=None, tool_id=None ): - """Return module for type and (optional) tool_id initialized with new / default state.""" - assert type in self.module_types - return self.module_types[type].new( trans, tool_id ) - def from_dict( self, trans, repository_id, changeset_revision, step_dict, **kwd ): - """Return module initialized from the data in dictionary `step_dict`.""" - type = step_dict[ 'type' ] - assert type in self.module_types - return self.module_types[ type ].from_dict( trans, repository_id, changeset_revision, step_dict, **kwd ) - def from_workflow_step( self, trans, repository_id, changeset_revision, tools_metadata, step ): - """Return module initialized from the WorkflowStep object `step`.""" - type = step.type - return self.module_types[ type ].from_workflow_step( trans, repository_id, changeset_revision, tools_metadata, step ) - -module_factory = RepoWorkflowModuleFactory( dict( data_input=RepoInputDataModule, tool=RepoToolModule ) ) - -def generate_workflow_image( trans, workflow_name, repository_metadata_id=None, repository_id=None ): - """ - Return an svg image representation of a workflow dictionary created when the workflow was exported. This method is called - from both Galaxy and the tool shed. When called from the tool shed, repository_metadata_id will have a value and repository_id - will be None. When called from Galaxy, repository_metadata_id will be None and repository_id will have a value. - """ - workflow_name = encoding_util.tool_shed_decode( workflow_name ) - if trans.webapp.name == 'tool_shed': - # We're in the tool shed. - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) - repository_id = trans.security.encode_id( repository_metadata.repository_id ) - changeset_revision = repository_metadata.changeset_revision - metadata = repository_metadata.metadata - else: - # We're in Galaxy. - repository = suc.get_tool_shed_repository_by_id( trans, repository_id ) - changeset_revision = repository.changeset_revision - metadata = repository.metadata - # metadata[ 'workflows' ] is a list of tuples where each contained tuple is - # [ <relative path to the .ga file in the repository>, <exported workflow dict> ] - for workflow_tup in metadata[ 'workflows' ]: - workflow_dict = workflow_tup[1] - if workflow_dict[ 'name' ] == workflow_name: - break - if 'tools' in metadata: - tools_metadata = metadata[ 'tools' ] - else: - tools_metadata = [] - workflow, missing_tool_tups = get_workflow_from_dict( trans=trans, - workflow_dict=workflow_dict, - tools_metadata=tools_metadata, - repository_id=repository_id, - changeset_revision=changeset_revision ) - data = [] - canvas = svgfig.canvas( style="stroke:black; fill:none; stroke-width:1px; stroke-linejoin:round; text-anchor:left" ) - text = svgfig.SVG( "g" ) - connectors = svgfig.SVG( "g" ) - boxes = svgfig.SVG( "g" ) - svgfig.Text.defaults[ "font-size" ] = "10px" - in_pos = {} - out_pos = {} - margin = 5 - # Spacing between input/outputs. - line_px = 16 - # Store px width for boxes of each step. - widths = {} - max_width, max_x, max_y = 0, 0, 0 - for step in workflow.steps: - step.upgrade_messages = {} - module = module_factory.from_workflow_step( trans, repository_id, changeset_revision, tools_metadata, step ) - tool_errors = module.type == 'tool' and not module.tool - module_data_inputs = get_workflow_data_inputs( step, module ) - module_data_outputs = get_workflow_data_outputs( step, module, workflow.steps ) - step_dict = { - 'id' : step.order_index, - 'data_inputs' : module_data_inputs, - 'data_outputs' : module_data_outputs, - 'position' : step.position, - 'tool_errors' : tool_errors - } - input_conn_dict = {} - for conn in step.input_connections: - input_conn_dict[ conn.input_name ] = dict( id=conn.output_step.order_index, output_name=conn.output_name ) - step_dict[ 'input_connections' ] = input_conn_dict - data.append( step_dict ) - x, y = step.position[ 'left' ], step.position[ 'top' ] - count = 0 - module_name = get_workflow_module_name( module, missing_tool_tups ) - max_len = len( module_name ) * 1.5 - text.append( svgfig.Text( x, y + 20, module_name, **{ "font-size": "14px" } ).SVG() ) - y += 45 - for di in module_data_inputs: - cur_y = y + count * line_px - if step.order_index not in in_pos: - in_pos[ step.order_index ] = {} - in_pos[ step.order_index ][ di[ 'name' ] ] = ( x, cur_y ) - text.append( svgfig.Text( x, cur_y, di[ 'label' ] ).SVG() ) - count += 1 - max_len = max( max_len, len( di[ 'label' ] ) ) - if len( module.get_data_inputs() ) > 0: - y += 15 - for do in module_data_outputs: - cur_y = y + count * line_px - if step.order_index not in out_pos: - out_pos[ step.order_index ] = {} - out_pos[ step.order_index ][ do[ 'name' ] ] = ( x, cur_y ) - text.append( svgfig.Text( x, cur_y, do[ 'name' ] ).SVG() ) - count += 1 - max_len = max( max_len, len( do['name' ] ) ) - widths[ step.order_index ] = max_len * 5.5 - max_x = max( max_x, step.position[ 'left' ] ) - max_y = max( max_y, step.position[ 'top' ] ) - max_width = max( max_width, widths[ step.order_index ] ) - for step_dict in data: - tool_unavailable = step_dict[ 'tool_errors' ] - width = widths[ step_dict[ 'id' ] ] - x, y = step_dict[ 'position' ][ 'left' ], step_dict[ 'position' ][ 'top' ] - # Only highlight missing tools if displaying in the tool shed. - if trans.webapp.name == 'tool_shed' and tool_unavailable: - fill = "#EBBCB2" - else: - fill = "#EBD9B2" - boxes.append( svgfig.Rect( x - margin, y, x + width - margin, y + 30, fill=fill ).SVG() ) - box_height = ( len( step_dict[ 'data_inputs' ] ) + len( step_dict[ 'data_outputs' ] ) ) * line_px + margin - # Draw separator line. - if len( step_dict[ 'data_inputs' ] ) > 0: - box_height += 15 - sep_y = y + len( step_dict[ 'data_inputs' ] ) * line_px + 40 - text.append( svgfig.Line( x - margin, sep_y, x + width - margin, sep_y ).SVG() ) - # Define an input/output box. - boxes.append( svgfig.Rect( x - margin, y + 30, x + width - margin, y + 30 + box_height, fill="#ffffff" ).SVG() ) - for conn, output_dict in step_dict[ 'input_connections' ].iteritems(): - in_coords = in_pos[ step_dict[ 'id' ] ][ conn ] - # out_pos_index will be a step number like 1, 2, 3... - out_pos_index = output_dict[ 'id' ] - # out_pos_name will be a string like 'o', 'o2', etc. - out_pos_name = output_dict[ 'output_name' ] - if out_pos_index in out_pos: - # out_conn_index_dict will be something like: - # 7: {'o': (824.5, 618)} - out_conn_index_dict = out_pos[ out_pos_index ] - if out_pos_name in out_conn_index_dict: - out_conn_pos = out_pos[ out_pos_index ][ out_pos_name ] - else: - # Take any key / value pair available in out_conn_index_dict. - # A problem will result if the dictionary is empty. - if out_conn_index_dict.keys(): - key = out_conn_index_dict.keys()[0] - out_conn_pos = out_pos[ out_pos_index ][ key ] - adjusted = ( out_conn_pos[ 0 ] + widths[ output_dict[ 'id' ] ], out_conn_pos[ 1 ] ) - text.append( svgfig.SVG( "circle", - cx=out_conn_pos[ 0 ] + widths[ output_dict[ 'id' ] ] - margin, - cy=out_conn_pos[ 1 ] - margin, - r = 5, - fill="#ffffff" ) ) - connectors.append( svgfig.Line( adjusted[ 0 ], - adjusted[ 1 ] - margin, - in_coords[ 0 ] - 10, - in_coords[ 1 ], - arrow_end = "true" ).SVG() ) - canvas.append( connectors ) - canvas.append( boxes ) - canvas.append( text ) - width, height = ( max_x + max_width + 50 ), max_y + 300 - canvas[ 'width' ] = "%s px" % width - canvas[ 'height' ] = "%s px" % height - canvas[ 'viewBox' ] = "0 0 %s %s" % ( width, height ) - trans.response.set_content_type( "image/svg+xml" ) - return canvas.standalone_xml() -def get_workflow_data_inputs( step, module ): - if module.type == 'tool': - if module.tool: - return module.get_data_inputs() - else: - data_inputs = [] - for wfsc in step.input_connections: - data_inputs_dict = {} - data_inputs_dict[ 'extensions' ] = [ '' ] - data_inputs_dict[ 'name' ] = wfsc.input_name - data_inputs_dict[ 'label' ] = 'Unknown' - data_inputs.append( data_inputs_dict ) - return data_inputs - return module.get_data_inputs() -def get_workflow_data_outputs( step, module, steps ): - if module.type == 'tool': - if module.tool: - return module.get_data_outputs() - else: - data_outputs = [] - data_outputs_dict = {} - data_outputs_dict[ 'extensions' ] = [ 'input' ] - found = False - for workflow_step in steps: - for wfsc in workflow_step.input_connections: - if step.name == wfsc.output_step.name: - data_outputs_dict[ 'name' ] = wfsc.output_name - found = True - break - if found: - break - if not found: - # We're at the last step of the workflow. - data_outputs_dict[ 'name' ] = 'output' - data_outputs.append( data_outputs_dict ) - return data_outputs - return module.get_data_outputs() -def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id, changeset_revision ): - """ - Return an in-memory Workflow object from the dictionary object created when it was exported. This method is called from - both Galaxy and the tool shed to retrieve a Workflow object that can be displayed as an SVG image. This method is also - called from Galaxy to retrieve a Workflow object that can be used for saving to the Galaxy database. - """ - trans.workflow_building_mode = True - workflow = trans.model.Workflow() - workflow.name = workflow_dict[ 'name' ] - workflow.has_errors = False - steps = [] - # Keep ids for each step that we need to use to make connections. - steps_by_external_id = {} - # Keep track of tools required by the workflow that are not available in - # the tool shed repository. Each tuple in the list of missing_tool_tups - # will be ( tool_id, tool_name, tool_version ). - missing_tool_tups = [] - # First pass to build step objects and populate basic values - for key, step_dict in workflow_dict[ 'steps' ].iteritems(): - # Create the model class for the step - step = trans.model.WorkflowStep() - step.name = step_dict[ 'name' ] - step.position = step_dict[ 'position' ] - module = module_factory.from_dict( trans, repository_id, changeset_revision, step_dict, tools_metadata=tools_metadata, secure=False ) - if module.type == 'tool' and module.tool is None: - # A required tool is not available in the current repository. - step.tool_errors = 'unavailable' - missing_tool_tup = ( step_dict[ 'tool_id' ], step_dict[ 'name' ], step_dict[ 'tool_version' ] ) - if missing_tool_tup not in missing_tool_tups: - missing_tool_tups.append( missing_tool_tup ) - module.save_to_step( step ) - if step.tool_errors: - workflow.has_errors = True - # Stick this in the step temporarily. - step.temp_input_connections = step_dict[ 'input_connections' ] - steps.append( step ) - steps_by_external_id[ step_dict[ 'id' ] ] = step - # Second pass to deal with connections between steps. - for step in steps: - # Input connections. - for input_name, conn_dict in step.temp_input_connections.iteritems(): - if conn_dict: - output_step = steps_by_external_id[ conn_dict[ 'id' ] ] - conn = trans.model.WorkflowStepConnection() - conn.input_step = step - conn.input_name = input_name - conn.output_step = output_step - conn.output_name = conn_dict[ 'output_name' ] - step.input_connections.append( conn ) - del step.temp_input_connections - # Order the steps if possible. - galaxy.webapps.galaxy.controllers.workflow.attach_ordered_steps( workflow, steps ) - # Return the in-memory Workflow object for display or later persistence to the Galaxy database. - return workflow, missing_tool_tups -def get_workflow_module_name( module, missing_tool_tups ): - module_name = module.get_name() - if module.type == 'tool' and module_name == 'unavailable': - for missing_tool_tup in missing_tool_tups: - missing_tool_id, missing_tool_name, missing_tool_version = missing_tool_tup - if missing_tool_id == module.tool_id: - module_name = '%s' % missing_tool_name - break - return module_name -def save_workflow( trans, workflow ): - """Use the received in-memory Workflow object for saving to the Galaxy database.""" - stored = trans.model.StoredWorkflow() - stored.name = workflow.name - workflow.stored_workflow = stored - stored.latest_workflow = workflow - stored.user = trans.user - trans.sa_session.add( stored ) - trans.sa_session.flush() - # Add a new entry to the Workflows menu. - if trans.user.stored_workflow_menu_entries is None: - trans.user.stored_workflow_menu_entries = [] - menuEntry = trans.model.StoredWorkflowMenuEntry() - menuEntry.stored_workflow = stored - trans.user.stored_workflow_menu_entries.append( menuEntry ) - trans.sa_session.flush() - return stored diff -r 3f2e8f5f0d7de7131aa878e867c12c352f1f79b5 -r a5c906f0e8a03b9ec884b224393c5cb767dcf37d lib/tool_shed/util/workflow_util.py --- /dev/null +++ b/lib/tool_shed/util/workflow_util.py @@ -0,0 +1,411 @@ +from galaxy import eggs +import pkg_resources + +pkg_resources.require( "SVGFig" ) + +import logging, svgfig +from galaxy.util import json +import tool_shed.util.shed_util_common as suc +from tool_shed.util import encoding_util, metadata_util, tool_util +from galaxy.workflow.modules import InputDataModule, ToolModule, WorkflowModuleFactory +import galaxy.webapps.galaxy.controllers.workflow +import galaxy.tools +import galaxy.tools.parameters + +log = logging.getLogger( __name__ ) + +class RepoInputDataModule( InputDataModule ): + + type = "data_input" + name = "Input dataset" + + @classmethod + def new( Class, trans, tools_metadata=None, tool_id=None ): + module = Class( trans ) + module.state = dict( name="Input Dataset" ) + return module + @classmethod + def from_dict( Class, trans, repository_id, changeset_revision, step_dict, tools_metadata=None, secure=True ): + module = Class( trans ) + state = json.from_json_string( step_dict[ "tool_state" ] ) + module.state = dict( name=state.get( "name", "Input Dataset" ) ) + return module + @classmethod + def from_workflow_step( Class, trans, repository_id, changeset_revision, tools_metadata, step ): + module = Class( trans ) + module.state = dict( name="Input Dataset" ) + if step.tool_inputs and "name" in step.tool_inputs: + module.state[ 'name' ] = step.tool_inputs[ 'name' ] + return module + +class RepoToolModule( ToolModule ): + + type = "tool" + + def __init__( self, trans, repository_id, changeset_revision, tools_metadata, tool_id ): + self.trans = trans + self.tools_metadata = tools_metadata + self.tool_id = tool_id + self.tool = None + self.errors = None + for tool_dict in tools_metadata: + if self.tool_id in [ tool_dict[ 'id' ], tool_dict[ 'guid' ] ]: + if trans.webapp.name == 'tool_shed': + # We're in the tool shed. + repository, self.tool, message = tool_util.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_dict[ 'tool_config' ] ) + if message and self.tool is None: + self.errors = 'unavailable' + break + else: + # We're in Galaxy. + self.tool = trans.app.toolbox.tools_by_id.get( self.tool_id, None ) + if self.tool is None: + self.errors = 'unavailable' + self.post_job_actions = {} + self.workflow_outputs = [] + self.state = None + @classmethod + def new( Class, trans, repository_id, changeset_revision, tools_metadata, tool_id=None ): + module = Class( trans, repository_id, changeset_revision, tools_metadata, tool_id ) + module.state = module.tool.new_state( trans, all_pages=True ) + return module + @classmethod + def from_dict( Class, trans, repository_id, changeset_revision, step_dict, tools_metadata, secure=True ): + tool_id = step_dict[ 'tool_id' ] + module = Class( trans, repository_id, changeset_revision, tools_metadata, tool_id ) + module.state = galaxy.tools.DefaultToolState() + if module.tool is not None: + module.state.decode( step_dict[ "tool_state" ], module.tool, module.trans.app, secure=secure ) + module.errors = step_dict.get( "tool_errors", None ) + return module + @classmethod + def from_workflow_step( Class, trans, repository_id, changeset_revision, tools_metadata, step ): + module = Class( trans, repository_id, changeset_revision, tools_metadata, step.tool_id ) + module.state = galaxy.tools.DefaultToolState() + if module.tool: + module.state.inputs = module.tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True ) + else: + module.state.inputs = {} + module.errors = step.tool_errors + return module + def get_data_inputs( self ): + data_inputs = [] + def callback( input, value, prefixed_name, prefixed_label ): + if isinstance( input, galaxy.tools.parameters.DataToolParameter ): + data_inputs.append( dict( name=prefixed_name, + label=prefixed_label, + extensions=input.extensions ) ) + if self.tool: + galaxy.tools.parameters.visit_input_values( self.tool.inputs, self.state.inputs, callback ) + return data_inputs + def get_data_outputs( self ): + data_outputs = [] + if self.tool: + data_inputs = None + for name, tool_output in self.tool.outputs.iteritems(): + if tool_output.format_source != None: + # Default to special name "input" which remove restrictions on connections + formats = [ 'input' ] + if data_inputs == None: + data_inputs = self.get_data_inputs() + # Find the input parameter referenced by format_source + for di in data_inputs: + # Input names come prefixed with conditional and repeat names separated by '|', + # so remove prefixes when comparing with format_source. + if di[ 'name' ] != None and di[ 'name' ].split( '|' )[ -1 ] == tool_output.format_source: + formats = di[ 'extensions' ] + else: + formats = [ tool_output.format ] + for change_elem in tool_output.change_format: + for when_elem in change_elem.findall( 'when' ): + format = when_elem.get( 'format', None ) + if format and format not in formats: + formats.append( format ) + data_outputs.append( dict( name=name, extensions=formats ) ) + return data_outputs + +class RepoWorkflowModuleFactory( WorkflowModuleFactory ): + def __init__( self, module_types ): + self.module_types = module_types + def new( self, trans, type, tools_metadata=None, tool_id=None ): + """Return module for type and (optional) tool_id initialized with new / default state.""" + assert type in self.module_types + return self.module_types[type].new( trans, tool_id ) + def from_dict( self, trans, repository_id, changeset_revision, step_dict, **kwd ): + """Return module initialized from the data in dictionary `step_dict`.""" + type = step_dict[ 'type' ] + assert type in self.module_types + return self.module_types[ type ].from_dict( trans, repository_id, changeset_revision, step_dict, **kwd ) + def from_workflow_step( self, trans, repository_id, changeset_revision, tools_metadata, step ): + """Return module initialized from the WorkflowStep object `step`.""" + type = step.type + return self.module_types[ type ].from_workflow_step( trans, repository_id, changeset_revision, tools_metadata, step ) + +module_factory = RepoWorkflowModuleFactory( dict( data_input=RepoInputDataModule, tool=RepoToolModule ) ) + +def generate_workflow_image( trans, workflow_name, repository_metadata_id=None, repository_id=None ): + """ + Return an svg image representation of a workflow dictionary created when the workflow was exported. This method is called + from both Galaxy and the tool shed. When called from the tool shed, repository_metadata_id will have a value and repository_id + will be None. When called from Galaxy, repository_metadata_id will be None and repository_id will have a value. + """ + workflow_name = encoding_util.tool_shed_decode( workflow_name ) + if trans.webapp.name == 'tool_shed': + # We're in the tool shed. + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_id = trans.security.encode_id( repository_metadata.repository_id ) + changeset_revision = repository_metadata.changeset_revision + metadata = repository_metadata.metadata + else: + # We're in Galaxy. + repository = suc.get_tool_shed_repository_by_id( trans, repository_id ) + changeset_revision = repository.changeset_revision + metadata = repository.metadata + # metadata[ 'workflows' ] is a list of tuples where each contained tuple is + # [ <relative path to the .ga file in the repository>, <exported workflow dict> ] + for workflow_tup in metadata[ 'workflows' ]: + workflow_dict = workflow_tup[1] + if workflow_dict[ 'name' ] == workflow_name: + break + if 'tools' in metadata: + tools_metadata = metadata[ 'tools' ] + else: + tools_metadata = [] + workflow, missing_tool_tups = get_workflow_from_dict( trans=trans, + workflow_dict=workflow_dict, + tools_metadata=tools_metadata, + repository_id=repository_id, + changeset_revision=changeset_revision ) + data = [] + canvas = svgfig.canvas( style="stroke:black; fill:none; stroke-width:1px; stroke-linejoin:round; text-anchor:left" ) + text = svgfig.SVG( "g" ) + connectors = svgfig.SVG( "g" ) + boxes = svgfig.SVG( "g" ) + svgfig.Text.defaults[ "font-size" ] = "10px" + in_pos = {} + out_pos = {} + margin = 5 + # Spacing between input/outputs. + line_px = 16 + # Store px width for boxes of each step. + widths = {} + max_width, max_x, max_y = 0, 0, 0 + for step in workflow.steps: + step.upgrade_messages = {} + module = module_factory.from_workflow_step( trans, repository_id, changeset_revision, tools_metadata, step ) + tool_errors = module.type == 'tool' and not module.tool + module_data_inputs = get_workflow_data_inputs( step, module ) + module_data_outputs = get_workflow_data_outputs( step, module, workflow.steps ) + step_dict = { + 'id' : step.order_index, + 'data_inputs' : module_data_inputs, + 'data_outputs' : module_data_outputs, + 'position' : step.position, + 'tool_errors' : tool_errors + } + input_conn_dict = {} + for conn in step.input_connections: + input_conn_dict[ conn.input_name ] = dict( id=conn.output_step.order_index, output_name=conn.output_name ) + step_dict[ 'input_connections' ] = input_conn_dict + data.append( step_dict ) + x, y = step.position[ 'left' ], step.position[ 'top' ] + count = 0 + module_name = get_workflow_module_name( module, missing_tool_tups ) + max_len = len( module_name ) * 1.5 + text.append( svgfig.Text( x, y + 20, module_name, **{ "font-size": "14px" } ).SVG() ) + y += 45 + for di in module_data_inputs: + cur_y = y + count * line_px + if step.order_index not in in_pos: + in_pos[ step.order_index ] = {} + in_pos[ step.order_index ][ di[ 'name' ] ] = ( x, cur_y ) + text.append( svgfig.Text( x, cur_y, di[ 'label' ] ).SVG() ) + count += 1 + max_len = max( max_len, len( di[ 'label' ] ) ) + if len( module.get_data_inputs() ) > 0: + y += 15 + for do in module_data_outputs: + cur_y = y + count * line_px + if step.order_index not in out_pos: + out_pos[ step.order_index ] = {} + out_pos[ step.order_index ][ do[ 'name' ] ] = ( x, cur_y ) + text.append( svgfig.Text( x, cur_y, do[ 'name' ] ).SVG() ) + count += 1 + max_len = max( max_len, len( do['name' ] ) ) + widths[ step.order_index ] = max_len * 5.5 + max_x = max( max_x, step.position[ 'left' ] ) + max_y = max( max_y, step.position[ 'top' ] ) + max_width = max( max_width, widths[ step.order_index ] ) + for step_dict in data: + tool_unavailable = step_dict[ 'tool_errors' ] + width = widths[ step_dict[ 'id' ] ] + x, y = step_dict[ 'position' ][ 'left' ], step_dict[ 'position' ][ 'top' ] + # Only highlight missing tools if displaying in the tool shed. + if trans.webapp.name == 'tool_shed' and tool_unavailable: + fill = "#EBBCB2" + else: + fill = "#EBD9B2" + boxes.append( svgfig.Rect( x - margin, y, x + width - margin, y + 30, fill=fill ).SVG() ) + box_height = ( len( step_dict[ 'data_inputs' ] ) + len( step_dict[ 'data_outputs' ] ) ) * line_px + margin + # Draw separator line. + if len( step_dict[ 'data_inputs' ] ) > 0: + box_height += 15 + sep_y = y + len( step_dict[ 'data_inputs' ] ) * line_px + 40 + text.append( svgfig.Line( x - margin, sep_y, x + width - margin, sep_y ).SVG() ) + # Define an input/output box. + boxes.append( svgfig.Rect( x - margin, y + 30, x + width - margin, y + 30 + box_height, fill="#ffffff" ).SVG() ) + for conn, output_dict in step_dict[ 'input_connections' ].iteritems(): + in_coords = in_pos[ step_dict[ 'id' ] ][ conn ] + # out_pos_index will be a step number like 1, 2, 3... + out_pos_index = output_dict[ 'id' ] + # out_pos_name will be a string like 'o', 'o2', etc. + out_pos_name = output_dict[ 'output_name' ] + if out_pos_index in out_pos: + # out_conn_index_dict will be something like: + # 7: {'o': (824.5, 618)} + out_conn_index_dict = out_pos[ out_pos_index ] + if out_pos_name in out_conn_index_dict: + out_conn_pos = out_pos[ out_pos_index ][ out_pos_name ] + else: + # Take any key / value pair available in out_conn_index_dict. + # A problem will result if the dictionary is empty. + if out_conn_index_dict.keys(): + key = out_conn_index_dict.keys()[0] + out_conn_pos = out_pos[ out_pos_index ][ key ] + adjusted = ( out_conn_pos[ 0 ] + widths[ output_dict[ 'id' ] ], out_conn_pos[ 1 ] ) + text.append( svgfig.SVG( "circle", + cx=out_conn_pos[ 0 ] + widths[ output_dict[ 'id' ] ] - margin, + cy=out_conn_pos[ 1 ] - margin, + r = 5, + fill="#ffffff" ) ) + connectors.append( svgfig.Line( adjusted[ 0 ], + adjusted[ 1 ] - margin, + in_coords[ 0 ] - 10, + in_coords[ 1 ], + arrow_end = "true" ).SVG() ) + canvas.append( connectors ) + canvas.append( boxes ) + canvas.append( text ) + width, height = ( max_x + max_width + 50 ), max_y + 300 + canvas[ 'width' ] = "%s px" % width + canvas[ 'height' ] = "%s px" % height + canvas[ 'viewBox' ] = "0 0 %s %s" % ( width, height ) + trans.response.set_content_type( "image/svg+xml" ) + return canvas.standalone_xml() +def get_workflow_data_inputs( step, module ): + if module.type == 'tool': + if module.tool: + return module.get_data_inputs() + else: + data_inputs = [] + for wfsc in step.input_connections: + data_inputs_dict = {} + data_inputs_dict[ 'extensions' ] = [ '' ] + data_inputs_dict[ 'name' ] = wfsc.input_name + data_inputs_dict[ 'label' ] = 'Unknown' + data_inputs.append( data_inputs_dict ) + return data_inputs + return module.get_data_inputs() +def get_workflow_data_outputs( step, module, steps ): + if module.type == 'tool': + if module.tool: + return module.get_data_outputs() + else: + data_outputs = [] + data_outputs_dict = {} + data_outputs_dict[ 'extensions' ] = [ 'input' ] + found = False + for workflow_step in steps: + for wfsc in workflow_step.input_connections: + if step.name == wfsc.output_step.name: + data_outputs_dict[ 'name' ] = wfsc.output_name + found = True + break + if found: + break + if not found: + # We're at the last step of the workflow. + data_outputs_dict[ 'name' ] = 'output' + data_outputs.append( data_outputs_dict ) + return data_outputs + return module.get_data_outputs() +def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id, changeset_revision ): + """ + Return an in-memory Workflow object from the dictionary object created when it was exported. This method is called from + both Galaxy and the tool shed to retrieve a Workflow object that can be displayed as an SVG image. This method is also + called from Galaxy to retrieve a Workflow object that can be used for saving to the Galaxy database. + """ + trans.workflow_building_mode = True + workflow = trans.model.Workflow() + workflow.name = workflow_dict[ 'name' ] + workflow.has_errors = False + steps = [] + # Keep ids for each step that we need to use to make connections. + steps_by_external_id = {} + # Keep track of tools required by the workflow that are not available in + # the tool shed repository. Each tuple in the list of missing_tool_tups + # will be ( tool_id, tool_name, tool_version ). + missing_tool_tups = [] + # First pass to build step objects and populate basic values + for key, step_dict in workflow_dict[ 'steps' ].iteritems(): + # Create the model class for the step + step = trans.model.WorkflowStep() + step.name = step_dict[ 'name' ] + step.position = step_dict[ 'position' ] + module = module_factory.from_dict( trans, repository_id, changeset_revision, step_dict, tools_metadata=tools_metadata, secure=False ) + if module.type == 'tool' and module.tool is None: + # A required tool is not available in the current repository. + step.tool_errors = 'unavailable' + missing_tool_tup = ( step_dict[ 'tool_id' ], step_dict[ 'name' ], step_dict[ 'tool_version' ] ) + if missing_tool_tup not in missing_tool_tups: + missing_tool_tups.append( missing_tool_tup ) + module.save_to_step( step ) + if step.tool_errors: + workflow.has_errors = True + # Stick this in the step temporarily. + step.temp_input_connections = step_dict[ 'input_connections' ] + steps.append( step ) + steps_by_external_id[ step_dict[ 'id' ] ] = step + # Second pass to deal with connections between steps. + for step in steps: + # Input connections. + for input_name, conn_dict in step.temp_input_connections.iteritems(): + if conn_dict: + output_step = steps_by_external_id[ conn_dict[ 'id' ] ] + conn = trans.model.WorkflowStepConnection() + conn.input_step = step + conn.input_name = input_name + conn.output_step = output_step + conn.output_name = conn_dict[ 'output_name' ] + step.input_connections.append( conn ) + del step.temp_input_connections + # Order the steps if possible. + galaxy.webapps.galaxy.controllers.workflow.attach_ordered_steps( workflow, steps ) + # Return the in-memory Workflow object for display or later persistence to the Galaxy database. + return workflow, missing_tool_tups +def get_workflow_module_name( module, missing_tool_tups ): + module_name = module.get_name() + if module.type == 'tool' and module_name == 'unavailable': + for missing_tool_tup in missing_tool_tups: + missing_tool_id, missing_tool_name, missing_tool_version = missing_tool_tup + if missing_tool_id == module.tool_id: + module_name = '%s' % missing_tool_name + break + return module_name +def save_workflow( trans, workflow ): + """Use the received in-memory Workflow object for saving to the Galaxy database.""" + stored = trans.model.StoredWorkflow() + stored.name = workflow.name + workflow.stored_workflow = stored + stored.latest_workflow = workflow + stored.user = trans.user + trans.sa_session.add( stored ) + trans.sa_session.flush() + # Add a new entry to the Workflows menu. + if trans.user.stored_workflow_menu_entries is None: + trans.user.stored_workflow_menu_entries = [] + menuEntry = trans.model.StoredWorkflowMenuEntry() + menuEntry.stored_workflow = stored + trans.user.stored_workflow_menu_entries.append( menuEntry ) + trans.sa_session.flush() + return stored Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add blank lines to shed_util_common.py per new Galaxy dev team standards.
by commits-noreply＠bitbucket.org 16 Mar '13

16 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3f2e8f5f0d7d/ changeset: 3f2e8f5f0d7d user: greg date: 2013-03-16 16:34:03 summary: Add blank lines to shed_util_common.py per new Galaxy dev team standards. affected #: 1 file diff -r 74da2fbf09633268e1e6b6043bfd7e0515bd8422 -r 3f2e8f5f0d7de7131aa878e867c12c352f1f79b5 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -107,12 +107,14 @@ option_value = trans.security.encode_id( repository.id ) repositories_select_field.add_option( option_label, option_value ) return repositories_select_field + def changeset_is_malicious( trans, id, changeset_revision, **kwd ): """Check the malicious flag in repository metadata for a specified change set""" repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: return repository_metadata.malicious return False + def changeset_is_valid( app, repository, changeset_revision ): repo = hg.repository( get_configured_ui(), repository.repo_path( app ) ) for changeset in repo.changelog: @@ -120,6 +122,7 @@ if changeset_revision == changeset_hash: return True return False + def clean_repository_clone_url( repository_clone_url ): if repository_clone_url.find( '@' ) > 0: # We have an url that includes an authenticated user, something like: @@ -134,11 +137,13 @@ else: tmp_url = repository_clone_url return tmp_url + def clean_tool_shed_url( tool_shed_url ): if tool_shed_url.find( ':' ) > 0: # Eliminate the port, if any, since it will result in an invalid directory name. return tool_shed_url.split( ':' )[ 0 ] return tool_shed_url.rstrip( '/' ) + def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" try: @@ -153,6 +158,7 @@ error_message = 'Error cloning repository: %s' % str( e ) log.debug( error_message ) return False, error_message + def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): # Persist the current in-memory list of config_elems to a file named by the value of config_filename. fd, filename = tempfile.mkstemp() @@ -164,6 +170,7 @@ os.close( fd ) shutil.move( filename, os.path.abspath( config_filename ) ) os.chmod( config_filename, 0644 ) + def copy_file_from_manifest( repo, ctx, filename, dir ): """Copy the latest version of the file named filename from the repository manifest to the directory to which dir refers.""" for changeset in reversed_upper_bounded_changelog( repo, ctx ): @@ -176,6 +183,7 @@ fh.close() return file_path return None + def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): # The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used @@ -231,10 +239,12 @@ sa_session.add( tool_shed_repository ) sa_session.flush() return tool_shed_repository + def generate_clone_url_for_installed_repository( app, repository ): """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" tool_shed_url = get_url_from_repository_tool_shed( app, repository ) return url_join( tool_shed_url, 'repos', repository.owner, repository.name ) + def generate_clone_url_for_repository_in_tool_shed( trans, repository ): """Generate the URL for cloning a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) @@ -244,12 +254,14 @@ return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name ) else: return '%s/repos/%s/%s' % ( base_url, repository.user.username, repository.name ) + def generate_clone_url_from_repo_info_tup( repo_info_tup ): """Generate teh URL for cloning a repositoyr given a tuple of toolshed, name, owner, changeset_revision.""" # Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab'] toolshed, name, owner, changeset_revision = repo_info_tup # Don't include the changeset_revision in clone urls. return url_join( toolshed, 'repos', owner, name ) + def generate_sharable_link_for_repository_in_tool_shed( trans, repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) @@ -258,6 +270,7 @@ if changeset_revision: sharable_url += '/%s' % changeset_revision return sharable_url + def generate_tool_elem( tool_shed, repository_name, changeset_revision, owner, tool_file_path, tool, tool_section ): if tool_section is not None: tool_elem = SubElement( tool_section, 'tool' ) @@ -278,6 +291,7 @@ version_elem = SubElement( tool_elem, 'version' ) version_elem.text = tool.version return tool_elem + def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for @@ -286,6 +300,7 @@ """ tmp_url = clean_repository_clone_url( repository_clone_url ) return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) + def generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ): """ Keep track of the section in the tool panel in which this repository's tools will be contained by parsing the shed_tool_conf in @@ -333,6 +348,7 @@ else: tool_panel_dict[ guid ] = [ tool_section_dict ] return tool_panel_dict + def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_revision ): """ Generate a repository installation directory that guarantees repositories with the same name will always be installed in different directories. @@ -345,6 +361,7 @@ repo_path = items[ 1 ] tool_shed_url = clean_tool_shed_url( tool_shed_url ) return url_join( tool_shed_url, 'repos', repo_path, changeset_revision ) + def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): """Return the absolute path to a specified disk file contained in a repository.""" stripped_file_name = strip_path( file_name ) @@ -355,21 +372,25 @@ if name == stripped_file_name: return os.path.abspath( os.path.join( root, name ) ) return file_path + def get_categories( trans ): """Get all categories from the database.""" return trans.sa_session.query( trans.model.Category ) \ .filter( trans.model.Category.table.c.deleted==False ) \ .order_by( trans.model.Category.table.c.name ) \ .all() + def get_category( trans, id ): """Get a category from the database.""" return trans.sa_session.query( trans.model.Category ).get( trans.security.decode_id( id ) ) + def get_category_by_name( trans, name ): """Get a category from the database via name.""" try: return trans.sa_session.query( trans.model.Category ).filter_by( name=name ).one() except sqlalchemy.orm.exc.NoResultFound: return None + def get_changectx_for_changeset( repo, changeset_revision, **kwd ): """Retrieve a specified changectx from a repository.""" for changeset in repo.changelog: @@ -377,6 +398,7 @@ if str( ctx ) == changeset_revision: return ctx return None + def get_config( config_file, repo, ctx, dir ): """Return the latest version of config_filename from the repository manifest.""" config_file = strip_path( config_file ) @@ -387,6 +409,7 @@ if ctx_file_name == config_file: return get_named_tmpfile_from_ctx( changeset_ctx, ctx_file, dir ) return None + def get_config_from_disk( config_file, relative_install_dir ): for root, dirs, files in os.walk( relative_install_dir ): if root.find( '.hg' ) < 0: @@ -394,6 +417,7 @@ if name == config_file: return os.path.abspath( os.path.join( root, name ) ) return None + def get_configured_ui(): """Configure any desired ui settings.""" _ui = ui.ui() @@ -403,12 +427,14 @@ # quiet = True _ui.setconfig( 'ui', 'quiet', True ) return _ui + def get_ctx_rev( tool_shed_url, name, owner, changeset_revision ): url = url_join( tool_shed_url, 'repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % ( name, owner, changeset_revision ) ) response = urllib2.urlopen( url ) ctx_rev = response.read() response.close() return ctx_rev + def get_ctx_file_path_from_manifest( filename, repo, changeset_revision ): """Get the ctx file path for the latest revision of filename from the repository manifest up to the value of changeset_revision.""" stripped_filename = strip_path( filename ) @@ -420,6 +446,7 @@ if ctx_file_name == stripped_filename: return manifest_ctx, ctx_file return None, None + def get_file_context_from_ctx( ctx, filename ): # We have to be careful in determining if we found the correct file because multiple files with the same name may be in different directories # within ctx if the files were moved within the change set. For example, in the following ctx.files() list, the former may have been moved to @@ -440,9 +467,11 @@ if deleted: return 'DELETED' return None + def get_installed_tool_shed_repository( trans, id ): """Get a repository on the Galaxy side from the database via id""" return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) + def get_named_tmpfile_from_ctx( ctx, filename, dir ): filename = strip_path( filename ) for ctx_file in ctx.files(): @@ -464,6 +493,7 @@ fh.close() return tmp_filename return None + def get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision ): """ Return the installable changeset_revision in the repository changelog after the changeset to which after_changeset_revision refers. If there @@ -484,6 +514,7 @@ # We've found the changeset in the changelog for which we need to get the next downloadable changset. found_after_changeset_revision = True return None + def get_or_create_tool_shed_repository( trans, tool_shed, name, owner, changeset_revision ): repository = get_repository_for_dependency_relationship( trans.app, tool_shed, name, owner, changeset_revision ) if not repository: @@ -502,6 +533,7 @@ owner=owner, dist_to_shed=False ) return repository + def get_ordered_downloadable_changeset_revisions( repository, repo ): """Return an ordered list of changeset_revisions defined by a repository changelog.""" changeset_tups = [] @@ -516,6 +548,7 @@ sorted_changeset_tups = sorted( changeset_tups ) sorted_changeset_revisions = [ changeset_tup[ 1 ] for changeset_tup in sorted_changeset_tups ] return sorted_changeset_revisions + def get_previous_downloadable_changeset_revision( repository, repo, before_changeset_revision ): """ Return the installable changeset_revision in the repository changelog prior to the changeset to which before_changeset_revision @@ -537,6 +570,7 @@ return INITIAL_CHANGELOG_HASH else: previous_changeset_revision = changeset_revision + def get_repo_info_tuple_contents( repo_info_tuple ): # Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced. if len( repo_info_tuple ) == 6: @@ -545,12 +579,14 @@ elif len( repo_info_tuple ) == 7: description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple return description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies + def get_repository_by_id( trans, id ): """Get a repository from the database via id.""" if trans.webapp.name == 'galaxy': return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) else: return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) + def get_repository_by_name( app, name ): """Get a repository from the database via name.""" sa_session = app.model.context.current @@ -558,6 +594,7 @@ return sa_session.query( app.model.ToolShedRepository ).filter_by( name=name ).first() else: return sa_session.query( app.model.Repository ).filter_by( name=name ).first() + def get_repository_by_name_and_owner( app, name, owner ): """Get a repository from the database via name and owner""" sa_session = app.model.context.current @@ -574,6 +611,7 @@ app.model.Repository.table.c.user_id == user.id ) ) \ .first() return None + def get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ): repository = get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( app=app, tool_shed=tool_shed, @@ -587,6 +625,7 @@ owner=owner, changeset_revision=changeset_revision ) return repository + def get_repository_file_contents( file_path ): if checkers.is_gzip( file_path ): safe_str = to_safe_string( '\ngzip compressed file\n' ) @@ -605,6 +644,7 @@ safe_str = '%s%s' % ( safe_str, to_safe_string( large_str ) ) break return safe_str + def get_repository_files( trans, folder_path ): contents = [] for item in os.listdir( folder_path ): @@ -618,9 +658,11 @@ if contents: contents.sort() return contents + def get_repository_in_tool_shed( trans, id ): """Get a repository on the tool shed side from the database via id.""" return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) + def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): """Get metadata for a specified repository change set from the database.""" # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. Duplicate records were somehow @@ -640,16 +682,19 @@ elif all_metadata_records: return all_metadata_records[ 0 ] return None + def get_repository_owner( cleaned_repository_url ): items = cleaned_repository_url.split( '/repos/' ) repo_path = items[ 1 ] if repo_path.startswith( '/' ): repo_path = repo_path.replace( '/', '', 1 ) return repo_path.lstrip( '/' ).split( '/' )[ 0 ] + def get_repository_owner_from_clone_url( repository_clone_url ): tmp_url = clean_repository_clone_url( repository_clone_url ) tool_shed = tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) return get_repository_owner( tmp_url ) + def get_repository_tools_tups( app, metadata_dict ): repository_tools_tups = [] index, shed_conf_dict = get_shed_tool_conf_dict( app, metadata_dict.get( 'shed_config_filename' ) ) @@ -666,11 +711,13 @@ if tool: repository_tools_tups.append( ( relative_path, guid, tool ) ) return repository_tools_tups + def get_reversed_changelog_changesets( repo ): reversed_changelog = [] for changeset in repo.changelog: reversed_changelog.insert( 0, changeset ) return reversed_changelog + def get_revision_label( trans, repository, changeset_revision ): """Return a string consisting of the human read-able changeset rev and the changeset revision string.""" repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) @@ -679,6 +726,7 @@ return "%s:%s" % ( str( ctx.rev() ), changeset_revision ) else: return "-1:%s" % changeset_revision + def get_rev_label_from_changeset_revision( repo, changeset_revision ): ctx = get_changectx_for_changeset( repo, changeset_revision ) if ctx: @@ -688,6 +736,7 @@ rev = '-1' label = "-1:%s" % changeset_revision return rev, label + def get_shed_tool_conf_dict( app, shed_tool_conf ): """Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry in the shed_tool_conf_dict associated with the file.""" for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): @@ -697,6 +746,7 @@ file_name = strip_path( shed_tool_conf_dict[ 'config_filename' ] ) if shed_tool_conf == file_name: return index, shed_tool_conf_dict + def get_tool_panel_config_tool_path_install_dir( app, repository ): # Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the # repository is installed. This method assumes all repository tools are defined in a single shed-related tool panel config. @@ -715,6 +765,7 @@ tool_path = shed_config_dict[ 'tool_path' ] relative_install_dir = partial_install_dir return shed_tool_conf, tool_path, relative_install_dir + def get_tool_path_by_shed_tool_conf_filename( trans, shed_tool_conf ): """ Return the tool_path config setting for the received shed_tool_conf file by searching the tool box's in-memory list of shed_tool_confs for the @@ -729,10 +780,12 @@ if file_name == shed_tool_conf: return shed_tool_conf_dict[ 'tool_path' ] return None + def get_tool_shed_repository_by_id( trans, repository_id ): return trans.sa_session.query( trans.model.ToolShedRepository ) \ .filter( trans.model.ToolShedRepository.table.c.id == trans.security.decode_id( repository_id ) ) \ .first() + def get_tool_shed_repository_by_shed_name_owner_changeset_revision( app, tool_shed, name, owner, changeset_revision ): # This method is used only in Galaxy, not the tool shed. sa_session = app.model.context.current @@ -745,6 +798,7 @@ app.model.ToolShedRepository.table.c.owner == owner, app.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ .first() + def get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( app, tool_shed, name, owner, installed_changeset_revision ): # This method is used only in Galaxy, not the tool shed. sa_session = app.model.context.current @@ -757,9 +811,11 @@ app.model.ToolShedRepository.table.c.owner == owner, app.model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ .first() + def get_tool_shed_from_clone_url( repository_clone_url ): tmp_url = clean_repository_clone_url( repository_clone_url ) return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) + def get_url_from_repository_tool_shed( app, repository ): """ The stored value of repository.tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is @@ -772,6 +828,7 @@ return shed_url # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. return None + def get_url_from_tool_shed( app, tool_shed ): # The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is something like: # http://toolshed.g2.bx.psu.edu/ @@ -782,9 +839,11 @@ return shed_url # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. return None + def get_user( trans, id ): """Get a user from the database by id.""" return trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( id ) ) + def get_user_by_username( app, username ): """Get a user from the database by username.""" sa_session = app.model.context.current @@ -795,6 +854,7 @@ return user except Exception, e: return None + def handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=False, admin_only=False ): # There are 2 complementary features that enable a tool shed user to receive email notification: # 1. Within User Preferences, they can elect to receive email when the first (or first valid) @@ -886,6 +946,7 @@ util.send_mail( frm, to, subject, body, trans.app.config ) except Exception, e: log.exception( "An error occurred sending a tool shed repository update alert by email." ) + def open_repository_files_folder( trans, folder_path ): try: files_list = get_repository_files( trans, folder_path ) @@ -907,12 +968,14 @@ "key": full_path } folder_contents.append( node ) return folder_contents + def remove_dir( dir ): if os.path.exists( dir ): try: shutil.rmtree( dir ) except: pass + def repository_was_previously_installed( trans, tool_shed_url, repository_name, repo_info_tuple ): """ Handle the case where the repository was previously installed using an older changeset_revsion, but later the repository was updated @@ -940,6 +1003,7 @@ if tool_shed_repository and tool_shed_repository.status not in [ trans.model.ToolShedRepository.installation_status.NEW ]: return tool_shed_repository, previous_changeset_revision return None, None + def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ): """ Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision, but up to and @@ -963,8 +1027,10 @@ if changeset_hash == included_upper_bounds_changeset_revision: break return reversed_changelog + def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revision ): return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) + def strip_path( fpath ): if not fpath: return fpath @@ -973,6 +1039,7 @@ except: file_name = fpath return file_name + def to_safe_string( text, to_html=True ): """Translates the characters in text to an html string""" if text: @@ -1003,10 +1070,13 @@ translated.append( '' ) return ''.join( translated ) return text + def tool_shed_from_repository_clone_url( repository_clone_url ): return clean_repository_clone_url( repository_clone_url ).split( '/repos/' )[ 0 ].rstrip( '/' ) + def tool_shed_is_this_tool_shed( toolshed_base_url ): return toolshed_base_url.rstrip( '/' ) == str( url_for( '/', qualified=True ) ).rstrip( '/' ) + def translate_string( raw_text, to_html=True ): if raw_text: if len( raw_text ) <= MAX_CONTENT_SIZE: @@ -1017,6 +1087,7 @@ else: translated_string = '' return translated_string + def update_in_shed_tool_config( app, repository ): """ A tool shed repository is being updated so change the shed_tool_conf file. Parse the config file to generate the entire list @@ -1050,6 +1121,7 @@ elem = guid_to_tool_elem_dict[ guid ] config_elems.append( elem ) config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) + def update_repository( repo, ctx_rev=None ): """ Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired @@ -1067,11 +1139,13 @@ # It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that # purging is not supported by the mercurial API. commands.update( get_configured_ui(), repo, rev=ctx_rev ) + def update_tool_shed_repository_status( app, tool_shed_repository, status ): sa_session = app.model.context.current tool_shed_repository.status = status sa_session.add( tool_shed_repository ) sa_session.flush() + def url_join( *args ): parts = [] for arg in args: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Extract repository review methods from shed_util_comman and put them into the new tool shed review_util component.
by commits-noreply＠bitbucket.org 16 Mar '13

16 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/74da2fbf0963/ changeset: 74da2fbf0963 user: greg date: 2013-03-16 16:25:43 summary: Extract repository review methods from shed_util_comman and put them into the new tool shed review_util component. affected #: 4 files diff -r e40606362e962b4c452960c7c5f9b33b5b13f96c -r 74da2fbf09633268e1e6b6043bfd7e0515bd8422 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -11,7 +11,7 @@ from galaxy.util import json from galaxy.model.orm import and_, or_ import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, metadata_util, readme_util, repository_dependency_util, tool_dependency_util, tool_util +from tool_shed.util import encoding_util, metadata_util, readme_util, repository_dependency_util, review_util, tool_dependency_util, tool_util from tool_shed.galaxy_install import repository_util from galaxy.webapps.tool_shed.util import common_util, container_util, workflow_util import galaxy.tools @@ -1709,16 +1709,16 @@ categories = suc.get_categories( trans ) selected_categories = [ rca.category_id for rca in repository.categories ] # Determine if the current changeset revision has been reviewed by the current user. - reviewed_by_user = suc.changeset_revision_reviewed_by_user( trans, trans.user, repository, changeset_revision ) + reviewed_by_user = review_util.changeset_revision_reviewed_by_user( trans, trans.user, repository, changeset_revision ) if reviewed_by_user: - review = suc.get_review_by_repository_id_changeset_revision_user_id( trans=trans, - repository_id=id, - changeset_revision=changeset_revision, - user_id=trans.security.encode_id( trans.user.id ) ) + review = review_util.get_review_by_repository_id_changeset_revision_user_id( trans=trans, + repository_id=id, + changeset_revision=changeset_revision, + user_id=trans.security.encode_id( trans.user.id ) ) review_id = trans.security.encode_id( review.id ) else: review_id = None - can_browse_repository_reviews = suc.can_browse_repository_reviews( trans, repository ) + can_browse_repository_reviews = review_util.can_browse_repository_reviews( trans, repository ) containers_dict = container_util.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ) return trans.fill_template( '/webapps/tool_shed/repository/manage_repository.mako', repo_name=repo_name, @@ -2496,17 +2496,17 @@ message += malicious_error status = 'error' # Determine if the current changeset revision has been reviewed by the current user. - reviewed_by_user = suc.changeset_revision_reviewed_by_user( trans, trans.user, repository, changeset_revision ) + reviewed_by_user = review_util.changeset_revision_reviewed_by_user( trans, trans.user, repository, changeset_revision ) if reviewed_by_user: - review = suc.get_review_by_repository_id_changeset_revision_user_id( trans=trans, - repository_id=id, - changeset_revision=changeset_revision, - user_id=trans.security.encode_id( trans.user.id ) ) + review = review_util.get_review_by_repository_id_changeset_revision_user_id( trans=trans, + repository_id=id, + changeset_revision=changeset_revision, + user_id=trans.security.encode_id( trans.user.id ) ) review_id = trans.security.encode_id( review.id ) else: review_id = None containers_dict = container_util.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ) - can_browse_repository_reviews = suc.can_browse_repository_reviews( trans, repository ) + can_browse_repository_reviews = review_util.can_browse_repository_reviews( trans, repository ) return trans.fill_template( '/webapps/tool_shed/repository/view_repository.mako', repo=repo, repository=repository, @@ -2590,12 +2590,12 @@ add_id_to_name=False, downloadable=False ) trans.app.config.tool_data_path = original_tool_data_path - reviewed_by_user = suc.changeset_revision_reviewed_by_user( trans, trans.user, repository, changeset_revision ) + reviewed_by_user = review_util.changeset_revision_reviewed_by_user( trans, trans.user, repository, changeset_revision ) if reviewed_by_user: - review = suc.get_review_by_repository_id_changeset_revision_user_id( trans=trans, - repository_id=repository_id, - changeset_revision=changeset_revision, - user_id=trans.security.encode_id( trans.user.id ) ) + review = review_util.get_review_by_repository_id_changeset_revision_user_id( trans=trans, + repository_id=repository_id, + changeset_revision=changeset_revision, + user_id=trans.security.encode_id( trans.user.id ) ) review_id = trans.security.encode_id( review.id ) else: review_id = None diff -r e40606362e962b4c452960c7c5f9b33b5b13f96c -r 74da2fbf09633268e1e6b6043bfd7e0515bd8422 lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -6,6 +6,7 @@ from galaxy.webapps.tool_shed.util import common_util from galaxy.webapps.tool_shed.util.container_util import STRSEP import tool_shed.util.shed_util_common as suc +from tool_shed.util import review_util from galaxy.util.odict import odict import tool_shed.grids.repository_review_grids as repository_review_grids import tool_shed.grids.util as grids_util @@ -36,7 +37,7 @@ message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) encoded_review_id = kwd[ 'id' ] - review = suc.get_review( trans, encoded_review_id ) + review = review_util.get_review( trans, encoded_review_id ) if kwd.get( 'approve_repository_review_button', False ): approved_select_field_name = '%s%sapproved' % ( encoded_review_id, STRSEP ) approved_select_field_value = str( kwd[ approved_select_field_name ] ) @@ -70,7 +71,7 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - review = suc.get_review( trans, kwd[ 'id' ] ) + review = review_util.get_review( trans, kwd[ 'id' ] ) repository = review.repository repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) rev, changeset_revision_label = suc.get_rev_label_from_changeset_revision( repo, review.changeset_revision ) @@ -108,7 +109,7 @@ if not name or not description: message = 'Enter a valid name and a description' status = 'error' - elif suc.get_component_by_name( trans, name ): + elif review_util.get_component_by_name( trans, name ): message = 'A component with that name already exists' status = 'error' else: @@ -142,15 +143,15 @@ if changeset_revision: # Make sure there is not already a review of the revision by the user. repository = suc.get_repository_in_tool_shed( trans, repository_id ) - if suc.get_review_by_repository_id_changeset_revision_user_id( trans=trans, - repository_id=repository_id, - changeset_revision=changeset_revision, - user_id=trans.security.encode_id( trans.user.id ) ): + if review_util.get_review_by_repository_id_changeset_revision_user_id( trans=trans, + repository_id=repository_id, + changeset_revision=changeset_revision, + user_id=trans.security.encode_id( trans.user.id ) ): message = "You have already created a review for revision %s of repository %s." % ( changeset_revision, repository.name ) status = "error" else: # See if there are any reviews for previous changeset revisions that the user can copy. - if not create_without_copying and not previous_review_id and suc.has_previous_repository_reviews( trans, repository, changeset_revision ): + if not create_without_copying and not previous_review_id and review_util.has_previous_repository_reviews( trans, repository, changeset_revision ): return trans.response.send_redirect( web.url_for( controller='repository_review', action='select_previous_review', **kwd ) ) @@ -168,7 +169,7 @@ trans.sa_session.add( review ) trans.sa_session.flush() if previous_review_id: - review_to_copy = suc.get_review( trans, previous_review_id ) + review_to_copy = review_util.get_review( trans, previous_review_id ) self.copy_review( trans, review_to_copy, review ) review_id = trans.security.encode_id( review.id ) message = "Begin your review of revision %s of repository %s." \ @@ -205,7 +206,7 @@ action='manage_categories', message=message, status='error' ) ) - component = suc.get_component( trans, id ) + component = review_util.get_component( trans, id ) if params.get( 'edit_component_button', False ): new_description = util.restore_text( params.get( 'description', '' ) ).strip() if component.description != new_description: @@ -231,9 +232,9 @@ message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) review_id = kwd.get( 'id', None ) - review = suc.get_review( trans, review_id ) + review = review_util.get_review( trans, review_id ) components_dict = odict() - for component in suc.get_components( trans ): + for component in review_util.get_components( trans ): components_dict[ component.name ] = dict( component=component, component_review=None ) repository = review.repository repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) @@ -283,8 +284,8 @@ approved = str( v ) elif component_review_attr == 'rating': rating = int( str( v ) ) - component = suc.get_component( trans, component_id ) - component_review = suc.get_component_review_by_repository_review_id_component_id( trans, review_id, component_id ) + component = review_util.get_component( trans, component_id ) + component_review = review_util.get_component_review_by_repository_review_id_component_id( trans, review_id, component_id ) if component_review: # See if the existing component review should be updated. if component_review.comment != comment or \ @@ -481,7 +482,7 @@ rev, changeset_revision_label = suc.get_rev_label_from_changeset_revision( repo, changeset_revision ) if changeset_revision in reviewed_revision_hashes: # Find the review for this changeset_revision - repository_reviews = suc.get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) + repository_reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) # Determine if the current user can add a review to this revision. can_add_review = trans.user not in [ repository_review.user for repository_review in repository_reviews ] repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) @@ -521,7 +522,7 @@ repo = hg.repository( suc.get_configured_ui(), repo_dir ) installable = changeset_revision in [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] rev, changeset_revision_label = suc.get_rev_label_from_changeset_revision( repo, changeset_revision ) - reviews = suc.get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) + reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako', repository=repository, changeset_revision=changeset_revision, @@ -541,7 +542,7 @@ if 'operation' in kwd: operation = kwd['operation'].lower() # The value of the received id is the encoded review id. - review = suc.get_review( trans, kwd[ 'id' ] ) + review = review_util.get_review( trans, kwd[ 'id' ] ) repository = review.repository kwd[ 'id' ] = trans.security.encode_id( repository.id ) if operation == "inspect repository revisions": @@ -587,7 +588,7 @@ repository = suc.get_repository_in_tool_shed( trans, kwd[ 'id' ] ) changeset_revision = kwd.get( 'changeset_revision', None ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) - previous_reviews_dict = suc.get_previous_repository_reviews( trans, repository, changeset_revision ) + previous_reviews_dict = review_util.get_previous_repository_reviews( trans, repository, changeset_revision ) rev, changeset_revision_label = suc.get_rev_label_from_changeset_revision( repo, changeset_revision ) return trans.fill_template( '/webapps/tool_shed/repository_review/select_previous_review.mako', repository=repository, diff -r e40606362e962b4c452960c7c5f9b33b5b13f96c -r 74da2fbf09633268e1e6b6043bfd7e0515bd8422 lib/tool_shed/util/review_util.py --- /dev/null +++ b/lib/tool_shed/util/review_util.py @@ -0,0 +1,100 @@ +import os, logging +from galaxy.util.odict import odict +import tool_shed.util.shed_util_common as suc +from galaxy.model.orm import and_ + +from galaxy import eggs +import pkg_resources + +pkg_resources.require( 'mercurial' ) +from mercurial import hg, ui, commands + +log = logging.getLogger( __name__ ) + +def can_browse_repository_reviews( trans, repository ): + """Determine if there are any reviews of the received repository for which the current user has permission to browse any component reviews.""" + user = trans.user + if user: + for review in repository.reviews: + for component_review in review.component_reviews: + if trans.app.security_agent.user_can_browse_component_review( trans.app, repository, component_review, user ): + return True + return False + +def changeset_revision_reviewed_by_user( trans, user, repository, changeset_revision ): + """Determine if the current changeset revision has been reviewed by the current user.""" + for review in repository.reviews: + if review.changeset_revision == changeset_revision and review.user == user: + return True + return False + +def get_component( trans, id ): + """Get a component from the database.""" + return trans.sa_session.query( trans.model.Component ).get( trans.security.decode_id( id ) ) + +def get_component_review( trans, id ): + """Get a component_review from the database""" + return trans.sa_session.query( trans.model.ComponentReview ).get( trans.security.decode_id( id ) ) + +def get_component_by_name( trans, name ): + """Get a component from the database via a name.""" + return trans.sa_session.query( trans.app.model.Component ) \ + .filter( trans.app.model.Component.table.c.name==name ) \ + .first() + +def get_component_review_by_repository_review_id_component_id( trans, repository_review_id, component_id ): + """Get a component_review from the database via repository_review_id and component_id.""" + return trans.sa_session.query( trans.model.ComponentReview ) \ + .filter( and_( trans.model.ComponentReview.table.c.repository_review_id == trans.security.decode_id( repository_review_id ), + trans.model.ComponentReview.table.c.component_id == trans.security.decode_id( component_id ) ) ) \ + .first() + +def get_components( trans ): + return trans.sa_session.query( trans.app.model.Component ) \ + .order_by( trans.app.model.Component.name ) \ + .all() + +def get_previous_repository_reviews( trans, repository, changeset_revision ): + """Return an ordered dictionary of repository reviews up to and including the received changeset revision.""" + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] + previous_reviews_dict = odict() + for changeset in suc.reversed_upper_bounded_changelog( repo, changeset_revision ): + previous_changeset_revision = str( repo.changectx( changeset ) ) + if previous_changeset_revision in reviewed_revision_hashes: + previous_rev, previous_changeset_revision_label = suc.get_rev_label_from_changeset_revision( repo, previous_changeset_revision ) + revision_reviews = get_reviews_by_repository_id_changeset_revision( trans, + trans.security.encode_id( repository.id ), + previous_changeset_revision ) + previous_reviews_dict[ previous_changeset_revision ] = dict( changeset_revision_label=previous_changeset_revision_label, + reviews=revision_reviews ) + return previous_reviews_dict + +def get_review( trans, id ): + """Get a repository_review from the database via id.""" + return trans.sa_session.query( trans.model.RepositoryReview ).get( trans.security.decode_id( id ) ) + +def get_review_by_repository_id_changeset_revision_user_id( trans, repository_id, changeset_revision, user_id ): + """Get a repository_review from the database via repository id, changeset_revision and user_id.""" + return trans.sa_session.query( trans.model.RepositoryReview ) \ + .filter( and_( trans.model.RepositoryReview.repository_id == trans.security.decode_id( repository_id ), + trans.model.RepositoryReview.changeset_revision == changeset_revision, + trans.model.RepositoryReview.user_id == trans.security.decode_id( user_id ) ) ) \ + .first() + +def get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ): + """Get all repository_reviews from the database via repository id and changeset_revision.""" + return trans.sa_session.query( trans.model.RepositoryReview ) \ + .filter( and_( trans.model.RepositoryReview.repository_id == trans.security.decode_id( repository_id ), + trans.model.RepositoryReview.changeset_revision == changeset_revision ) ) \ + .all() + +def has_previous_repository_reviews( trans, repository, changeset_revision ): + """Determine if a repository has a changeset revision review prior to the received changeset revision.""" + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] + for changeset in suc.reversed_upper_bounded_changelog( repo, changeset_revision ): + previous_changeset_revision = str( repo.changectx( changeset ) ) + if previous_changeset_revision in reviewed_revision_hashes: + return True + return False diff -r e40606362e962b4c452960c7c5f9b33b5b13f96c -r 74da2fbf09633268e1e6b6043bfd7e0515bd8422 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -3,7 +3,6 @@ from time import gmtime, strftime from galaxy import util from galaxy.util import json -from galaxy.util.odict import odict from galaxy.web import url_for from galaxy.web.form_builder import SelectField from galaxy.datatypes import checkers @@ -108,15 +107,6 @@ option_value = trans.security.encode_id( repository.id ) repositories_select_field.add_option( option_label, option_value ) return repositories_select_field -def can_browse_repository_reviews( trans, repository ): - """Determine if there are any reviews of the received repository for which the current user has permission to browse any component reviews.""" - user = trans.user - if user: - for review in repository.reviews: - for component_review in review.component_reviews: - if trans.app.security_agent.user_can_browse_component_review( trans.app, repository, component_review, user ): - return True - return False def changeset_is_malicious( trans, id, changeset_revision, **kwd ): """Check the malicious flag in repository metadata for a specified change set""" repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) @@ -130,12 +120,6 @@ if changeset_revision == changeset_hash: return True return False -def changeset_revision_reviewed_by_user( trans, user, repository, changeset_revision ): - """Determine if the current changeset revision has been reviewed by the current user.""" - for review in repository.reviews: - if review.changeset_revision == changeset_revision and review.user == user: - return True - return False def clean_repository_clone_url( repository_clone_url ): if repository_clone_url.find( '@' ) > 0: # We have an url that includes an authenticated user, something like: @@ -393,27 +377,6 @@ if str( ctx ) == changeset_revision: return ctx return None -def get_component( trans, id ): - """Get a component from the database.""" - return trans.sa_session.query( trans.model.Component ).get( trans.security.decode_id( id ) ) -def get_component_by_name( trans, name ): - """Get a component from the database via a name.""" - return trans.sa_session.query( trans.app.model.Component ) \ - .filter( trans.app.model.Component.table.c.name==name ) \ - .first() -def get_component_review( trans, id ): - """Get a component_review from the database""" - return trans.sa_session.query( trans.model.ComponentReview ).get( trans.security.decode_id( id ) ) -def get_component_review_by_repository_review_id_component_id( trans, repository_review_id, component_id ): - """Get a component_review from the database via repository_review_id and component_id.""" - return trans.sa_session.query( trans.model.ComponentReview ) \ - .filter( and_( trans.model.ComponentReview.table.c.repository_review_id == trans.security.decode_id( repository_review_id ), - trans.model.ComponentReview.table.c.component_id == trans.security.decode_id( component_id ) ) ) \ - .first() -def get_components( trans ): - return trans.sa_session.query( trans.app.model.Component ) \ - .order_by( trans.app.model.Component.name ) \ - .all() def get_config( config_file, repo, ctx, dir ): """Return the latest version of config_filename from the repository manifest.""" config_file = strip_path( config_file ) @@ -574,21 +537,6 @@ return INITIAL_CHANGELOG_HASH else: previous_changeset_revision = changeset_revision -def get_previous_repository_reviews( trans, repository, changeset_revision ): - """Return an ordered dictionary of repository reviews up to and including the received changeset revision.""" - repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) - reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] - previous_reviews_dict = odict() - for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ): - previous_changeset_revision = str( repo.changectx( changeset ) ) - if previous_changeset_revision in reviewed_revision_hashes: - previous_rev, previous_changeset_revision_label = get_rev_label_from_changeset_revision( repo, previous_changeset_revision ) - revision_reviews = get_reviews_by_repository_id_changeset_revision( trans, - trans.security.encode_id( repository.id ), - previous_changeset_revision ) - previous_reviews_dict[ previous_changeset_revision ] = dict( changeset_revision_label=previous_changeset_revision_label, - reviews=revision_reviews ) - return previous_reviews_dict def get_repo_info_tuple_contents( repo_info_tuple ): # Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced. if len( repo_info_tuple ) == 6: @@ -723,22 +671,6 @@ for changeset in repo.changelog: reversed_changelog.insert( 0, changeset ) return reversed_changelog -def get_review( trans, id ): - """Get a repository_review from the database via id.""" - return trans.sa_session.query( trans.model.RepositoryReview ).get( trans.security.decode_id( id ) ) -def get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ): - """Get all repository_reviews from the database via repository id and changeset_revision.""" - return trans.sa_session.query( trans.model.RepositoryReview ) \ - .filter( and_( trans.model.RepositoryReview.repository_id == trans.security.decode_id( repository_id ), - trans.model.RepositoryReview.changeset_revision == changeset_revision ) ) \ - .all() -def get_review_by_repository_id_changeset_revision_user_id( trans, repository_id, changeset_revision, user_id ): - """Get a repository_review from the database via repository id, changeset_revision and user_id.""" - return trans.sa_session.query( trans.model.RepositoryReview ) \ - .filter( and_( trans.model.RepositoryReview.repository_id == trans.security.decode_id( repository_id ), - trans.model.RepositoryReview.changeset_revision == changeset_revision, - trans.model.RepositoryReview.user_id == trans.security.decode_id( user_id ) ) ) \ - .first() def get_revision_label( trans, repository, changeset_revision ): """Return a string consisting of the human read-able changeset rev and the changeset revision string.""" repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) @@ -954,15 +886,6 @@ util.send_mail( frm, to, subject, body, trans.app.config ) except Exception, e: log.exception( "An error occurred sending a tool shed repository update alert by email." ) -def has_previous_repository_reviews( trans, repository, changeset_revision ): - """Determine if a repository has a changeset revision review prior to the received changeset revision.""" - repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) - reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] - for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ): - previous_changeset_revision = str( repo.changectx( changeset ) ) - if previous_changeset_revision in reviewed_revision_hashes: - return True - return False def open_repository_files_folder( trans, folder_path ): try: files_list = get_repository_files( trans, folder_path ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Additional refactoring of shed_util_common into appropriate tool shed Galaxy utility components.
by commits-noreply＠bitbucket.org 16 Mar '13

16 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e40606362e96/ changeset: e40606362e96 user: greg date: 2013-03-16 15:40:26 summary: Additional refactoring of shed_util_common into appropriate tool shed Galaxy utility components. affected #: 12 files diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -7,7 +7,7 @@ from galaxy.model.orm import or_ import tool_shed.util.shed_util_common as suc from tool_shed.util import common_install_util, data_manager_util, datatype_util, encoding_util, metadata_util -from tool_shed.util import repository_dependency_util, tool_dependency_util, tool_util +from tool_shed.util import readme_util, repository_dependency_util, tool_dependency_util, tool_util from tool_shed.galaxy_install import repository_util from galaxy.webapps.tool_shed.util import workflow_util import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids @@ -1075,7 +1075,7 @@ name, repository_owner, changeset_revision, includes_tool_dependencies, installed_repository_dependencies, \ missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies = \ common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ) - readme_files_dict = suc.get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ) + readme_files_dict = readme_util.get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ) # We're handling 1 of 2 scenarios here: (1) we're installing a tool shed repository for the first time, so we've retrieved the list of installed # and missing repository dependencies from the database (2) we're handling the scenario where an error occurred during the installation process, # so we have a tool_shed_repository record in the database with associated repository dependency records. Since we have the repository @@ -1362,7 +1362,7 @@ includes_tool_dependencies = True if 'workflows' in metadata: includes_workflows = True - readme_files_dict = suc.build_readme_files_dict( metadata ) + readme_files_dict = readme_util.build_readme_files_dict( metadata ) tool_dependencies = metadata.get( 'tool_dependencies', None ) repository_dependencies = self.get_repository_dependencies( trans=trans, repository_id=repository_id, diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -11,9 +11,9 @@ from galaxy.util import json from galaxy.model.orm import and_, or_ import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, metadata_util, repository_dependency_util, tool_dependency_util +from tool_shed.util import encoding_util, metadata_util, readme_util, repository_dependency_util, tool_dependency_util, tool_util from tool_shed.galaxy_install import repository_util -from galaxy.webapps.tool_shed.util import common_util, workflow_util +from galaxy.webapps.tool_shed.util import common_util, container_util, workflow_util import galaxy.tools import tool_shed.grids.repository_grids as repository_grids import tool_shed.grids.util as grids_util @@ -638,7 +638,7 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - repository, tool, message = suc.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) + repository, tool, message = tool_util.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) if message: status = 'error' tool_state = self.__new_state( trans ) @@ -1069,7 +1069,7 @@ changeset_revision = kwd[ 'changeset_revision' ] repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), changeset_revision ) - return suc.build_readme_files_dict( repository_metadata.metadata ) + return readme_util.build_readme_files_dict( repository_metadata.metadata ) @web.json def get_repository_dependencies( self, trans, **kwd ): @@ -1289,7 +1289,7 @@ break if 'workflows' in metadata: includes_workflows = True - readme_files_dict = suc.build_readme_files_dict( metadata ) + readme_files_dict = readme_util.build_readme_files_dict( metadata ) # See if the repo_info_dict was populated with repository_dependencies or tool_dependencies. for name, repo_info_tuple in repo_info_dict.items(): description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ @@ -1461,18 +1461,18 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'error' ) - repository, tool, error_message = suc.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) + repository, tool, error_message = tool_util.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) tool_state = self.__new_state( trans ) is_malicious = suc.changeset_is_malicious( trans, repository_id, repository.tip( trans.app ) ) invalid_file_tups = [] if tool: - invalid_file_tups = suc.check_tool_input_params( trans.app, - repository.repo_path( trans.app ), - tool_config, - tool, - [] ) + invalid_file_tups = tool_util.check_tool_input_params( trans.app, + repository.repo_path( trans.app ), + tool_config, + tool, + [] ) if invalid_file_tups: - message = suc.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, {}, as_html=True, displaying_invalid_tool=True ) + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, {}, as_html=True, displaying_invalid_tool=True ) elif error_message: message = error_message try: @@ -1719,7 +1719,7 @@ else: review_id = None can_browse_repository_reviews = suc.can_browse_repository_reviews( trans, repository ) - containers_dict = suc.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ) + containers_dict = container_util.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ) return trans.fill_template( '/webapps/tool_shed/repository/manage_repository.mako', repo_name=repo_name, description=description, @@ -1832,7 +1832,7 @@ selected_value=changeset_revision, add_id_to_name=False, downloadable=False ) - containers_dict = suc.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ) + containers_dict = container_util.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ) return trans.fill_template( '/webapps/tool_shed/repository/preview_tools_in_changeset.mako', repository=repository, containers_dict=containers_dict, @@ -1920,7 +1920,7 @@ invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, id, **kwd ) if invalid_file_tups: repository = suc.get_repository_in_tool_shed( trans, id ) - message = suc.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict ) + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict ) status = 'error' else: message = "All repository metadata has been reset. " @@ -2505,7 +2505,7 @@ review_id = trans.security.encode_id( review.id ) else: review_id = None - containers_dict = suc.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ) + containers_dict = container_util.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ) can_browse_repository_reviews = suc.can_browse_repository_reviews( trans, repository ) return trans.fill_template( '/webapps/tool_shed/repository/view_repository.mako', repo=repo, @@ -2559,21 +2559,21 @@ guid = tool_metadata_dict[ 'guid' ] full_path_to_tool_config = os.path.abspath( relative_path_to_tool_config ) full_path_to_dir, tool_config_filename = os.path.split( full_path_to_tool_config ) - can_use_disk_file = suc.can_use_tool_config_disk_file( trans, repository, repo, full_path_to_tool_config, changeset_revision ) + can_use_disk_file = tool_util.can_use_tool_config_disk_file( trans, repository, repo, full_path_to_tool_config, changeset_revision ) if can_use_disk_file: trans.app.config.tool_data_path = work_dir - tool, valid, message, sample_files = suc.handle_sample_files_and_load_tool_from_disk( trans, + tool, valid, message, sample_files = tool_util.handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, full_path_to_tool_config, work_dir ) if message: status = 'error' else: - tool, message, sample_files = suc.handle_sample_files_and_load_tool_from_tmp_config( trans, - repo, - changeset_revision, - tool_config_filename, - work_dir ) + tool, message, sample_files = tool_util.handle_sample_files_and_load_tool_from_tmp_config( trans, + repo, + changeset_revision, + tool_config_filename, + work_dir ) if message: status = 'error' break diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -3,7 +3,7 @@ from galaxy import web, util from galaxy.datatypes import checkers import tool_shed.util.shed_util_common as suc -from tool_shed.util import metadata_util, repository_dependency_util, tool_dependency_util +from tool_shed.util import metadata_util, repository_dependency_util, tool_dependency_util, tool_util from galaxy import eggs eggs.require('mercurial') @@ -153,7 +153,7 @@ if full_path.endswith( 'tool_data_table_conf.xml.sample' ): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries # to the in-memory trans.app.tool_data_tables dictionary. - error, error_message = suc.handle_sample_tool_data_table_conf_file( trans.app, full_path ) + error, error_message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, full_path ) if error: message = '%s %s' % ( message, error_message ) # See if the content of the change set was valid. @@ -213,7 +213,7 @@ message += invalid_repository_dependencies_message status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - suc.reset_tool_data_tables( trans.app ) + tool_util.reset_tool_data_tables( trans.app ) trans.response.send_redirect( web.url_for( controller='repository', action='browse_repository', id=repository_id, @@ -223,7 +223,7 @@ else: status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - suc.reset_tool_data_tables( trans.app ) + tool_util.reset_tool_data_tables( trans.app ) selected_categories = [ trans.security.decode_id( id ) for id in category_ids ] return trans.fill_template( '/webapps/tool_shed/repository/upload.mako', repository=repository, @@ -360,7 +360,7 @@ if filename_in_archive.endswith( 'tool_data_table_conf.xml.sample' ): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries # to the in-memory trans.app.tool_data_tables dictionary. - error, message = suc.handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive ) + error, message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive ) if error: return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/galaxy/webapps/tool_shed/util/container_util.py --- a/lib/galaxy/webapps/tool_shed/util/container_util.py +++ b/lib/galaxy/webapps/tool_shed/util/container_util.py @@ -1,4 +1,5 @@ -import os, logging +import os, logging, threading +from tool_shed.util import readme_util log = logging.getLogger( __name__ ) @@ -170,6 +171,34 @@ self.repository_metadata_id = repository_metadata_id self.repository_id = repository_id +def add_orphan_settings_to_tool_dependencies( tool_dependencies, orphan_tool_dependencies ): + """Inspect all received tool dependencies and label those that are orphans within the repository.""" + orphan_env_dependencies = orphan_tool_dependencies.get( 'set_environment', None ) + new_tool_dependencies = {} + if tool_dependencies: + for td_key, requirements_dict in tool_dependencies.items(): + if td_key in [ 'set_environment' ]: + # "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}] + if orphan_env_dependencies: + new_set_environment_dict_list = [] + for set_environment_dict in requirements_dict: + if set_environment_dict in orphan_env_dependencies: + set_environment_dict[ 'is_orphan' ] = True + else: + set_environment_dict[ 'is_orphan' ] = False + new_set_environment_dict_list.append( set_environment_dict ) + new_tool_dependencies[ td_key ] = new_set_environment_dict_list + else: + new_tool_dependencies[ td_key ] = requirements_dict + else: + # {"R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"} + if td_key in orphan_tool_dependencies: + requirements_dict[ 'is_orphan' ] = True + else: + requirements_dict[ 'is_orphan' ] = False + new_tool_dependencies[ td_key ] = requirements_dict + return new_tool_dependencies + def build_data_managers_folder( trans, folder_id, data_managers, label=None ): """Return a folder hierarchy containing Data Managers.""" if data_managers: @@ -199,6 +228,7 @@ else: data_managers_root_folder = None return folder_id, data_managers_root_folder + def build_datatypes_folder( trans, folder_id, datatypes, label='Datatypes' ): """Return a folder hierarchy containing datatypes.""" if datatypes: @@ -249,6 +279,7 @@ else: datatypes_root_folder = None return folder_id, datatypes_root_folder + def build_invalid_data_managers_folder( trans, folder_id, data_managers, error_messages=None, label=None ): """Return a folder hierarchy containing invalid Data Managers.""" if data_managers or error_messages: @@ -285,6 +316,7 @@ else: data_managers_root_folder = None return folder_id, data_managers_root_folder + def build_invalid_repository_dependencies_root_folder( trans, folder_id, invalid_repository_dependencies_dict ): """Return a folder hierarchy containing invalid repository dependencies.""" label = 'Invalid repository dependencies' @@ -320,6 +352,7 @@ else: invalid_repository_dependencies_root_folder = None return folder_id, invalid_repository_dependencies_root_folder + def build_invalid_tool_dependencies_root_folder( trans, folder_id, invalid_tool_dependencies_dict ): """Return a folder hierarchy containing invalid tool dependencies.""" # # INvalid tool dependencies are always packages like: @@ -358,6 +391,7 @@ else: invalid_tool_dependencies_root_folder = None return folder_id, invalid_tool_dependencies_root_folder + def build_invalid_tools_folder( trans, folder_id, invalid_tool_configs, changeset_revision, repository=None, label='Invalid tools' ): """Return a folder hierarchy containing invalid tools.""" # TODO: Should we display invalid tools on the tool panel selection page when installing the repository into Galaxy? @@ -388,6 +422,7 @@ else: invalid_tools_root_folder = None return folder_id, invalid_tools_root_folder + def build_readme_files_folder( trans, folder_id, readme_files_dict, label='Readme files' ): """Return a folder hierarchy containing readme text files.""" if readme_files_dict: @@ -415,6 +450,254 @@ else: readme_files_root_folder = None return folder_id, readme_files_root_folder + +def build_repository_containers_for_galaxy( trans, repository, datatypes, invalid_tools, missing_repository_dependencies, missing_tool_dependencies, + readme_files_dict, repository_dependencies, tool_dependencies, valid_tools, workflows, valid_data_managers, + invalid_data_managers, data_managers_errors, new_install=False, reinstalling=False ): + """Return a dictionary of containers for the received repository's dependencies and readme files for display during installation to Galaxy.""" + containers_dict = dict( datatypes=None, + invalid_tools=None, + missing_tool_dependencies=None, + readme_files=None, + repository_dependencies=None, + missing_repository_dependencies=None, + tool_dependencies=None, + valid_tools=None, + workflows=None, + valid_data_managers=None, + invalid_data_managers=None ) + # Some of the tool dependency folders will include links to display tool dependency information, and some of these links require the repository + # id. However we need to be careful because sometimes the repository object is None. + if repository: + repository_id = repository.id + changeset_revision = repository.changeset_revision + else: + repository_id = None + changeset_revision = None + lock = threading.Lock() + lock.acquire( True ) + try: + folder_id = 0 + # Datatypes container. + if datatypes: + folder_id, datatypes_root_folder = build_datatypes_folder( trans, folder_id, datatypes ) + containers_dict[ 'datatypes' ] = datatypes_root_folder + # Invalid tools container. + if invalid_tools: + folder_id, invalid_tools_root_folder = build_invalid_tools_folder( trans, + folder_id, + invalid_tools, + changeset_revision, + repository=repository, + label='Invalid tools' ) + containers_dict[ 'invalid_tools' ] = invalid_tools_root_folder + # Readme files container. + if readme_files_dict: + folder_id, readme_files_root_folder = build_readme_files_folder( trans, folder_id, readme_files_dict ) + containers_dict[ 'readme_files' ] = readme_files_root_folder + # Installed repository dependencies container. + if repository_dependencies: + if new_install: + label = 'Repository dependencies' + else: + label = 'Installed repository dependencies' + folder_id, repository_dependencies_root_folder = build_repository_dependencies_folder( trans=trans, + folder_id=folder_id, + repository_dependencies=repository_dependencies, + label=label, + installed=True ) + containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder + # Missing repository dependencies container. + if missing_repository_dependencies: + folder_id, missing_repository_dependencies_root_folder = \ + build_repository_dependencies_folder( trans=trans, + folder_id=folder_id, + repository_dependencies=missing_repository_dependencies, + label='Missing repository dependencies', + installed=False ) + containers_dict[ 'missing_repository_dependencies' ] = missing_repository_dependencies_root_folder + # Installed tool dependencies container. + if tool_dependencies: + if new_install: + label = 'Tool dependencies' + else: + label = 'Installed tool dependencies' + # We only want to display the Status column if the tool_dependency is missing. + folder_id, tool_dependencies_root_folder = build_tool_dependencies_folder( trans, + folder_id, + tool_dependencies, + label=label, + missing=False, + new_install=new_install, + reinstalling=reinstalling ) + containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder + # Missing tool dependencies container. + if missing_tool_dependencies: + # We only want to display the Status column if the tool_dependency is missing. + folder_id, missing_tool_dependencies_root_folder = build_tool_dependencies_folder( trans, + folder_id, + missing_tool_dependencies, + label='Missing tool dependencies', + missing=True, + new_install=new_install, + reinstalling=reinstalling ) + containers_dict[ 'missing_tool_dependencies' ] = missing_tool_dependencies_root_folder + # Valid tools container. + if valid_tools: + folder_id, valid_tools_root_folder = build_tools_folder( trans, + folder_id, + valid_tools, + repository, + changeset_revision, + label='Valid tools' ) + containers_dict[ 'valid_tools' ] = valid_tools_root_folder + # Workflows container. + if workflows: + folder_id, workflows_root_folder = build_workflows_folder( trans=trans, + folder_id=folder_id, + workflows=workflows, + repository_metadata_id=None, + repository_id=repository_id, + label='Workflows' ) + containers_dict[ 'workflows' ] = workflows_root_folder + if valid_data_managers: + folder_id, valid_data_managers_root_folder = build_data_managers_folder( trans=trans, + folder_id=folder_id, + data_managers=valid_data_managers, + label='Valid Data Managers' ) + containers_dict[ 'valid_data_managers' ] = valid_data_managers_root_folder + if invalid_data_managers or data_managers_errors: + folder_id, invalid_data_managers_root_folder = build_invalid_data_managers_folder( trans=trans, + folder_id=folder_id, + data_managers=invalid_data_managers, + error_messages=data_managers_errors, + label='Invalid Data Managers' ) + containers_dict[ 'invalid_data_managers' ] = invalid_data_managers_root_folder + except Exception, e: + log.debug( "Exception in build_repository_containers_for_galaxy: %s" % str( e ) ) + finally: + lock.release() + return containers_dict + +def build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ): + """Return a dictionary of containers for the received repository's dependencies and contents for display in the tool shed.""" + containers_dict = dict( datatypes=None, + invalid_tools=None, + readme_files=None, + repository_dependencies=None, + tool_dependencies=None, + valid_tools=None, + workflows=None, + valid_data_managers=None + ) + if repository_metadata: + metadata = repository_metadata.metadata + lock = threading.Lock() + lock.acquire( True ) + try: + folder_id = 0 + # Datatypes container. + if metadata: + if 'datatypes' in metadata: + datatypes = metadata[ 'datatypes' ] + folder_id, datatypes_root_folder = build_datatypes_folder( trans, folder_id, datatypes ) + containers_dict[ 'datatypes' ] = datatypes_root_folder + # Invalid repository dependencies container. + if metadata: + if 'invalid_repository_dependencies' in metadata: + invalid_repository_dependencies = metadata[ 'invalid_repository_dependencies' ] + folder_id, invalid_repository_dependencies_root_folder = \ + build_invalid_repository_dependencies_root_folder( trans, + folder_id, + invalid_repository_dependencies ) + containers_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_root_folder + # Invalid tool dependencies container. + if metadata: + if 'invalid_tool_dependencies' in metadata: + invalid_tool_dependencies = metadata[ 'invalid_tool_dependencies' ] + folder_id, invalid_tool_dependencies_root_folder = \ + build_invalid_tool_dependencies_root_folder( trans, + folder_id, + invalid_tool_dependencies ) + containers_dict[ 'invalid_tool_dependencies' ] = invalid_tool_dependencies_root_folder + # Invalid tools container. + if metadata: + if 'invalid_tools' in metadata: + invalid_tool_configs = metadata[ 'invalid_tools' ] + folder_id, invalid_tools_root_folder = build_invalid_tools_folder( trans, + folder_id, + invalid_tool_configs, + changeset_revision, + repository=repository, + label='Invalid tools' ) + containers_dict[ 'invalid_tools' ] = invalid_tools_root_folder + # Readme files container. + if metadata: + if 'readme_files' in metadata: + readme_files_dict = readme_util.build_readme_files_dict( metadata ) + folder_id, readme_files_root_folder = build_readme_files_folder( trans, folder_id, readme_files_dict ) + containers_dict[ 'readme_files' ] = readme_files_root_folder + # Repository dependencies container. + folder_id, repository_dependencies_root_folder = build_repository_dependencies_folder( trans=trans, + folder_id=folder_id, + repository_dependencies=repository_dependencies, + label='Repository dependencies', + installed=False ) + if repository_dependencies_root_folder: + containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder + # Tool dependencies container. + if metadata: + if 'tool_dependencies' in metadata: + tool_dependencies = metadata[ 'tool_dependencies' ] + if trans.webapp.name == 'tool_shed': + if 'orphan_tool_dependencies' in metadata: + orphan_tool_dependencies = metadata[ 'orphan_tool_dependencies' ] + tool_dependencies = add_orphan_settings_to_tool_dependencies( tool_dependencies, orphan_tool_dependencies ) + folder_id, tool_dependencies_root_folder = build_tool_dependencies_folder( trans, + folder_id, + tool_dependencies, + missing=False, + new_install=False ) + containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder + # Valid tools container. + if metadata: + if 'tools' in metadata: + valid_tools = metadata[ 'tools' ] + folder_id, valid_tools_root_folder = build_tools_folder( trans, + folder_id, + valid_tools, + repository, + changeset_revision, + label='Valid tools' ) + containers_dict[ 'valid_tools' ] = valid_tools_root_folder + # Workflows container. + if metadata: + if 'workflows' in metadata: + workflows = metadata[ 'workflows' ] + folder_id, workflows_root_folder = build_workflows_folder( trans=trans, + folder_id=folder_id, + workflows=workflows, + repository_metadata_id=repository_metadata.id, + repository_id=None, + label='Workflows' ) + containers_dict[ 'workflows' ] = workflows_root_folder + # Valid Data Managers container + if metadata: + if 'data_manager' in metadata: + data_managers = metadata['data_manager'].get( 'data_managers', None ) + folder_id, data_managers_root_folder = build_data_managers_folder( trans, folder_id, data_managers, label="Data Managers" ) + containers_dict[ 'valid_data_managers' ] = data_managers_root_folder + error_messages = metadata['data_manager'].get( 'error_messages', None ) + data_managers = metadata['data_manager'].get( 'invalid_data_managers', None ) + folder_id, data_managers_root_folder = build_invalid_data_managers_folder( trans, folder_id, data_managers, error_messages, label="Invalid Data Managers" ) + containers_dict[ 'invalid_data_managers' ] = data_managers_root_folder + + except Exception, e: + log.debug( "Exception in build_repository_containers_for_tool_shed: %s" % str( e ) ) + finally: + lock.release() + return containers_dict + def build_repository_dependencies_folder( trans, folder_id, repository_dependencies, label='Repository dependencies', installed=False ): """Return a folder hierarchy containing repository dependencies.""" if repository_dependencies: @@ -438,6 +721,7 @@ else: repository_dependencies_root_folder = None return folder_id, repository_dependencies_root_folder + def build_tools_folder( trans, folder_id, tool_dicts, repository, changeset_revision, valid=True, label='Valid tools' ): """Return a folder hierarchy containing valid tools.""" if tool_dicts: @@ -494,6 +778,7 @@ else: tools_root_folder = None return folder_id, tools_root_folder + def build_tool_dependencies_folder( trans, folder_id, tool_dependencies, label='Tool dependencies', missing=False, new_install=False, reinstalling=False ): """Return a folder hierarchy containing tool dependencies.""" # When we're in Galaxy (not the tool shed) and the tool dependencies are not installed or are in an error state, they are considered missing. The tool @@ -603,6 +888,7 @@ else: tool_dependencies_root_folder = None return folder_id, tool_dependencies_root_folder + def build_workflows_folder( trans, folder_id, workflows, repository_metadata_id=None, repository_id=None, label='Workflows' ): """ Return a folder hierarchy containing workflow objects for each workflow dictionary in the received workflows list. When @@ -646,6 +932,7 @@ else: workflows_root_folder = None return folder_id, workflows_root_folder + def cast_empty_repository_dependency_folders( folder, repository_dependency_id ): """ Change any empty folders contained within the repository dependencies container into a repository dependency since it has no repository dependencies @@ -660,6 +947,7 @@ for sub_folder in folder.folders: return cast_empty_repository_dependency_folders( sub_folder, repository_dependency_id ) return folder, repository_dependency_id + def generate_repository_dependencies_folder_label_from_key( repository_name, repository_owner, changeset_revision, key ): """Return a repository dependency label based on the repository dependency key.""" if key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, key ): @@ -667,6 +955,7 @@ else: label = "Repository %s revision %s owned by %s" % ( repository_name, changeset_revision, repository_owner ) return label + def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner, changeset_revision ): # FIXME: assumes tool shed is current tool shed since repository dependencies across tool sheds is not yet supported. return '%s%s%s%s%s%s%s' % ( str( toolshed_base_url ).rstrip( '/' ), @@ -676,14 +965,17 @@ str( repository_owner ), STRSEP, str( changeset_revision ) ) + def generate_tool_dependencies_key( name, version, type ): return '%s%s%s%s%s' % ( str( name ), STRSEP, str( version ), STRSEP, str( type ) ) + def get_folder( folder, key ): if folder.key == key: return folder for sub_folder in folder.folders: return get_folder( sub_folder, key ) return None + def get_components_from_key( key ): # FIXME: assumes tool shed is current tool shed since repository dependencies across tool sheds is not yet supported. items = key.split( STRSEP ) @@ -692,6 +984,7 @@ repository_owner = items[ 2 ] changeset_revision = items[ 3 ] return toolshed_base_url, repository_name, repository_owner, changeset_revision + def handle_repository_dependencies_container_entry( trans, repository_dependencies_folder, rd_key, rd_value, folder_id, repository_dependency_id, folder_keys ): toolshed, repository_name, repository_owner, changeset_revision = get_components_from_key( rd_key ) folder = get_folder( repository_dependencies_folder, rd_key ) @@ -744,6 +1037,7 @@ # Insert the repository_dependency into the folder. sub_folder.repository_dependencies.append( repository_dependency ) return repository_dependencies_folder, folder_id, repository_dependency_id + def is_subfolder_of( folder, repository_dependency ): toolshed, repository_name, repository_owner, changeset_revision = repository_dependency key = generate_repository_dependencies_key_for_repository( toolshed, repository_name, repository_owner, changeset_revision ) @@ -751,15 +1045,18 @@ if key == sub_folder.key: return True return False + def key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, key ): toolshed_base_url, key_name, key_owner, key_changeset_revision = get_components_from_key( key ) return repository_name == key_name and repository_owner == key_owner and changeset_revision == key_changeset_revision + def populate_repository_dependencies_container( trans, repository_dependencies_folder, repository_dependencies, folder_id, repository_dependency_id ): folder_keys = repository_dependencies.keys() for key, value in repository_dependencies.items(): repository_dependencies_folder, folder_id, repository_dependency_id = \ handle_repository_dependencies_container_entry( trans, repository_dependencies_folder, key, value, folder_id, repository_dependency_id, folder_keys ) return repository_dependencies_folder, folder_id, repository_dependency_id + def print_folders( pad, folder ): # For debugging... pad_str = '' @@ -770,6 +1067,7 @@ print ' %s%s' % ( pad_str, repository_dependency.listify ) for sub_folder in folder.folders: print_folders( pad+5, sub_folder ) + def prune_repository_dependencies( folder ): """ Since the object used to generate a repository dependencies container is a dictionary and not an odict() (it must be json-serialize-able), the diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/galaxy/webapps/tool_shed/util/workflow_util.py --- a/lib/galaxy/webapps/tool_shed/util/workflow_util.py +++ b/lib/galaxy/webapps/tool_shed/util/workflow_util.py @@ -6,7 +6,7 @@ import logging, svgfig from galaxy.util import json import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, metadata_util +from tool_shed.util import encoding_util, metadata_util, tool_util from galaxy.workflow.modules import InputDataModule, ToolModule, WorkflowModuleFactory import galaxy.webapps.galaxy.controllers.workflow import galaxy.tools @@ -52,7 +52,7 @@ if self.tool_id in [ tool_dict[ 'id' ], tool_dict[ 'guid' ] ]: if trans.webapp.name == 'tool_shed': # We're in the tool shed. - repository, self.tool, message = suc.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_dict[ 'tool_config' ] ) + repository, self.tool, message = tool_util.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_dict[ 'tool_config' ] ) if message and self.tool is None: self.errors = 'unavailable' break diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -203,22 +203,22 @@ repository_missing_tool_dependencies=missing_tool_dependencies, required_repo_info_dicts=None ) # Since we are installing a new repository, most of the repository contents are set to None since we don't yet know what they are. - containers_dict = suc.build_repository_containers_for_galaxy( trans=trans, - repository=None, - datatypes=None, - invalid_tools=None, - missing_repository_dependencies=missing_repository_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=readme_files_dict, - repository_dependencies=installed_repository_dependencies, - tool_dependencies=installed_tool_dependencies, - valid_tools=None, - workflows=None, - valid_data_managers=None, - invalid_data_managers=None, - data_managers_errors=None, - new_install=True, - reinstalling=False ) + containers_dict = container_util.build_repository_containers_for_galaxy( trans=trans, + repository=None, + datatypes=None, + invalid_tools=None, + missing_repository_dependencies=missing_repository_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + readme_files_dict=readme_files_dict, + repository_dependencies=installed_repository_dependencies, + tool_dependencies=installed_tool_dependencies, + valid_tools=None, + workflows=None, + valid_data_managers=None, + invalid_data_managers=None, + data_managers_errors=None, + new_install=True, + reinstalling=False ) # Merge the missing_repository_dependencies container contents to the installed_repository_dependencies container. containers_dict = repository_dependency_util.merge_missing_repository_dependencies_to_installed_container( containers_dict ) # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -1,8 +1,9 @@ -import os, logging +import os, logging, urllib2 import tool_shed.util.shed_util_common as suc -from tool_shed.util import data_manager_util, datatype_util, tool_util +from tool_shed.util import encoding_util, data_manager_util, datatype_util, tool_util from tool_shed.galaxy_install.tool_dependencies.install_util import install_package, set_environment from galaxy import util +from galaxy.util import json from galaxy.webapps.tool_shed.util import container_util from galaxy import eggs @@ -78,7 +79,7 @@ else: installed_rd, missing_rd = get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple ) # Discover all repository dependencies and retrieve information for installing them. - required_repo_info_dicts = suc.get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) + required_repo_info_dicts = get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) # Display tool dependencies defined for each of the repository dependencies. if required_repo_info_dicts: all_tool_dependencies = {} @@ -234,6 +235,55 @@ missing_tool_dependencies = None return tool_dependencies, missing_tool_dependencies +def get_required_repo_info_dicts( tool_shed_url, repo_info_dicts ): + """ + Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of them to the list. All + repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through + this method is required to retrieve all repository dependencies. + """ + all_repo_info_dicts = [] + if repo_info_dicts: + # We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids. + required_repository_tups = [] + for repo_info_dict in repo_info_dicts: + for repository_name, repo_info_tup in repo_info_dict.items(): + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ + suc.get_repo_info_tuple_contents( repo_info_tup ) + if repository_dependencies: + for key, val in repository_dependencies.items(): + if key in [ 'root_key', 'description' ]: + continue + toolshed, name, owner, changeset_revision = container_util.get_components_from_key( key ) + components_list = [ toolshed, name, owner, changeset_revision ] + if components_list not in required_repository_tups: + required_repository_tups.append( components_list ) + for components_list in val: + if components_list not in required_repository_tups: + required_repository_tups.append( components_list ) + if required_repository_tups: + # The value of required_repository_tups is a list of tuples, so we need to encode it. + encoded_required_repository_tups = [] + for required_repository_tup in required_repository_tups: + encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) ) + encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups ) + encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str ) + url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict?encoded_str=%s' % encoded_required_repository_str ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + required_repo_info_dict = json.from_json_string( text ) + required_repo_info_dicts = [] + encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ] + for encoded_dict_str in encoded_dict_strings: + decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str ) + required_repo_info_dicts.append( decoded_dict ) + if required_repo_info_dicts: + for required_repo_info_dict in required_repo_info_dicts: + if required_repo_info_dict not in all_repo_info_dicts: + all_repo_info_dicts.append( required_repo_info_dict ) + return all_repo_info_dicts + def handle_tool_dependencies( app, tool_shed_repository, tool_dependencies_config, tool_dependencies ): """ Install and build tool dependencies defined in the tool_dependencies_config. This config's tag sets can currently refer to installation diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -5,7 +5,7 @@ from galaxy.datatypes import checkers from galaxy.tools.data_manager.manager import DataManager import tool_shed.util.shed_util_common as suc -from tool_shed.util import common_install_util, tool_dependency_util +from tool_shed.util import common_install_util, readme_util, tool_dependency_util, tool_util from galaxy.model.orm import and_ from galaxy import eggs @@ -417,7 +417,7 @@ # Parse the tool_config to get the guid. tool_config_path = suc.get_config_from_disk( tool_config, repository_files_dir ) full_path = os.path.abspath( tool_config_path ) - tool, valid, error_message = suc.load_tool_from_config( app, full_path ) + tool, valid, error_message = tool_util.load_tool_from_config( app, full_path ) if tool is None: guid = None else: @@ -533,7 +533,7 @@ metadata_dict[ 'sample_files' ] = sample_file_metadata_paths # Copy all sample files included in the repository to a single directory location so we can load tools that depend on them. for sample_file in sample_file_copy_paths: - suc.copy_sample_file( app, sample_file, dest_path=work_dir ) + tool_util.copy_sample_file( app, sample_file, dest_path=work_dir ) # If the list of sample files includes a tool_data_table_conf.xml.sample file, laad it's table elements into memory. relative_path, filename = os.path.split( sample_file ) if filename == 'tool_data_table_conf.xml.sample': @@ -578,13 +578,13 @@ log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) ) is_tool = False if is_tool: - tool, valid, error_message = suc.load_tool_from_config( app, full_path ) + tool, valid, error_message = tool_util.load_tool_from_config( app, full_path ) if tool is None: if not valid: invalid_tool_configs.append( name ) invalid_file_tups.append( ( name, error_message ) ) else: - invalid_files_and_errors_tups = suc.check_tool_input_params( app, files_dir, name, tool, sample_file_copy_paths ) + invalid_files_and_errors_tups = tool_util.check_tool_input_params( app, files_dir, name, tool, sample_file_copy_paths ) can_set_metadata = True for tup in invalid_files_and_errors_tups: if name in tup: @@ -1370,7 +1370,7 @@ response.close() readme_files_dict = json.from_json_string( raw_text ) else: - readme_files_dict = suc.build_readme_files_dict( repository.metadata, tool_path ) + readme_files_dict = readme_util.build_readme_files_dict( repository.metadata, tool_path ) else: readme_files_dict = None # Handle repository dependencies. @@ -1403,22 +1403,22 @@ valid_data_managers = metadata['data_manager'].get( 'data_managers', None ) invalid_data_managers = metadata['data_manager'].get( 'invalid_data_managers', None ) data_managers_errors = metadata['data_manager'].get( 'messages', None ) - containers_dict = suc.build_repository_containers_for_galaxy( trans=trans, - repository=repository, - datatypes=datatypes, - invalid_tools=invalid_tools, - missing_repository_dependencies=missing_repository_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=readme_files_dict, - repository_dependencies=installed_repository_dependencies, - tool_dependencies=installed_tool_dependencies, - valid_tools=valid_tools, - workflows=workflows, - valid_data_managers=valid_data_managers, - invalid_data_managers=invalid_data_managers, - data_managers_errors=data_managers_errors, - new_install=False, - reinstalling=reinstalling ) + containers_dict = container_util.build_repository_containers_for_galaxy( trans=trans, + repository=repository, + datatypes=datatypes, + invalid_tools=invalid_tools, + missing_repository_dependencies=missing_repository_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + readme_files_dict=readme_files_dict, + repository_dependencies=installed_repository_dependencies, + tool_dependencies=installed_tool_dependencies, + valid_tools=valid_tools, + workflows=workflows, + valid_data_managers=valid_data_managers, + invalid_data_managers=invalid_data_managers, + data_managers_errors=data_managers_errors, + new_install=False, + reinstalling=reinstalling ) else: containers_dict = dict( datatypes=None, invalid_tools=None, @@ -1583,7 +1583,7 @@ # Set tool version information for all downloadable changeset revisions. Get the list of changeset revisions from the changelog. reset_all_tool_versions( trans, id, repo ) # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - suc.reset_tool_data_tables( trans.app ) + tool_util.reset_tool_data_tables( trans.app ) return invalid_file_tups, metadata_dict def reset_metadata_on_selected_repositories( trans, **kwd ): @@ -1608,7 +1608,7 @@ repository = suc.get_installed_tool_shed_repository( trans, repository_id ) invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( trans, repository_id ) if invalid_file_tups: - message = suc.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) log.debug( message ) unsuccessful_count += 1 else: @@ -1733,10 +1733,10 @@ message += "be defined so this revision cannot be automatically installed into a local Galaxy instance." status = "error" if invalid_file_tups: - message = suc.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict ) + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict ) status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - suc.reset_tool_data_tables( trans.app ) + tool_util.reset_tool_data_tables( trans.app ) return message, status def set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=None, **kwd ): diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/tool_shed/util/readme_util.py --- /dev/null +++ b/lib/tool_shed/util/readme_util.py @@ -0,0 +1,40 @@ +import os, logging, urllib2 +import tool_shed.util.shed_util_common as suc +from galaxy.util import json + +log = logging.getLogger( __name__ ) + +def build_readme_files_dict( metadata, tool_path=None ): + """Return a dictionary of valid readme file name <-> readme file content pairs for all readme files contained in the received metadata.""" + readme_files_dict = {} + if metadata: + if 'readme_files' in metadata: + for relative_path_to_readme_file in metadata[ 'readme_files' ]: + readme_file_name = os.path.split( relative_path_to_readme_file )[ 1 ] + if tool_path: + full_path_to_readme_file = os.path.abspath( os.path.join( tool_path, relative_path_to_readme_file ) ) + else: + full_path_to_readme_file = os.path.abspath( relative_path_to_readme_file ) + try: + f = open( full_path_to_readme_file, 'r' ) + text = f.read() + f.close() + readme_files_dict[ readme_file_name ] = suc.translate_string( text, to_html=False ) + except Exception, e: + log.debug( "Error reading README file '%s' defined in metadata: %s" % ( str( relative_path_to_readme_file ), str( e ) ) ) + return readme_files_dict + +def get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ): + """Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section selection page.""" + name = repo_info_dict.keys()[ 0 ] + repo_info_tuple = repo_info_dict[ name ] + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = suc.get_repo_info_tuple_contents( repo_info_tuple ) + # Handle README files. + url = suc.url_join( tool_shed_url, + 'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' % \ + ( name, repository_owner, changeset_revision ) ) + response = urllib2.urlopen( url ) + raw_text = response.read() + response.close() + readme_files_dict = json.from_json_string( raw_text ) + return readme_files_dict diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -104,7 +104,7 @@ filtered_repo_info_dicts = [] # Discover all repository dependencies and retrieve information for installing them. Even if the user elected to not install repository dependencies we have # to make sure all repository dependency objects exist so that the appropriate repository dependency relationships can be built. - all_repo_info_dicts = suc.get_required_repo_info_dicts( tool_shed_url, repo_info_dicts ) + all_repo_info_dicts = common_install_util.get_required_repo_info_dicts( tool_shed_url, repo_info_dicts ) if not all_repo_info_dicts: # No repository dependencies were discovered so process the received repositories. all_repo_info_dicts = [ rid for rid in repo_info_dicts ] diff -r 8703329f3716e7c4603e3a3a57286f67bc3b89af -r e40606362e962b4c452960c7c5f9b33b5b13f96c lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -1,18 +1,14 @@ -import os, shutil, tempfile, logging, string, threading, urllib2, filecmp +import os, shutil, tempfile, logging, string, urllib2 from datetime import datetime from time import gmtime, strftime from galaxy import util -from galaxy.tools import parameters from galaxy.util import json from galaxy.util.odict import odict from galaxy.web import url_for from galaxy.web.form_builder import SelectField -from galaxy.webapps.tool_shed.util import container_util from galaxy.datatypes import checkers from galaxy.model.orm import and_ import sqlalchemy.orm.exc -from galaxy.tools.parameters import dynamic_options -from tool_shed.util import encoding_util from galaxy import eggs import pkg_resources @@ -89,297 +85,6 @@ '${host}' """ -def add_orphan_settings_to_tool_dependencies( tool_dependencies, orphan_tool_dependencies ): - """Inspect all received tool dependencies and label those that are orphans within the repository.""" - orphan_env_dependencies = orphan_tool_dependencies.get( 'set_environment', None ) - new_tool_dependencies = {} - if tool_dependencies: - for td_key, requirements_dict in tool_dependencies.items(): - if td_key in [ 'set_environment' ]: - # "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}] - if orphan_env_dependencies: - new_set_environment_dict_list = [] - for set_environment_dict in requirements_dict: - if set_environment_dict in orphan_env_dependencies: - set_environment_dict[ 'is_orphan' ] = True - else: - set_environment_dict[ 'is_orphan' ] = False - new_set_environment_dict_list.append( set_environment_dict ) - new_tool_dependencies[ td_key ] = new_set_environment_dict_list - else: - new_tool_dependencies[ td_key ] = requirements_dict - else: - # {"R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"} - if td_key in orphan_tool_dependencies: - requirements_dict[ 'is_orphan' ] = True - else: - requirements_dict[ 'is_orphan' ] = False - new_tool_dependencies[ td_key ] = requirements_dict - return new_tool_dependencies -def build_readme_files_dict( metadata, tool_path=None ): - """Return a dictionary of valid readme file name <-> readme file content pairs for all readme files contained in the received metadata.""" - readme_files_dict = {} - if metadata: - if 'readme_files' in metadata: - for relative_path_to_readme_file in metadata[ 'readme_files' ]: - readme_file_name = os.path.split( relative_path_to_readme_file )[ 1 ] - if tool_path: - full_path_to_readme_file = os.path.abspath( os.path.join( tool_path, relative_path_to_readme_file ) ) - else: - full_path_to_readme_file = os.path.abspath( relative_path_to_readme_file ) - try: - f = open( full_path_to_readme_file, 'r' ) - text = f.read() - f.close() - readme_files_dict[ readme_file_name ] = translate_string( text, to_html=False ) - except Exception, e: - log.debug( "Error reading README file '%s' defined in metadata: %s" % ( str( relative_path_to_readme_file ), str( e ) ) ) - return readme_files_dict -def build_repository_containers_for_galaxy( trans, repository, datatypes, invalid_tools, missing_repository_dependencies, missing_tool_dependencies, - readme_files_dict, repository_dependencies, tool_dependencies, valid_tools, workflows, valid_data_managers, - invalid_data_managers, data_managers_errors, new_install=False, reinstalling=False ): - """Return a dictionary of containers for the received repository's dependencies and readme files for display during installation to Galaxy.""" - containers_dict = dict( datatypes=None, - invalid_tools=None, - missing_tool_dependencies=None, - readme_files=None, - repository_dependencies=None, - missing_repository_dependencies=None, - tool_dependencies=None, - valid_tools=None, - workflows=None, - valid_data_managers=None, - invalid_data_managers=None ) - # Some of the tool dependency folders will include links to display tool dependency information, and some of these links require the repository - # id. However we need to be careful because sometimes the repository object is None. - if repository: - repository_id = repository.id - changeset_revision = repository.changeset_revision - else: - repository_id = None - changeset_revision = None - lock = threading.Lock() - lock.acquire( True ) - try: - folder_id = 0 - # Datatypes container. - if datatypes: - folder_id, datatypes_root_folder = container_util.build_datatypes_folder( trans, folder_id, datatypes ) - containers_dict[ 'datatypes' ] = datatypes_root_folder - # Invalid tools container. - if invalid_tools: - folder_id, invalid_tools_root_folder = container_util.build_invalid_tools_folder( trans, - folder_id, - invalid_tools, - changeset_revision, - repository=repository, - label='Invalid tools' ) - containers_dict[ 'invalid_tools' ] = invalid_tools_root_folder - # Readme files container. - if readme_files_dict: - folder_id, readme_files_root_folder = container_util.build_readme_files_folder( trans, folder_id, readme_files_dict ) - containers_dict[ 'readme_files' ] = readme_files_root_folder - # Installed repository dependencies container. - if repository_dependencies: - if new_install: - label = 'Repository dependencies' - else: - label = 'Installed repository dependencies' - folder_id, repository_dependencies_root_folder = container_util.build_repository_dependencies_folder( trans=trans, - folder_id=folder_id, - repository_dependencies=repository_dependencies, - label=label, - installed=True ) - containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder - # Missing repository dependencies container. - if missing_repository_dependencies: - folder_id, missing_repository_dependencies_root_folder = \ - container_util.build_repository_dependencies_folder( trans=trans, - folder_id=folder_id, - repository_dependencies=missing_repository_dependencies, - label='Missing repository dependencies', - installed=False ) - containers_dict[ 'missing_repository_dependencies' ] = missing_repository_dependencies_root_folder - # Installed tool dependencies container. - if tool_dependencies: - if new_install: - label = 'Tool dependencies' - else: - label = 'Installed tool dependencies' - # We only want to display the Status column if the tool_dependency is missing. - folder_id, tool_dependencies_root_folder = container_util.build_tool_dependencies_folder( trans, - folder_id, - tool_dependencies, - label=label, - missing=False, - new_install=new_install, - reinstalling=reinstalling ) - containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder - # Missing tool dependencies container. - if missing_tool_dependencies: - # We only want to display the Status column if the tool_dependency is missing. - folder_id, missing_tool_dependencies_root_folder = container_util.build_tool_dependencies_folder( trans, - folder_id, - missing_tool_dependencies, - label='Missing tool dependencies', - missing=True, - new_install=new_install, - reinstalling=reinstalling ) - containers_dict[ 'missing_tool_dependencies' ] = missing_tool_dependencies_root_folder - # Valid tools container. - if valid_tools: - folder_id, valid_tools_root_folder = container_util.build_tools_folder( trans, - folder_id, - valid_tools, - repository, - changeset_revision, - label='Valid tools' ) - containers_dict[ 'valid_tools' ] = valid_tools_root_folder - # Workflows container. - if workflows: - folder_id, workflows_root_folder = container_util.build_workflows_folder( trans=trans, - folder_id=folder_id, - workflows=workflows, - repository_metadata_id=None, - repository_id=repository_id, - label='Workflows' ) - containers_dict[ 'workflows' ] = workflows_root_folder - if valid_data_managers: - folder_id, valid_data_managers_root_folder = container_util.build_data_managers_folder( trans=trans, - folder_id=folder_id, - data_managers=valid_data_managers, - label='Valid Data Managers' ) - containers_dict[ 'valid_data_managers' ] = valid_data_managers_root_folder - if invalid_data_managers or data_managers_errors: - folder_id, invalid_data_managers_root_folder = container_util.build_invalid_data_managers_folder( trans=trans, - folder_id=folder_id, - data_managers=invalid_data_managers, - error_messages=data_managers_errors, - label='Invalid Data Managers' ) - containers_dict[ 'invalid_data_managers' ] = invalid_data_managers_root_folder - except Exception, e: - log.debug( "Exception in build_repository_containers_for_galaxy: %s" % str( e ) ) - finally: - lock.release() - return containers_dict -def build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ): - """Return a dictionary of containers for the received repository's dependencies and contents for display in the tool shed.""" - containers_dict = dict( datatypes=None, - invalid_tools=None, - readme_files=None, - repository_dependencies=None, - tool_dependencies=None, - valid_tools=None, - workflows=None, - valid_data_managers=None - ) - if repository_metadata: - metadata = repository_metadata.metadata - lock = threading.Lock() - lock.acquire( True ) - try: - folder_id = 0 - # Datatypes container. - if metadata: - if 'datatypes' in metadata: - datatypes = metadata[ 'datatypes' ] - folder_id, datatypes_root_folder = container_util.build_datatypes_folder( trans, folder_id, datatypes ) - containers_dict[ 'datatypes' ] = datatypes_root_folder - # Invalid repository dependencies container. - if metadata: - if 'invalid_repository_dependencies' in metadata: - invalid_repository_dependencies = metadata[ 'invalid_repository_dependencies' ] - folder_id, invalid_repository_dependencies_root_folder = \ - container_util.build_invalid_repository_dependencies_root_folder( trans, - folder_id, - invalid_repository_dependencies ) - containers_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_root_folder - # Invalid tool dependencies container. - if metadata: - if 'invalid_tool_dependencies' in metadata: - invalid_tool_dependencies = metadata[ 'invalid_tool_dependencies' ] - folder_id, invalid_tool_dependencies_root_folder = \ - container_util.build_invalid_tool_dependencies_root_folder( trans, - folder_id, - invalid_tool_dependencies ) - containers_dict[ 'invalid_tool_dependencies' ] = invalid_tool_dependencies_root_folder - # Invalid tools container. - if metadata: - if 'invalid_tools' in metadata: - invalid_tool_configs = metadata[ 'invalid_tools' ] - folder_id, invalid_tools_root_folder = container_util.build_invalid_tools_folder( trans, - folder_id, - invalid_tool_configs, - changeset_revision, - repository=repository, - label='Invalid tools' ) - containers_dict[ 'invalid_tools' ] = invalid_tools_root_folder - # Readme files container. - if metadata: - if 'readme_files' in metadata: - readme_files_dict = build_readme_files_dict( metadata ) - folder_id, readme_files_root_folder = container_util.build_readme_files_folder( trans, folder_id, readme_files_dict ) - containers_dict[ 'readme_files' ] = readme_files_root_folder - # Repository dependencies container. - folder_id, repository_dependencies_root_folder = container_util.build_repository_dependencies_folder( trans=trans, - folder_id=folder_id, - repository_dependencies=repository_dependencies, - label='Repository dependencies', - installed=False ) - if repository_dependencies_root_folder: - containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder - # Tool dependencies container. - if metadata: - if 'tool_dependencies' in metadata: - tool_dependencies = metadata[ 'tool_dependencies' ] - if trans.webapp.name == 'tool_shed': - if 'orphan_tool_dependencies' in metadata: - orphan_tool_dependencies = metadata[ 'orphan_tool_dependencies' ] - tool_dependencies = add_orphan_settings_to_tool_dependencies( tool_dependencies, orphan_tool_dependencies ) - folder_id, tool_dependencies_root_folder = container_util.build_tool_dependencies_folder( trans, - folder_id, - tool_dependencies, - missing=False, - new_install=False ) - containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder - # Valid tools container. - if metadata: - if 'tools' in metadata: - valid_tools = metadata[ 'tools' ] - folder_id, valid_tools_root_folder = container_util.build_tools_folder( trans, - folder_id, - valid_tools, - repository, - changeset_revision, - label='Valid tools' ) - containers_dict[ 'valid_tools' ] = valid_tools_root_folder - # Workflows container. - if metadata: - if 'workflows' in metadata: - workflows = metadata[ 'workflows' ] - folder_id, workflows_root_folder = container_util.build_workflows_folder( trans=trans, - folder_id=folder_id, - workflows=workflows, - repository_metadata_id=repository_metadata.id, - repository_id=None, - label='Workflows' ) - containers_dict[ 'workflows' ] = workflows_root_folder - # Valid Data Managers container - if metadata: - if 'data_manager' in metadata: - data_managers = metadata['data_manager'].get( 'data_managers', None ) - folder_id, data_managers_root_folder = container_util.build_data_managers_folder( trans, folder_id, data_managers, label="Data Managers" ) - containers_dict[ 'valid_data_managers' ] = data_managers_root_folder - error_messages = metadata['data_manager'].get( 'error_messages', None ) - data_managers = metadata['data_manager'].get( 'invalid_data_managers', None ) - folder_id, data_managers_root_folder = container_util.build_invalid_data_managers_folder( trans, folder_id, data_managers, error_messages, label="Invalid Data Managers" ) - containers_dict[ 'invalid_data_managers' ] = data_managers_root_folder - - except Exception, e: - log.debug( "Exception in build_repository_containers_for_tool_shed: %s" % str( e ) ) - finally: - lock.release() - return containers_dict def build_repository_ids_select_field( trans, name='repository_ids', multiple=True, display='checkboxes' ): """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) @@ -412,25 +117,6 @@ if trans.app.security_agent.user_can_browse_component_review( trans.app, repository, component_review, user ): return True return False -def can_use_tool_config_disk_file( trans, repository, repo, file_path, changeset_revision ): - """ - Determine if repository's tool config file on disk can be used. This method is restricted to tool config files since, with the - exception of tool config files, multiple files with the same name will likely be in various directories in the repository and we're - comparing file names only (not relative paths). - """ - if not file_path or not os.path.exists( file_path ): - # The file no longer exists on disk, so it must have been deleted at some previous point in the change log. - return False - if changeset_revision == repository.tip( trans.app ): - return True - file_name = strip_path( file_path ) - latest_version_of_file = get_latest_tool_config_revision_from_repository_manifest( repo, file_name, changeset_revision ) - can_use_disk_file = filecmp.cmp( file_path, latest_version_of_file ) - try: - os.unlink( latest_version_of_file ) - except: - pass - return can_use_disk_file def changeset_is_malicious( trans, id, changeset_revision, **kwd ): """Check the malicious flag in repository metadata for a specified change set""" repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) @@ -450,50 +136,6 @@ if review.changeset_revision == changeset_revision and review.user == user: return True return False -def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files ): - """ - Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make - sure the files exist. - """ - invalid_files_and_errors_tups = [] - correction_msg = '' - for input_param in tool.input_params: - if isinstance( input_param, parameters.basic.SelectToolParameter ) and input_param.is_dynamic: - # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, make sure all requirements exist. - options = input_param.dynamic_options or input_param.options - if options and isinstance( options, dynamic_options.DynamicOptions ): - if options.tool_data_table or options.missing_tool_data_table_name: - # Make sure the repository contains a tool_data_table_conf.xml.sample file. - sample_tool_data_table_conf = get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) - if sample_tool_data_table_conf: - error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf ) - if error: - invalid_files_and_errors_tups.append( ( 'tool_data_table_conf.xml.sample', correction_msg ) ) - else: - options.missing_tool_data_table_name = None - else: - correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. Upload a file named tool_data_table_conf.xml.sample " - correction_msg += "to the repository that includes the required entry to correct this error. " - invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) - if options.index_file or options.missing_index_file: - # Make sure the repository contains the required xxx.loc.sample file. - index_file = options.index_file or options.missing_index_file - index_file_name = strip_path( index_file ) - sample_found = False - for sample_file in sample_files: - sample_file_name = strip_path( sample_file ) - if sample_file_name == '%s.sample' % index_file_name: - options.index_file = index_file_name - options.missing_index_file = None - if options.tool_data_table: - options.tool_data_table.missing_index_file = None - sample_found = True - break - if not sample_found: - correction_msg = "This file refers to a file named %s. " % str( index_file_name ) - correction_msg += "Upload a file named %s.sample to the repository to correct this error." % str( index_file_name ) - invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) - return invalid_files_and_errors_tups def clean_repository_clone_url( repository_clone_url ): if repository_clone_url.find( '@' ) > 0: # We have an url that includes an authenticated user, something like: @@ -527,17 +169,6 @@ error_message = 'Error cloning repository: %s' % str( e ) log.debug( error_message ) return False, error_message -def concat_messages( msg1, msg2 ): - if msg1: - if msg2: - message = '%s %s' % ( msg1, msg2 ) - else: - message = msg1 - elif msg2: - message = msg2 - else: - message = '' - return message def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): # Persist the current in-memory list of config_elems to a file named by the value of config_filename. fd, filename = tempfile.mkstemp() @@ -549,17 +180,6 @@ os.close( fd ) shutil.move( filename, os.path.abspath( config_filename ) ) os.chmod( config_filename, 0644 ) -def copy_disk_sample_files_to_dir( trans, repo_files_dir, dest_path ): - """Copy all files currently on disk that end with the .sample extension to the directory to which dest_path refers.""" - sample_files = [] - for root, dirs, files in os.walk( repo_files_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name.endswith( '.sample' ): - relative_path = os.path.join( root, name ) - copy_sample_file( trans.app, relative_path, dest_path=dest_path ) - sample_files.append( name ) - return sample_files def copy_file_from_manifest( repo, ctx, filename, dir ): """Copy the latest version of the file named filename from the repository manifest to the directory to which dir refers.""" for changeset in reversed_upper_bounded_changelog( repo, ctx ): @@ -572,21 +192,6 @@ fh.close() return file_path return None -def copy_sample_file( app, filename, dest_path=None ): - """Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path is ~/tool-data.""" - if dest_path is None: - dest_path = os.path.abspath( app.config.tool_data_path ) - sample_file_name = strip_path( filename ) - copied_file = sample_file_name.replace( '.sample', '' ) - full_source_path = os.path.abspath( filename ) - full_destination_path = os.path.join( dest_path, sample_file_name ) - # Don't copy a file to itself - not sure how this happens, but sometimes it does... - if full_source_path != full_destination_path: - # It's ok to overwrite the .sample version of the file. - shutil.copy( full_source_path, full_destination_path ) - # Only create the .loc file if it does not yet exist. We don't overwrite it in case it contains stuff proprietary to the local instance. - if not os.path.exists( os.path.join( dest_path, copied_file ) ): - shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): # The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used @@ -642,20 +247,6 @@ sa_session.add( tool_shed_repository ) sa_session.flush() return tool_shed_repository -def ensure_required_repositories_exist_for_reinstall( trans, repository_dependencies ): - """ - Inspect the received repository_dependencies dictionary and make sure tool_shed_repository objects exist in the database for each entry. These - tool_shed_repositories do not necessarily have to exist on disk, and if they do not, their status will be marked appropriately. They must exist - in the database in order for repository dependency relationships to be properly built. - """ - for key, val in repository_dependencies.items(): - if key in [ 'root_key', 'description' ]: - continue - tool_shed, name, owner, changeset_revision = container_util.get_components_from_key( key ) - repository = get_or_create_tool_shed_repository( trans, tool_shed, name, owner, changeset_revision ) - for repository_components_list in val: - tool_shed, name, owner, changeset_revision = repository_components_list - repository = get_or_create_tool_shed_repository( trans, tool_shed, name, owner, changeset_revision ) def generate_clone_url_for_installed_repository( app, repository ): """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" tool_shed_url = get_url_from_repository_tool_shed( app, repository ) @@ -675,42 +266,6 @@ toolshed, name, owner, changeset_revision = repo_info_tup # Don't include the changeset_revision in clone urls. return url_join( toolshed, 'repos', owner, name ) -def generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): - if as_html: - new_line = ' ' - bold_start = '' - bold_end = '' - else: - new_line = '\n' - bold_start = '' - bold_end = '' - message = '' - if not displaying_invalid_tool: - if metadata_dict: - message += "Metadata may have been defined for some items in revision '%s'. " % str( repository.tip( trans.app ) ) - message += "Correct the following problems if necessary and reset metadata.%s" % new_line - else: - message += "Metadata cannot be defined for revision '%s' so this revision cannot be automatically " % str( repository.tip( trans.app ) ) - message += "installed into a local Galaxy instance. Correct the following problems and reset metadata.%s" % new_line - for itc_tup in invalid_file_tups: - tool_file, exception_msg = itc_tup - if exception_msg.find( 'No such file or directory' ) >= 0: - exception_items = exception_msg.split() - missing_file_items = exception_items[ 7 ].split( '/' ) - missing_file = missing_file_items[ -1 ].rstrip( '\'' ) - if missing_file.endswith( '.loc' ): - sample_ext = '%s.sample' % missing_file - else: - sample_ext = missing_file - correction_msg = "This file refers to a missing file %s%s%s. " % ( bold_start, str( missing_file ), bold_end ) - correction_msg += "Upload a file named %s%s%s to the repository to correct this error." % ( bold_start, sample_ext, bold_end ) - else: - if as_html: - correction_msg = exception_msg - else: - correction_msg = exception_msg.replace( ' ', new_line ).replace( '', bold_start ).replace( '', bold_end ) - message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) - return message def generate_sharable_link_for_repository_in_tool_shed( trans, repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) @@ -925,64 +480,6 @@ def get_installed_tool_shed_repository( trans, id ): """Get a repository on the Galaxy side from the database via id""" return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) -def get_latest_tool_config_revision_from_repository_manifest( repo, filename, changeset_revision ): - """ - Get the latest revision of a tool config file named filename from the repository manifest up to the value of changeset_revision. - This method is restricted to tool_config files rather than any file since it is likely that, with the exception of tool config files, - multiple files will have the same name in various directories within the repository. - """ - stripped_filename = strip_path( filename ) - for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ): - manifest_ctx = repo.changectx( changeset ) - for ctx_file in manifest_ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if ctx_file_name == stripped_filename: - try: - fctx = manifest_ctx[ ctx_file ] - except LookupError: - # The ctx_file may have been moved in the change set. For example, 'ncbi_blastp_wrapper.xml' was moved to - # 'tools/ncbi_blast_plus/ncbi_blastp_wrapper.xml', so keep looking for the file until we find the new location. - continue - fh = tempfile.NamedTemporaryFile( 'wb' ) - tmp_filename = fh.name - fh.close() - fh = open( tmp_filename, 'wb' ) - fh.write( fctx.data() ) - fh.close() - return tmp_filename - return None -def get_list_of_copied_sample_files( repo, ctx, dir ): - """ - Find all sample files (files in the repository with the special .sample extension) in the reversed repository manifest up to ctx. Copy - each discovered file to dir and return the list of filenames. If a .sample file was added in a changeset and then deleted in a later - changeset, it will be returned in the deleted_sample_files list. The caller will set the value of app.config.tool_data_path to dir in - order to load the tools and generate metadata for them. - """ - deleted_sample_files = [] - sample_files = [] - for changeset in reversed_upper_bounded_changelog( repo, ctx ): - changeset_ctx = repo.changectx( changeset ) - for ctx_file in changeset_ctx.files(): - ctx_file_name = strip_path( ctx_file ) - # If we decide in the future that files deleted later in the changelog should not be used, we can use the following if statement. - # if ctx_file_name.endswith( '.sample' ) and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files: - if ctx_file_name.endswith( '.sample' ) and ctx_file_name not in sample_files: - fctx = get_file_context_from_ctx( changeset_ctx, ctx_file ) - if fctx in [ 'DELETED' ]: - # Since the possibly future used if statement above is commented out, the same file that was initially added will be - # discovered in an earlier changeset in the change log and fall through to the else block below. In other words, if - # a file named blast2go.loc.sample was added in change set 0 and then deleted in changeset 3, the deleted file in changeset - # 3 will be handled here, but the later discovered file in changeset 0 will be handled in the else block below. In this - # way, the file contents will always be found for future tools even though the file was deleted. - if ctx_file_name not in deleted_sample_files: - deleted_sample_files.append( ctx_file_name ) - else: - sample_files.append( ctx_file_name ) - tmp_ctx_file_name = os.path.join( dir, ctx_file_name.replace( '.sample', '' ) ) - fh = open( tmp_ctx_file_name, 'wb' ) - fh.write( fctx.data() ) - fh.close() - return sample_files, deleted_sample_files def get_named_tmpfile_from_ctx( ctx, filename, dir ): filename = strip_path( filename ) for ctx_file in ctx.files(): @@ -1092,20 +589,6 @@ previous_reviews_dict[ previous_changeset_revision ] = dict( changeset_revision_label=previous_changeset_revision_label, reviews=revision_reviews ) return previous_reviews_dict -def get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ): - """Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section selection page.""" - name = repo_info_dict.keys()[ 0 ] - repo_info_tuple = repo_info_dict[ name ] - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = get_repo_info_tuple_contents( repo_info_tuple ) - # Handle README files. - url = url_join( tool_shed_url, - 'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' % \ - ( name, repository_owner, changeset_revision ) ) - response = urllib2.urlopen( url ) - raw_text = response.read() - response.close() - readme_files_dict = json.from_json_string( raw_text ) - return readme_files_dict def get_repo_info_tuple_contents( repo_info_tuple ): # Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced. if len( repo_info_tuple ) == 6: @@ -1235,54 +718,6 @@ if tool: repository_tools_tups.append( ( relative_path, guid, tool ) ) return repository_tools_tups -def get_required_repo_info_dicts( tool_shed_url, repo_info_dicts ): - """ - Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of them to the list. All - repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through - this method is required to retrieve all repository dependencies. - """ - all_repo_info_dicts = [] - if repo_info_dicts: - # We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids. - required_repository_tups = [] - for repo_info_dict in repo_info_dicts: - for repository_name, repo_info_tup in repo_info_dict.items(): - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ - get_repo_info_tuple_contents( repo_info_tup ) - if repository_dependencies: - for key, val in repository_dependencies.items(): - if key in [ 'root_key', 'description' ]: - continue - toolshed, name, owner, changeset_revision = container_util.get_components_from_key( key ) - components_list = [ toolshed, name, owner, changeset_revision ] - if components_list not in required_repository_tups: - required_repository_tups.append( components_list ) - for components_list in val: - if components_list not in required_repository_tups: - required_repository_tups.append( components_list ) - if required_repository_tups: - # The value of required_repository_tups is a list of tuples, so we need to encode it. - encoded_required_repository_tups = [] - for required_repository_tup in required_repository_tups: - encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) ) - encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups ) - encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str ) - url = url_join( tool_shed_url, '/repository/get_required_repo_info_dict?encoded_str=%s' % encoded_required_repository_str ) - response = urllib2.urlopen( url ) - text = response.read() - response.close() - if text: - required_repo_info_dict = json.from_json_string( text ) - required_repo_info_dicts = [] - encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ] - for encoded_dict_str in encoded_dict_strings: - decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str ) - required_repo_info_dicts.append( decoded_dict ) - if required_repo_info_dicts: - for required_repo_info_dict in required_repo_info_dicts: - if required_repo_info_dict not in all_repo_info_dicts: - all_repo_info_dicts.append( required_repo_info_dict ) - return all_repo_info_dicts def get_reversed_changelog_changesets( repo ): reversed_changelog = [] for changeset in repo.changelog: @@ -1519,57 +954,6 @@ util.send_mail( frm, to, subject, body, trans.app.config ) except Exception, e: log.exception( "An error occurred sending a tool shed repository update alert by email." ) -def handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, tool_config_filepath, work_dir ): - # Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. - message = '' - sample_files = copy_disk_sample_files_to_dir( trans, repo_files_dir, work_dir ) - if sample_files: - if 'tool_data_table_conf.xml.sample' in sample_files: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) - tool, valid, message2 = load_tool_from_config( trans.app, tool_config_filepath ) - message = concat_messages( message, message2 ) - return tool, valid, message, sample_files -def handle_sample_files_and_load_tool_from_tmp_config( trans, repo, changeset_revision, tool_config_filename, work_dir ): - tool = None - message = '' - ctx = get_changectx_for_changeset( repo, changeset_revision ) - # We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in - # the manifest, but have been deleted from disk. - sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) - if sample_files: - trans.app.config.tool_data_path = work_dir - if 'tool_data_table_conf.xml.sample' in sample_files: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - if tool_data_table_config: - error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) - if error: - log.debug( message ) - manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) - if manifest_ctx and ctx_file: - tool, message2 = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir ) - message = concat_messages( message, message2 ) - return tool, message, sample_files -def handle_sample_tool_data_table_conf_file( app, filename, persist=False ): - """ - Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary. If persist is True (should only occur - if call is from the Galaxy side, not the tool shed), the new entries will be appended to Galaxy's shed_tool_data_table_conf.xml file on disk. - """ - error = False - message = '' - try: - new_table_elems, message = app.tool_data_tables.add_new_entries_from_config_file( config_filename=filename, - tool_data_path=app.config.tool_data_path, - shed_tool_data_table_config=app.config.shed_tool_data_table_config, - persist=persist ) - if message: - error = True - except Exception, e: - message = str( e ) - error = True - return error, message def has_previous_repository_reviews( trans, repository, changeset_revision ): """Determine if a repository has a changeset revision review prior to the received changeset revision.""" repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) @@ -1579,87 +963,6 @@ if previous_changeset_revision in reviewed_revision_hashes: return True return False -def load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config_filename ): - """ - Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value of tool_config_filename. The value of changeset_revision - is a valid (downloadable) changset revision. The tool config will be located in the repository manifest between the received valid changeset - revision and the first changeset revision in the repository, searching backwards. - """ - original_tool_data_path = trans.app.config.tool_data_path - repository = get_repository_in_tool_shed( trans, repository_id ) - repo_files_dir = repository.repo_path( trans.app ) - repo = hg.repository( get_configured_ui(), repo_files_dir ) - message = '' - tool = None - can_use_disk_file = False - tool_config_filepath = get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename ) - work_dir = tempfile.mkdtemp() - can_use_disk_file = can_use_tool_config_disk_file( trans, repository, repo, tool_config_filepath, changeset_revision ) - if can_use_disk_file: - trans.app.config.tool_data_path = work_dir - tool, valid, message, sample_files = handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, tool_config_filepath, work_dir ) - if tool is not None: - invalid_files_and_errors_tups = check_tool_input_params( trans.app, - repo_files_dir, - tool_config_filename, - tool, - sample_files ) - if invalid_files_and_errors_tups: - message2 = generate_message_for_invalid_tools( trans, - invalid_files_and_errors_tups, - repository, - metadata_dict=None, - as_html=True, - displaying_invalid_tool=True ) - message = concat_messages( message, message2 ) - else: - tool, message, sample_files = handle_sample_files_and_load_tool_from_tmp_config( trans, repo, changeset_revision, tool_config_filename, work_dir ) - remove_dir( work_dir ) - trans.app.config.tool_data_path = original_tool_data_path - # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - reset_tool_data_tables( trans.app ) - return repository, tool, message -def load_tool_from_config( app, full_path ): - try: - tool = app.toolbox.load_tool( full_path ) - valid = True - error_message = None - except KeyError, e: - tool = None - valid = False - error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) - error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' - error_message += 'this error. ' - except Exception, e: - tool = None - valid = False - error_message = str( e ) - return tool, valid, error_message -def load_tool_from_tmp_config( trans, repo, ctx, ctx_file, work_dir ): - tool = None - message = '' - tmp_tool_config = get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) - if tmp_tool_config: - element_tree = util.parse_xml( tmp_tool_config ) - element_tree_root = element_tree.getroot() - # Look for code files required by the tool config. - tmp_code_files = [] - for code_elem in element_tree_root.findall( 'code' ): - code_file_name = code_elem.get( 'file' ) - tmp_code_file_name = copy_file_from_manifest( repo, ctx, code_file_name, work_dir ) - if tmp_code_file_name: - tmp_code_files.append( tmp_code_file_name ) - tool, valid, message = load_tool_from_config( trans.app, tmp_tool_config ) - for tmp_code_file in tmp_code_files: - try: - os.unlink( tmp_code_file ) - except: - pass - try: - os.unlink( tmp_tool_config ) - except: - pass - return tool, message def open_repository_files_folder( trans, folder_path ): try: files_list = get_repository_files( trans, folder_path ) @@ -1714,9 +1017,6 @@ if tool_shed_repository and tool_shed_repository.status not in [ trans.model.ToolShedRepository.installation_status.NEW ]: return tool_shed_repository, previous_changeset_revision return None, None -def reset_tool_data_tables( app ): - # Reset the tool_data_tables to an empty dictionary. - app.tool_data_tables.data_tables = {} def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ): """ Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision, but up to and This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Provide backwards compatibility for sorter
by commits-noreply＠bitbucket.org 15 Mar '13

15 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8703329f3716/ changeset: 8703329f3716 user: guerler date: 2013-03-15 22:29:29 summary: Provide backwards compatibility for sorter affected #: 1 file diff -r b9afd514145740b4d5999158d51bd7dc4a409a99 -r 8703329f3716e7c4603e3a3a57286f67bc3b89af tools/filters/sorter.xml --- a/tools/filters/sorter.xml +++ b/tools/filters/sorter.xml @@ -3,59 +3,131 @@ <command interpreter="python"> sorter.py --input=$input - --output=$output + --output=$out_file1 + + #set $style = '' if (str($style) == 'alpha') else 'n' + #set $order = '' if (str($order) == 'ASC') else 'r' --key=$column,$column$style$order + #for $col in $column_set: - --key=${col.other_column},${col.other_column}${col.other_style}${col.other_order} + #set $other_column = str($col.other_column) + #set $other_style = '' if (str($col.other_style) == "alpha") else 'n' + #set $other_order = '' if (str($col.other_order) == "ASC") else 'r' + --key=$other_column,$other_column$other_style$other_order #end for </command><inputs><param format="tabular" name="input" type="data" label="Sort Dataset" /><param name="column" label="on column" type="data_column" data_ref="input" accept_default="true"/><param name="style" type="select" label="with flavor"> - <option value="n">Numerical sort</option> - <option value="">Alphabetical sort</option> + <option value="num">Numerical sort</option> + <option value="alpha">Alphabetical sort</option></param><param name="order" type="select" label="everything in"> - <option value="r">Descending order</option> - <option value="">Ascending order</option> + <option value="DESC">Descending order</option> + <option value="ASC">Ascending order</option></param><repeat name="column_set" title="Column selection"><param name="other_column" label="on column" type="data_column" data_ref="input" accept_default="true" /><param name="other_style" type="select" label="with flavor"> - <option value="n">Numerical sort</option> - <option value="">Alphabetical sort</option> + <option value="num">Numerical sort</option> + <option value="alpha">Alphabetical sort</option></param><param name="other_order" type="select" label="everything in"> - <option value="r">Descending order</option> - <option value="">Ascending order</option> + <option value="DESC">Descending order</option> + <option value="ASC">Ascending order</option></param></repeat></inputs><outputs> - <data format="input" name="output" metadata_source="input"/> + <data format="input" name="out_file1" metadata_source="input"/></outputs><tests><test><param name="input" value="sort_in1.bed"/><param name="column" value="1"/> - <param name="style" value=""/> - <param name="order" value=""/> + <param name="style" value="alpha"/> + <param name="order" value="ASC"/><param name="other_column" value="3"/> - <param name="other_style" value="n"/> - <param name="other_order" value="r"/> - <output name="output" file="sort_out1.bed"/> + <param name="other_style" value="num"/> + <param name="other_order" value="DESC"/> + <output name="out_file1" file="sort_out1.bed"/></test><test><param name="input" value="sort_in1.bed"/><param name="column" value="1"/> - <param name="style" value=""/> - <param name="order" value=""/> + <param name="style" value="alpha"/> + <param name="order" value="ASC"/><param name="other_column" value="3"/> - <param name="other_style" value="n"/> - <param name="other_order" value=""/> - <output name="output" file="sort_out2.bed"/> + <param name="other_style" value="num"/> + <param name="other_order" value="ASC"/> + <output name="out_file1" file="sort_out2.bed"/></test></tests> - + <help> + + .. class:: infomark + + **TIP:** If your data is not TAB delimited, use *Text Manipulation->Convert* + + ----- + + **Syntax** + + This tool sorts the dataset on any number of columns in either ascending or descending order. + + * Numerical sort orders numbers by their magnitude, ignores all characters besides numbers, and evaluates a string of numbers to the value they signify. + * Alphabetical sort is a phonebook type sort based on the conventional order of letters in an alphabet. Each nth letter is compared with the nth letter of other words in the list, starting at the first letter of each word and advancing to the second, third, fourth, and so on, until the order is established. Therefore, in an alphabetical sort, 2 comes after 100 (1 < 2). + + ----- + + **Examples** + + The list of numbers 4,17,3,5 collates to 3,4,5,17 by numerical sorting, while it collates to 17,3,4,5 by alphabetical sorting. + + Sorting the following:: + + Q d 7 II jhu 45 + A kk 4 I h 111 + Pd p 1 ktY WS 113 + A g 10 H ZZ 856 + A edf 4 tw b 234 + BBB rt 10 H ZZ 100 + A rew 10 d b 1111 + C sd 19 YH aa 10 + Hah c 23 ver bb 467 + MN gtr 1 a X 32 + N j 9 a T 205 + BBB rrf 10 b Z 134 + odfr ws 6 Weg dew 201 + C f 3 WW SW 34 + A jhg 4 I b 345 + Pd gf 7 Gthe de 567 + rS hty 90 YY LOp 89 + A g 10 H h 43 + A g 4 I h 500 + + on columns 1 (alpha), 3 (num), and 6 (num) in ascending order will yield:: + + A kk 4 I h 111 + A edf 4 tw b 234 + A jhg 4 I b 345 + A g 4 I h 500 + A g 10 H h 43 + A g 10 H ZZ 856 + A rew 10 d b 1111 + BBB rt 10 H ZZ 100 + BBB rrf 10 b Z 134 + C f 3 WW SW 34 + C sd 19 YH aa 10 + Hah c 23 ver bb 467 + MN gtr 1 a X 32 + N j 9 a T 205 + odfr ws 6 Weg dew 201 + Pd p 1 ktY WS 113 + Pd gf 7 Gthe de 567 + Q d 7 II jhu 45 + rS hty 90 YY LOp 89 + + </help></tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for allow_workflow_parameters during recursive call of check_and_update_param_values_helper.
by commits-noreply＠bitbucket.org 15 Mar '13

15 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b9afd5141457/ changeset: b9afd5141457 user: dan date: 2013-03-15 22:23:21 summary: Fix for allow_workflow_parameters during recursive call of check_and_update_param_values_helper. affected #: 1 file diff -r fea372d824394fa70cb37d306827190dc9b47556 -r b9afd514145740b4d5999158d51bd7dc4a409a99 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2349,7 +2349,7 @@ messages[ input.name ] = { input.test_param.name: "No value found for '%s%s', used default" % ( prefix, input.label ) } test_value = input.test_param.get_initial_value( trans, context ) current_case = input.get_current_case( test_value, trans ) - self.check_and_update_param_values_helper( input.cases[ current_case ].inputs, {}, trans, messages[ input.name ], context, prefix ) + self.check_and_update_param_values_helper( input.cases[ current_case ].inputs, {}, trans, messages[ input.name ], context, prefix, allow_workflow_parameters=allow_workflow_parameters ) elif isinstance( input, Repeat ): if input.min: messages[ input.name ] = [] @@ -2357,7 +2357,7 @@ rep_prefix = prefix + "%s %d > " % ( input.title, i + 1 ) rep_dict = dict() messages[ input.name ].append( rep_dict ) - self.check_and_update_param_values_helper( input.inputs, {}, trans, rep_dict, context, rep_prefix ) + self.check_and_update_param_values_helper( input.inputs, {}, trans, rep_dict, context, rep_prefix, allow_workflow_parameters=allow_workflow_parameters ) else: messages[ input.name ] = "No value found for '%s%s', used default" % ( prefix, input.label ) values[ input.name ] = input.get_initial_value( trans, context ) @@ -2366,7 +2366,7 @@ if isinstance( input, Repeat ): for i, d in enumerate( values[ input.name ] ): rep_prefix = prefix + "%s %d > " % ( input.title, i + 1 ) - self.check_and_update_param_values_helper( input.inputs, d, trans, messages, context, rep_prefix ) + self.check_and_update_param_values_helper( input.inputs, d, trans, messages, context, rep_prefix, allow_workflow_parameters=allow_workflow_parameters ) elif isinstance( input, Conditional ): group_values = values[ input.name ] if input.test_param.name not in group_values: @@ -2378,7 +2378,7 @@ messages[ child_input.name ] = "Value no longer valid for '%s%s', replaced with default" % ( prefix, child_input.label ) else: current = group_values["__current_case__"] - self.check_and_update_param_values_helper( input.cases[current].inputs, group_values, trans, messages, context, prefix ) + self.check_and_update_param_values_helper( input.cases[current].inputs, group_values, trans, messages, context, prefix, allow_workflow_parameters=allow_workflow_parameters ) else: # Regular tool parameter, no recursion needed try: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for tool_shed/util/metadata_util.py.generate_guid_for_object().
by commits-noreply＠bitbucket.org 15 Mar '13

15 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/fea372d82439/ changeset: fea372d82439 user: dan date: 2013-03-15 21:36:44 summary: Fix for tool_shed/util/metadata_util.py.generate_guid_for_object(). affected #: 1 file diff -r 1cc6973069b5e7afeb5c2d401c262b0d39efd24b -r fea372d824394fa70cb37d306827190dc9b47556 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -466,7 +466,7 @@ return valid_tool_dependencies_dict def generate_guid_for_object( repository_clone_url, guid_type, obj_id, version ): - tmp_url = clean_repository_clone_url( repository_clone_url ) + tmp_url = suc.clean_repository_clone_url( repository_clone_url ) return '%s/%s/%s/%s' % ( tmp_url, guid_type, obj_id, version ) def generate_metadata_for_changeset_revision( app, repository, changeset_revision, repository_clone_url, shed_config_dict=None, relative_install_dir=None, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Remove ~/tool_shed/util/shed_util.py
by commits-noreply＠bitbucket.org 15 Mar '13

15 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1cc6973069b5/ changeset: 1cc6973069b5 user: greg date: 2013-03-15 21:09:08 summary: Remove ~/tool_shed/util/shed_util.py affected #: 1 file diff -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 -r 1cc6973069b5e7afeb5c2d401c262b0d39efd24b lib/tool_shed/util/shed_util.py --- a/lib/tool_shed/util/shed_util.py +++ /dev/null @@ -1,24 +0,0 @@ -import os, tempfile, shutil, logging, urllib2, threading -from galaxy.datatypes import checkers -from galaxy.web import url_for -from galaxy import util -from galaxy.util import json -from galaxy.webapps.tool_shed.util import container_util -from tool_shed.galaxy_install.tool_dependencies.install_util import install_package, set_environment -from galaxy.model.orm import and_ -import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, repository_dependency_util, tool_dependency_util - -from galaxy import eggs -import pkg_resources - -pkg_resources.require( 'mercurial' ) -from mercurial import hg, ui, commands - -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude -from elementtree.ElementTree import Element, SubElement - -log = logging.getLogger( __name__ ) - - Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Refactor shed_util and shed_util_common into appropriate tool shed Galaxy utilities components.
by commits-noreply＠bitbucket.org 15 Mar '13

15 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/320c9b7f08c4/ changeset: 320c9b7f08c4 user: greg date: 2013-03-15 21:01:09 summary: Refactor shed_util and shed_util_common into appropriate tool shed Galaxy utilities components. affected #: 24 files diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -5,10 +5,10 @@ from galaxy.web.framework.helpers import iff, grids from galaxy.util import json from galaxy.model.orm import or_ -import tool_shed.util.shed_util as shed_util import tool_shed.util.shed_util_common as suc -import tool_shed.util.metadata_util as metadata_util -from tool_shed.util import encoding_util +from tool_shed.util import common_install_util, data_manager_util, datatype_util, encoding_util, metadata_util +from tool_shed.util import repository_dependency_util, tool_dependency_util, tool_util +from tool_shed.galaxy_install import repository_util from galaxy.webapps.tool_shed.util import workflow_util import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids import pkg_resources @@ -36,7 +36,7 @@ repository_id = kwd[ 'id' ] repository = suc.get_installed_tool_shed_repository( trans, repository_id ) try: - shed_util.activate_repository( trans, repository ) + common_install_util.activate_repository( trans, repository ) except Exception, e: error_message = "Error activating repository %s: %s" % ( repository.name, str( e ) ) log.debug( error_message ) @@ -92,7 +92,7 @@ # can reset the metadata if necessary. This will ensure that information about repository dependencies and tool dependencies # will be current. Only allow selecting a different section in the tool panel if the repository was uninstalled and it contained # tools that should be displayed in the tool panel. - changeset_revision_dict = shed_util.get_update_to_changeset_revision_and_ctx_rev( trans, repository ) + changeset_revision_dict = repository_util.get_update_to_changeset_revision_and_ctx_rev( trans, repository ) current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) if current_changeset_revision and current_ctx_rev: @@ -142,7 +142,7 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - tool_dependency = shed_util.get_tool_dependency( trans, kwd[ 'id' ] ) + tool_dependency = tool_dependency_util.get_tool_dependency( trans, kwd[ 'id' ] ) if tool_dependency.in_error_state: message = "This tool dependency is not installed correctly (see the Tool dependency installation error below). " message += "Choose Uninstall this tool dependency from the Repository Actions menu, correct problems " @@ -217,16 +217,16 @@ if params.get( 'deactivate_or_uninstall_repository_button', False ): if tool_shed_repository.includes_tools_for_display_in_tool_panel: # Handle tool panel alterations. - shed_util.remove_from_tool_panel( trans, tool_shed_repository, shed_tool_conf, uninstall=remove_from_disk_checked ) + tool_util.remove_from_tool_panel( trans, tool_shed_repository, shed_tool_conf, uninstall=remove_from_disk_checked ) if tool_shed_repository.includes_data_managers: - shed_util.remove_from_data_manager( trans.app, tool_shed_repository ) + data_manager_util.remove_from_data_manager( trans.app, tool_shed_repository ) if tool_shed_repository.includes_datatypes: # Deactivate proprietary datatypes. - installed_repository_dict = shed_util.load_installed_datatypes( trans.app, tool_shed_repository, repository_install_dir, deactivate=True ) + installed_repository_dict = datatype_util.load_installed_datatypes( trans.app, tool_shed_repository, repository_install_dir, deactivate=True ) if installed_repository_dict and 'converter_path' in installed_repository_dict: - shed_util.load_installed_datatype_converters( trans.app, installed_repository_dict, deactivate=True ) + datatype_util.load_installed_datatype_converters( trans.app, installed_repository_dict, deactivate=True ) if installed_repository_dict and 'display_path' in installed_repository_dict: - shed_util.load_installed_display_applications( trans.app, installed_repository_dict, deactivate=True ) + datatype_util.load_installed_display_applications( trans.app, installed_repository_dict, deactivate=True ) if remove_from_disk_checked: try: # Remove the repository from disk. @@ -244,7 +244,7 @@ tool_shed_repository.uninstalled = True # Remove all installed tool dependencies, but don't touch any repository dependencies.. for tool_dependency in tool_shed_repository.installed_tool_dependencies: - uninstalled, error_message = shed_util.remove_tool_dependency( trans, tool_dependency ) + uninstalled, error_message = tool_dependency_util.remove_tool_dependency( trans, tool_dependency ) if error_message: errors = '%s %s' % ( errors, error_message ) tool_shed_repository.deleted = True @@ -371,7 +371,7 @@ return repo_information_dict def get_versions_of_tool( self, app, guid ): - tool_version = shed_util.get_tool_version( app, guid ) + tool_version = tool_util.get_tool_version( app, guid ) return tool_version.get_version_ids( app, reverse=True ) def handle_repository_contents( self, trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, @@ -395,26 +395,26 @@ trans.sa_session.add( tool_shed_repository ) trans.sa_session.flush() if 'tool_dependencies' in metadata_dict and not reinstalling: - tool_dependencies = shed_util.create_tool_dependency_objects( trans.app, tool_shed_repository, relative_install_dir, set_status=True ) + tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, tool_shed_repository, relative_install_dir, set_status=True ) if 'tools' in metadata_dict: - tool_panel_dict = shed_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) + tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) sample_files = metadata_dict.get( 'sample_files', [] ) - tool_index_sample_files = shed_util.get_tool_index_sample_files( sample_files ) - shed_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) + tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) + tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) sample_files_copied = [ str( s ) for s in tool_index_sample_files ] repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = shed_util.handle_missing_data_table_entry( trans.app, relative_install_dir, tool_path, repository_tools_tups ) + repository_tools_tups = tool_util.handle_missing_data_table_entry( trans.app, relative_install_dir, tool_path, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups, sample_files_copied = shed_util.handle_missing_index_file( trans.app, + repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups, sample_files_copied ) # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. - shed_util.copy_sample_files( trans.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) - shed_util.add_to_tool_panel( app=trans.app, + tool_util.copy_sample_files( trans.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) + tool_util.add_to_tool_panel( app=trans.app, repository_name=tool_shed_repository.name, repository_clone_url=repository_clone_url, changeset_revision=tool_shed_repository.installed_changeset_revision, @@ -424,8 +424,13 @@ tool_panel_dict=tool_panel_dict, new_install=True ) if 'data_manager' in metadata_dict: - new_data_managers = shed_util.install_data_managers( trans.app, trans.app.config.shed_data_manager_config_file, metadata_dict, shed_config_dict, relative_install_dir, - tool_shed_repository, repository_tools_tups ) + new_data_managers = data_manager_util.install_data_managers( trans.app, + trans.app.config.shed_data_manager_config_file, + metadata_dict, + shed_config_dict, + relative_install_dir, + tool_shed_repository, + repository_tools_tups ) if 'datatypes' in metadata_dict: tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: @@ -437,16 +442,16 @@ files_dir = os.path.join( shed_config_dict['tool_path'], files_dir ) datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', files_dir ) # Load data types required by tools. - converter_path, display_path = shed_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False ) + converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. - repository_dict = shed_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) + repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, + name=tool_shed_repository.name, + owner=tool_shed_repository.owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) if converter_path: # Load proprietary datatype converters trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict ) @@ -533,10 +538,10 @@ tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', tool_shed_repository.repo_path( trans.app ) ) - installed_tool_dependencies = shed_util.handle_tool_dependencies( app=trans.app, - tool_shed_repository=tool_shed_repository, - tool_dependencies_config=tool_dependencies_config, - tool_dependencies=tool_dependencies ) + installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, + tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_dependencies ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == trans.app.model.ToolDependency.installation_status.ERROR: message += ' %s' % suc.to_safe_string( installed_tool_dependency.error_message ) @@ -564,7 +569,7 @@ tool_dependency_ids = util.listify( params.get( 'id', None ) ) tool_dependencies = [] for tool_dependency_id in tool_dependency_ids: - tool_dependency = shed_util.get_tool_dependency( trans, tool_dependency_id ) + tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_id ) tool_dependencies.append( tool_dependency ) if kwd.get( 'install_tool_dependencies_button', False ): # Filter tool dependencies to only those that are installed. @@ -616,10 +621,10 @@ if isinstance( repo_info_dict, basestring ): repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict ) # Clone each repository to the configured location. - shed_util.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING ) + suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING ) repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple - relative_clone_dir = shed_util.generate_tool_shed_repository_install_dir( repository_clone_url, tool_shed_repository.installed_changeset_revision ) + relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, tool_shed_repository.installed_changeset_revision ) clone_dir = os.path.join( tool_path, relative_clone_dir ) relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name ) install_dir = os.path.join( tool_path, relative_install_dir ) @@ -627,12 +632,12 @@ if cloned_ok: if reinstalling: # Since we're reinstalling the repository we need to find the latest changeset revision to which is can be updated. - changeset_revision_dict = shed_util.get_update_to_changeset_revision_and_ctx_rev( trans, tool_shed_repository ) + changeset_revision_dict = repository_util.get_update_to_changeset_revision_and_ctx_rev( trans, tool_shed_repository ) current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) if current_ctx_rev != ctx_rev: repo = hg.repository( suc.get_configured_ui(), path=os.path.abspath( install_dir ) ) - shed_util.pull_repository( repo, repository_clone_url, current_changeset_revision ) + repository_util.pull_repository( repo, repository_clone_url, current_changeset_revision ) suc.update_repository( repo, ctx_rev=current_ctx_rev ) self.handle_repository_contents( trans, tool_shed_repository=tool_shed_repository, @@ -647,9 +652,9 @@ metadata = tool_shed_repository.metadata if 'tools' in metadata: # Get the tool_versions from the tool shed for each tool in the installed change set. - shed_util.update_tool_shed_repository_status( trans.app, - tool_shed_repository, - trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) + suc.update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) url = suc.url_join( tool_shed_url, '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ @@ -659,7 +664,7 @@ response.close() if text: tool_version_dicts = json.from_json_string( text ) - shed_util.handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository ) + tool_util.handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository ) else: message += "Version information for the tools included in the %s repository is missing. " % name message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions " @@ -668,20 +673,20 @@ if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp() # Install tool dependencies. - shed_util.update_tool_shed_repository_status( trans.app, - tool_shed_repository, - trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + suc.update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir )#relative_install_dir ) - installed_tool_dependencies = shed_util.handle_tool_dependencies( app=trans.app, - tool_shed_repository=tool_shed_repository, - tool_dependencies_config=tool_dependencies_config, - tool_dependencies=tool_shed_repository.tool_dependencies ) + installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, + tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_shed_repository.tool_dependencies ) try: shutil.rmtree( work_dir ) except: pass - shed_util.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) + suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) else: # An error occurred while cloning the repository, so reset everything necessary to enable another attempt. self.set_repository_attributes( trans, @@ -739,12 +744,12 @@ trans.sa_session.add( repository ) trans.sa_session.flush() message = "The repository information has been updated." - containers_dict = shed_util.populate_containers_dict_from_repository_metadata( trans=trans, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - repository=repository, - reinstalling=False, - required_repo_info_dicts=None ) + containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + repository=repository, + reinstalling=False, + required_repo_info_dicts=None ) return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', repository=repository, description=description, @@ -825,9 +830,9 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - tool_dependency_ids = shed_util.get_tool_dependency_ids( as_string=False, **kwd ) + tool_dependency_ids = tool_dependency_util.get_tool_dependency_ids( as_string=False, **kwd ) # We need a tool_shed_repository, so get it from one of the tool_dependencies. - tool_dependency = shed_util.get_tool_dependency( trans, tool_dependency_ids[ 0 ] ) + tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_ids[ 0 ] ) tool_shed_repository = tool_dependency.tool_shed_repository self.tool_dependency_grid.title = "Tool shed repository '%s' tool dependencies" % tool_shed_repository.name self.tool_dependency_grid.global_actions = \ @@ -868,7 +873,7 @@ elif operation == 'uninstall': tool_dependencies_for_uninstallation = [] for tool_dependency_id in tool_dependency_ids: - tool_dependency = shed_util.get_tool_dependency( trans, tool_dependency_id ) + tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_id ) if tool_dependency.status in [ trans.model.ToolDependency.installation_status.INSTALLED, trans.model.ToolDependency.installation_status.ERROR ]: tool_dependencies_for_uninstallation.append( tool_dependency ) @@ -883,7 +888,7 @@ if trans.app.config.tool_dependency_dir: tool_dependencies_for_installation = [] for tool_dependency_id in tool_dependency_ids: - tool_dependency = shed_util.get_tool_dependency( trans, tool_dependency_id ) + tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_id ) if tool_dependency.status in [ trans.model.ToolDependency.installation_status.NEVER_INSTALLED, trans.model.ToolDependency.installation_status.UNINSTALLED ]: tool_dependencies_for_installation.append( tool_dependency ) @@ -978,15 +983,15 @@ install_tool_dependencies = False tool_path = suc.get_tool_path_by_shed_tool_conf_filename( trans, shed_tool_conf ) created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \ - shed_util.create_repository_dependency_objects( trans, - tool_path, - tool_shed_url, - repo_info_dicts, - reinstalling=False, - install_repository_dependencies=install_repository_dependencies, - no_changes_checked=False, - tool_panel_section=tool_panel_section, - new_tool_panel_section=new_tool_panel_section ) + repository_dependency_util.create_repository_dependency_objects( trans, + tool_path, + tool_shed_url, + repo_info_dicts, + reinstalling=False, + install_repository_dependencies=install_repository_dependencies, + no_changes_checked=False, + tool_panel_section=tool_panel_section, + new_tool_panel_section=new_tool_panel_section ) if message and len( repo_info_dicts ) == 1: installed_tool_shed_repository = created_or_updated_tool_shed_repositories[ 0 ] message+= 'Click <a href="%s">here</a> to manage the repository. ' % \ @@ -1069,39 +1074,39 @@ repo_info_dict = repo_info_dicts[ 0 ] name, repository_owner, changeset_revision, includes_tool_dependencies, installed_repository_dependencies, \ missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies = \ - shed_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ) - readme_files_dict = shed_util.get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ) + common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ) + readme_files_dict = suc.get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ) # We're handling 1 of 2 scenarios here: (1) we're installing a tool shed repository for the first time, so we've retrieved the list of installed # and missing repository dependencies from the database (2) we're handling the scenario where an error occurred during the installation process, # so we have a tool_shed_repository record in the database with associated repository dependency records. Since we have the repository # dependencies in either case, we'll merge the list of missing repository dependencies into the list of installed repository dependencies since # each displayed repository dependency will display a status, whether installed or missing. - containers_dict = shed_util.populate_containers_dict_for_new_install( trans=trans, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=readme_files_dict, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies ) + containers_dict = repository_util.populate_containers_dict_for_new_install( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + readme_files_dict=readme_files_dict, + installed_repository_dependencies=installed_repository_dependencies, + missing_repository_dependencies=missing_repository_dependencies, + installed_tool_dependencies=installed_tool_dependencies, + missing_tool_dependencies=missing_tool_dependencies ) else: # We're installing a list of repositories, each of which may have tool dependencies or repository dependencies. containers_dicts = [] for repo_info_dict in repo_info_dicts: name, repository_owner, changeset_revision, includes_tool_dependencies, installed_repository_dependencies, \ missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies = \ - shed_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ) - containers_dict = shed_util.populate_containers_dict_for_new_install( trans=trans, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=None, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies ) + common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ) + containers_dict = repository_util.populate_containers_dict_for_new_install( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + readme_files_dict=None, + installed_repository_dependencies=installed_repository_dependencies, + missing_repository_dependencies=missing_repository_dependencies, + installed_tool_dependencies=installed_tool_dependencies, + missing_tool_dependencies=missing_tool_dependencies ) containers_dicts.append( containers_dict ) # Merge all containers into a single container. - containers_dict = shed_util.merge_containers_dicts_for_new_install( containers_dicts ) + containers_dict = repository_util.merge_containers_dicts_for_new_install( containers_dicts ) # Handle tool dependencies check box. if trans.app.config.tool_dependency_dir is None: if includes_tool_dependencies: @@ -1165,8 +1170,8 @@ install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) repository_clone_url = suc.generate_clone_url_for_installed_repository( trans.app, tool_shed_repository ) - clone_dir = os.path.join( tool_path, shed_util.generate_tool_shed_repository_install_dir( repository_clone_url, - tool_shed_repository.installed_changeset_revision ) ) + clone_dir = os.path.join( tool_path, suc.generate_tool_shed_repository_install_dir( repository_clone_url, + tool_shed_repository.installed_changeset_revision ) ) relative_install_dir = os.path.join( clone_dir, tool_shed_repository.name ) tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) tool_section = None @@ -1179,12 +1184,11 @@ includes_tool_dependencies = tool_shed_repository.includes_tool_dependencies if tool_shed_repository.includes_tools_for_display_in_tool_panel: # Handle the selected tool panel location for loading tools included in the tool shed repository. - tool_section, new_tool_panel_section, tool_panel_section_key = \ - shed_util.handle_tool_panel_selection( trans=trans, - metadata=metadata, - no_changes_checked=no_changes_checked, - tool_panel_section=tool_panel_section, - new_tool_panel_section=new_tool_panel_section ) + tool_section, new_tool_panel_section, tool_panel_section_key = tool_util.handle_tool_panel_selection( trans=trans, + metadata=metadata, + no_changes_checked=no_changes_checked, + tool_panel_section=tool_panel_section, + new_tool_panel_section=new_tool_panel_section ) # The repository's status must be updated from 'Uninstall' to 'New' when initiating reinstall so the repository_installation_updater will function. tool_shed_repository = suc.create_or_update_tool_shed_repository( trans.app, tool_shed_repository.name, @@ -1217,28 +1221,28 @@ tool_dependencies = metadata.get( 'tool_dependencies', None ) else: tool_dependencies = None - repo_info_dict = suc.create_repo_info_dict( trans=trans, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.changeset_revision, - ctx_rev=ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = repository_util.create_repo_info_dict( trans=trans, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.changeset_revision, + ctx_rev=ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + repository=None, + repository_metadata=None, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) repo_info_dicts.append( repo_info_dict ) # Make sure all tool_shed_repository records exist. created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \ - shed_util.create_repository_dependency_objects( trans=trans, - tool_path=tool_path, - tool_shed_url=tool_shed_url, - repo_info_dicts=repo_info_dicts, - reinstalling=True, - install_repository_dependencies=install_repository_dependencies, - no_changes_checked=no_changes_checked, - tool_panel_section=tool_panel_section, - new_tool_panel_section=new_tool_panel_section ) + repository_dependency_util.create_repository_dependency_objects( trans=trans, + tool_path=tool_path, + tool_shed_url=tool_shed_url, + repo_info_dicts=repo_info_dicts, + reinstalling=True, + install_repository_dependencies=install_repository_dependencies, + no_changes_checked=no_changes_checked, + tool_panel_section=tool_panel_section, + new_tool_panel_section=new_tool_panel_section ) # Default the selected tool panel location for loading tools included in each newly installed required tool shed repository to the location # selected for the repository selected for reinstallation. for index, tps_key in enumerate( tool_panel_section_keys ): @@ -1365,19 +1369,19 @@ repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, changeset_revision=tool_shed_repository.changeset_revision ) - repo_info_dict = suc.create_repo_info_dict( trans=trans, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.changeset_revision, - ctx_rev=tool_shed_repository.ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = repository_util.create_repo_info_dict( trans=trans, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.changeset_revision, + ctx_rev=tool_shed_repository.ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + repository=None, + repository_metadata=None, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) repository_name, repository_owner, changeset_revision, includes_tool_dependencies, installed_repository_dependencies, \ missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies = \ - shed_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ) + common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ) if installed_repository_dependencies or missing_repository_dependencies: has_repository_dependencies = True else: @@ -1387,7 +1391,7 @@ if 'tool_panel_section' in metadata: tool_panel_dict = metadata[ 'tool_panel_section' ] if tool_panel_dict: - if shed_util.panel_entry_per_tool( tool_panel_dict ): + if tool_util.panel_entry_per_tool( tool_panel_dict ): # The following forces everything to be loaded into 1 section (or no section) in the tool panel. tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] tool_section_dict = tool_section_dicts[ 0 ] @@ -1415,17 +1419,17 @@ original_section_name = '' tool_panel_section_select_field = None shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans ) - containers_dict = shed_util.populate_containers_dict_for_new_install( trans=trans, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=readme_files_dict, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies ) + containers_dict = repository_util.populate_containers_dict_for_new_install( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + readme_files_dict=readme_files_dict, + installed_repository_dependencies=installed_repository_dependencies, + missing_repository_dependencies=missing_repository_dependencies, + installed_tool_dependencies=installed_tool_dependencies, + missing_tool_dependencies=missing_tool_dependencies ) # Since we're reinstalling we'll merge the list of missing repository dependencies into the list of installed repository dependencies since each displayed # repository dependency will display a status, whether installed or missing. - containers_dict = suc.merge_missing_repository_dependencies_to_installed_container( containers_dict ) + containers_dict = repository_dependency_util.merge_missing_repository_dependencies_to_installed_container( containers_dict ) # Handle repository dependencies check box. install_repository_dependencies_check_box = CheckboxField( 'install_repository_dependencies', checked=True ) # Handle tool dependencies check box. @@ -1566,7 +1570,7 @@ response.close() if text: tool_version_dicts = json.from_json_string( text ) - shed_util.handle_tool_versions( trans.app, tool_version_dicts, repository ) + tool_util.handle_tool_versions( trans.app, tool_version_dicts, repository ) message = "Tool versions have been set for all included tools." status = 'done' else: @@ -1576,12 +1580,12 @@ status = 'error' shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) - containers_dict = shed_util.populate_containers_dict_from_repository_metadata( trans=trans, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - repository=repository, - reinstalling=False, - required_repo_info_dicts=None ) + containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + repository=repository, + reinstalling=False, + required_repo_info_dicts=None ) return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', repository=repository, description=repository.description, @@ -1622,7 +1626,7 @@ tool_dependency_ids = util.listify( params.get( 'id', None ) ) tool_dependencies = [] for tool_dependency_id in tool_dependency_ids: - tool_dependency = shed_util.get_tool_dependency( trans, tool_dependency_id ) + tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_id ) tool_dependencies.append( tool_dependency ) tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository if kwd.get( 'uninstall_tool_dependencies_button', False ): @@ -1633,7 +1637,7 @@ if tool_dependency.can_uninstall: tool_dependencies_for_uninstallation.append( tool_dependency ) for tool_dependency in tool_dependencies_for_uninstallation: - uninstalled, error_message = shed_util.remove_tool_dependency( trans, tool_dependency ) + uninstalled, error_message = tool_dependency_util.remove_tool_dependency( trans, tool_dependency ) if error_message: errors = True message = '%s %s' % ( message, error_message ) @@ -1681,12 +1685,12 @@ repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, name ) ) repo = hg.repository( suc.get_configured_ui(), path=repo_files_dir ) repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) - shed_util.pull_repository( repo, repository_clone_url, latest_ctx_rev ) + repository_util.pull_repository( repo, repository_clone_url, latest_ctx_rev ) suc.update_repository( repo, latest_ctx_rev ) tool_shed = suc.clean_tool_shed_url( tool_shed_url ) # Remove old Data Manager entries if repository.includes_data_managers: - shed_util.remove_from_data_manager( trans.app, repository ) + data_manager_util.remove_from_data_manager( trans.app, repository ) # Update the repository metadata. metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app, repository=repository, @@ -1710,7 +1714,7 @@ if tool_panel_dict is None: tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( trans.app, repository ) repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) - shed_util.add_to_tool_panel( app=trans.app, + tool_util.add_to_tool_panel( app=trans.app, repository_name=repository.name, repository_clone_url=repository_clone_url, changeset_revision=repository.installed_changeset_revision, @@ -1721,12 +1725,16 @@ new_install=False ) # Add new Data Manager entries if 'data_manager' in metadata_dict: - new_data_managers = shed_util.install_data_managers( trans.app, trans.app.config.shed_data_manager_config_file, metadata_dict, - repository.get_shed_config_dict( trans.app ), os.path.join( relative_install_dir, name ), - repository, repository_tools_tups ) + new_data_managers = data_manager_util.install_data_managers( trans.app, + trans.app.config.shed_data_manager_config_file, + metadata_dict, + repository.get_shed_config_dict( trans.app ), + os.path.join( relative_install_dir, name ), + repository, + repository_tools_tups ) # Create tool_dependency records if necessary. if 'tool_dependencies' in metadata_dict: - tool_dependencies = shed_util.create_tool_dependency_objects( trans.app, repository, relative_install_dir, set_status=False ) + tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, repository, relative_install_dir, set_status=False ) message = "The installed repository named '%s' has been updated to change set revision '%s'. " % ( name, latest_changeset_revision ) # See if any tool dependencies can be installed. shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) @@ -1829,10 +1837,6 @@ def can_select_tool_panel_section(): pass -def get_tool_dependency( trans, id ): - """Get a tool_dependency from the database via id""" - return trans.sa_session.query( trans.model.ToolDependency ).get( trans.security.decode_id( id ) ) - def have_shed_tool_conf_for_install( trans ): if not trans.app.toolbox.shed_tool_confs: return False diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/galaxy/webapps/tool_shed/api/repository_revision_contents.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revision_contents.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revision_contents.py @@ -1,6 +1,7 @@ import logging from galaxy.web.framework.helpers import time_ago import tool_shed.util.shed_util_common as suc +from tool_shed.util import metadata_util from galaxy import web from galaxy.web.base.controller import BaseAPIController @@ -29,7 +30,7 @@ rval = [] repository_metadata_id = kwd.get( 'repository_metadata_id', None ) try: - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) repository_dict = repository_metadata.as_dict( value_mapper=default_value_mapper( trans, repository_metadata ) ) repository_dict[ 'url' ] = web.url_for( 'repository_revision_contents', repository_metadata_id=repository_metadata_id ) rval.append( repository_dict ) diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -1,6 +1,7 @@ import datetime from galaxy.web.framework.helpers import time_ago import tool_shed.util.shed_util_common as suc +from tool_shed.util import metadata_util from galaxy import web, util from galaxy.model.orm import and_, or_ from galaxy.web.base.controller import BaseAPIController @@ -73,7 +74,7 @@ Displays information about a repository_metadata record in the Tool Shed. """ try: - repository_metadata = suc.get_repository_metadata_by_id( trans, id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) repository_data = repository_metadata.get_api_value( view='element', value_mapper=default_value_mapper( trans, repository_metadata ) ) repository_data[ 'contents_url' ] = web.url_for( 'repository_revision_contents', repository_metadata_id=id ) @@ -91,7 +92,7 @@ """ repository_metadata_id = kwd.get( 'id', None ) try: - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) flush_needed = False for key, new_value in payload.items(): if hasattr( repository_metadata, key ): diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -106,7 +106,7 @@ # The received id is a RepositoryMetadata object id, so we need to get the # associated Repository and redirect to view_or_manage_repository with the # changeset_revision. - repository_metadata = suc.get_repository_metadata_by_id( trans, kwd[ 'id' ] ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, kwd[ 'id' ] ) repository = repository_metadata.repository kwd[ 'id' ] = trans.security.encode_id( repository.id ) kwd[ 'changeset_revision' ] = repository_metadata.changeset_revision @@ -196,7 +196,7 @@ ids = util.listify( id ) count = 0 for repository_metadata_id in ids: - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) trans.sa_session.delete( repository_metadata ) trans.sa_session.flush() count += 1 @@ -332,7 +332,7 @@ for repository_metadata in repository.metadata_revisions: metadata = repository_metadata.metadata if metadata: - if suc.is_downloadable( metadata ): + if metadata_util.is_downloadable( metadata ): repository_metadata.downloadable = True trans.sa_session.add( repository_metadata ) repository.deleted = False diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -11,10 +11,9 @@ from galaxy.util import json from galaxy.model.orm import and_, or_ import tool_shed.util.shed_util_common as suc -import tool_shed.util.metadata_util as metadata_util -from tool_shed.util import encoding_util -from galaxy.webapps.tool_shed.util import workflow_util -from galaxy.webapps.tool_shed.util import common_util +from tool_shed.util import encoding_util, metadata_util, repository_dependency_util, tool_dependency_util +from tool_shed.galaxy_install import repository_util +from galaxy.webapps.tool_shed.util import common_util, workflow_util import galaxy.tools import tool_shed.grids.repository_grids as repository_grids import tool_shed.grids.util as grids_util @@ -85,7 +84,7 @@ operation = kwd[ 'operation' ].lower() # The received id is a RepositoryMetadata id. repository_metadata_id = kwd[ 'id' ] - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) repository_id = trans.security.encode_id( repository_metadata.repository_id ) changeset_revision = repository_metadata.changeset_revision new_kwd = dict( id=repository_id, @@ -266,7 +265,7 @@ operation = kwd[ 'operation' ].lower() # The received id is a RepositoryMetadata id. repository_metadata_id = kwd[ 'id' ] - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) repository_id = trans.security.encode_id( repository_metadata.repository_id ) changeset_revision = repository_metadata.changeset_revision new_kwd = dict( id=repository_id, @@ -287,7 +286,7 @@ operation = kwd[ 'operation' ].lower() # The received id is a RepositoryMetadata id. repository_metadata_id = kwd['id' ] - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) repository_id = trans.security.encode_id( repository_metadata.repository_id ) changeset_revision = repository_metadata.changeset_revision new_kwd = dict( id=repository_id, @@ -304,7 +303,7 @@ operation = kwd[ 'operation' ].lower() # The received id is a RepositoryMetadata id. repository_metadata_id = kwd[ 'id' ] - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) repository_id = trans.security.encode_id( repository_metadata.repository_id ) changeset_revision = repository_metadata.changeset_revision new_kwd = dict( id=repository_id, @@ -689,7 +688,7 @@ status = params.get( 'status', 'done' ) repository = suc.get_repository_by_id( trans, repository_id ) if repository: - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) changeset_revision = repository_metadata.changeset_revision if repository_metadata: metadata = repository_metadata.metadata @@ -757,7 +756,7 @@ is_admin = trans.user_is_admin() if operation == "view_or_manage_repository": # The received id is a RepositoryMetadata id, so we have to get the repository id. - repository_metadata = suc.get_repository_metadata_by_id( trans, item_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, item_id ) repository_id = trans.security.encode_id( repository_metadata.repository.id ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) kwd[ 'id' ] = repository_id @@ -774,7 +773,7 @@ encoded_repository_ids = [] changeset_revisions = [] for repository_metadata_id in util.listify( item_id ): - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) encoded_repository_ids.append( trans.security.encode_id( repository_metadata.repository.id ) ) changeset_revisions.append( repository_metadata.changeset_revision ) new_kwd = {} @@ -844,7 +843,7 @@ is_admin = trans.user_is_admin() if operation == "view_or_manage_repository": # The received id is a RepositoryMetadata id, so we have to get the repository id. - repository_metadata = suc.get_repository_metadata_by_id( trans, item_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, item_id ) repository_id = trans.security.encode_id( repository_metadata.repository.id ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) kwd[ 'id' ] = repository_id @@ -861,7 +860,7 @@ encoded_repository_ids = [] changeset_revisions = [] for repository_metadata_id in util.listify( item_id ): - repository_metadata = suc.get_repository_metadata_by_id( trans, item_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, item_id ) encoded_repository_ids.append( trans.security.encode_id( repository_metadata.repository.id ) ) changeset_revisions.append( repository_metadata.changeset_revision ) new_kwd = {} @@ -1085,14 +1084,15 @@ if repository_metadata: metadata = repository_metadata.metadata if metadata: - repository_dependencies = suc.get_repository_dependencies_for_changeset_revision( trans=trans, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None, - circular_repository_dependencies=None ) + repository_dependencies = \ + repository_dependency_util.get_repository_dependencies_for_changeset_revision( trans=trans, + repository=repository, + repository_metadata=repository_metadata, + toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), + key_rd_dicts_to_be_processed=None, + all_repository_dependencies=None, + handled_key_rd_dicts=None, + circular_repository_dependencies=None ) if repository_dependencies: return encoding_util.tool_shed_encode( repository_dependencies ) return '' @@ -1144,16 +1144,16 @@ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) - repo_info_dict = suc.create_repo_info_dict( trans=trans, - repository_clone_url=repository_clone_url, - changeset_revision=changeset_revision, - ctx_rev=str( ctx.rev() ), - repository_owner=repository.user.username, - repository_name=repository.name, - repository=repository, - repository_metadata=repository_metadata, - tool_dependencies=None, - repository_dependencies=None ) + repo_info_dict = repository_util.create_repo_info_dict( trans=trans, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + ctx_rev=str( ctx.rev() ), + repository_owner=repository.user.username, + repository_name=repository.name, + repository=repository, + repository_metadata=repository_metadata, + tool_dependencies=None, + repository_dependencies=None ) repo_info_dicts.append( encoding_util.tool_shed_encode( repo_info_dict ) ) return dict( includes_tools=includes_tools, includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, @@ -1257,16 +1257,16 @@ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) - repo_info_dict = suc.create_repo_info_dict( trans=trans, - repository_clone_url=repository_clone_url, - changeset_revision=changeset_revision, - ctx_rev=str( ctx.rev() ), - repository_owner=repository.user.username, - repository_name=repository.name, - repository=repository, - repository_metadata=repository_metadata, - tool_dependencies=None, - repository_dependencies=None ) + repo_info_dict = repository_util.create_repo_info_dict( trans=trans, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + ctx_rev=str( ctx.rev() ), + repository_owner=repository.user.username, + repository_name=repository.name, + repository=repository, + repository_metadata=repository_metadata, + tool_dependencies=None, + repository_dependencies=None ) includes_data_managers = False includes_datatypes = False includes_tools = False @@ -1686,15 +1686,16 @@ if repository_metadata: metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. - repository_dependencies = suc.get_repository_dependencies_for_changeset_revision( trans=trans, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + repository_dependencies = \ + repository_dependency_util.get_repository_dependencies_for_changeset_revision( trans=trans, + repository=repository, + repository_metadata=repository_metadata, + toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), + key_rd_dicts_to_be_processed=None, + all_repository_dependencies=None, + handled_key_rd_dicts=None ) # Handle messaging for orphan tool dependencies. - orphan_message = suc.generate_message_for_orphan_tool_dependencies( metadata ) + orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( metadata ) if orphan_message: message += orphan_message status = 'warning' @@ -1809,13 +1810,14 @@ repository_metadata_id = trans.security.encode_id( repository_metadata.id ), metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. - repository_dependencies = suc.get_repository_dependencies_for_changeset_revision( trans=trans, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + repository_dependencies = \ + repository_dependency_util.get_repository_dependencies_for_changeset_revision( trans=trans, + repository=repository, + repository_metadata=repository_metadata, + toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), + key_rd_dicts_to_be_processed=None, + all_repository_dependencies=None, + handled_key_rd_dicts=None ) if metadata: if 'repository_dependencies' in metadata and not repository_dependencies: message += 'The repository dependency definitions for this repository are invalid and will be ignored.' @@ -2254,7 +2256,7 @@ repository = None if repository: repository_id = trans.security.encode_id( repository.id ) - repository_metadata = suc.get_repository_metadata_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) if not repository_metadata: # Get updates to the received changeset_revision if any exist. repo_dir = repository.repo_path( trans.app ) @@ -2262,7 +2264,7 @@ upper_bound_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if upper_bound_changeset_revision: changeset_revision = upper_bound_changeset_revision - repository_metadata = suc.get_repository_metadata_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) if repository_metadata: return trans.response.send_redirect( web.url_for( controller='repository', action='index', @@ -2471,15 +2473,16 @@ if repository_metadata: metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. - repository_dependencies = suc.get_repository_dependencies_for_changeset_revision( trans=trans, - repository=repository, - repository_metadata=repository_metadata, - toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), - key_rd_dicts_to_be_processed=None, - all_repository_dependencies=None, - handled_key_rd_dicts=None ) + repository_dependencies = \ + repository_dependency_util.get_repository_dependencies_for_changeset_revision( trans=trans, + repository=repository, + repository_metadata=repository_metadata, + toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), + key_rd_dicts_to_be_processed=None, + all_repository_dependencies=None, + handled_key_rd_dicts=None ) # Handle messaging for orphan tool dependencies. - orphan_message = suc.generate_message_for_orphan_tool_dependencies( metadata ) + orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( metadata ) if orphan_message: message += orphan_message status = 'warning' @@ -2621,7 +2624,7 @@ status = params.get( 'status', 'done' ) if workflow_name: workflow_name = encoding_util.tool_shed_decode( workflow_name ) - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) repository = suc.get_repository_in_tool_shed( trans, trans.security.encode_id( repository_metadata.repository_id ) ) changeset_revision = repository_metadata.changeset_revision metadata = repository_metadata.metadata diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -3,7 +3,7 @@ from galaxy import web, util from galaxy.datatypes import checkers import tool_shed.util.shed_util_common as suc -import tool_shed.util.metadata_util as metadata_util +from tool_shed.util import metadata_util, repository_dependency_util, tool_dependency_util from galaxy import eggs eggs.require('mercurial') @@ -198,17 +198,17 @@ # so warning messages are important because orphans are always valid. The repository owner must be warned in case they did not intend to define an # orphan dependency, but simply provided incorrect information (tool shed, name owner, changeset_revision) for the definition. # Handle messaging for orphan tool dependencies. - orphan_message = suc.generate_message_for_orphan_tool_dependencies( metadata_dict ) + orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( metadata_dict ) if orphan_message: message += orphan_message status = 'warning' # Handle messaging for invalid tool dependencies. - invalid_tool_dependencies_message = suc.generate_message_for_invalid_tool_dependencies( metadata_dict ) + invalid_tool_dependencies_message = tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict ) if invalid_tool_dependencies_message: message += invalid_tool_dependencies_message status = 'error' # Handle messaging for invalid repository dependencies. - invalid_repository_dependencies_message = suc.generate_message_for_invalid_repository_dependencies( metadata_dict ) + invalid_repository_dependencies_message = repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict ) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/galaxy/webapps/tool_shed/util/workflow_util.py --- a/lib/galaxy/webapps/tool_shed/util/workflow_util.py +++ b/lib/galaxy/webapps/tool_shed/util/workflow_util.py @@ -6,7 +6,7 @@ import logging, svgfig from galaxy.util import json import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util +from tool_shed.util import encoding_util, metadata_util from galaxy.workflow.modules import InputDataModule, ToolModule, WorkflowModuleFactory import galaxy.webapps.galaxy.controllers.workflow import galaxy.tools @@ -152,7 +152,7 @@ workflow_name = encoding_util.tool_shed_decode( workflow_name ) if trans.webapp.name == 'tool_shed': # We're in the tool shed. - repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) repository_id = trans.security.encode_id( repository_metadata.repository_id ) changeset_revision = repository_metadata.changeset_revision metadata = repository_metadata.metadata diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/tool_shed/galaxy_install/__init__.py --- a/lib/tool_shed/galaxy_install/__init__.py +++ b/lib/tool_shed/galaxy_install/__init__.py @@ -2,8 +2,8 @@ Classes encapsulating the management of repositories installed from Galaxy tool sheds. """ import os, logging -import tool_shed.util.shed_util import tool_shed.util.shed_util_common +import tool_shed.util.datatype_util from galaxy.model.orm import and_ from galaxy import eggs @@ -51,13 +51,13 @@ .order_by( self.model.ToolShedRepository.table.c.id ): relative_install_dir = self.get_repository_install_dir( tool_shed_repository ) if relative_install_dir: - installed_repository_dict = tool_shed.util.shed_util.load_installed_datatypes( self.app, tool_shed_repository, relative_install_dir ) + installed_repository_dict = tool_shed.util.datatype_util.load_installed_datatypes( self.app, tool_shed_repository, relative_install_dir ) if installed_repository_dict: self.installed_repository_dicts.append( installed_repository_dict ) def load_proprietary_converters_and_display_applications( self, deactivate=False ): for installed_repository_dict in self.installed_repository_dicts: if installed_repository_dict[ 'converter_path' ]: - tool_shed.util.shed_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=deactivate ) + tool_shed.util.datatype_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=deactivate ) if installed_repository_dict[ 'display_path' ]: - tool_shed.util.shed_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=deactivate ) + tool_shed.util.datatype_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=deactivate ) \ No newline at end of file diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py --- a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py +++ b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py @@ -3,7 +3,7 @@ from galaxy import model, util from galaxy.web.framework.helpers import iff, grids from galaxy.model.orm import or_ -import tool_shed.util.shed_util as shed_util +from tool_shed.util import tool_dependency_util log = logging.getLogger( __name__ ) @@ -345,7 +345,7 @@ ] def build_initial_query( self, trans, **kwd ): - tool_dependency_ids = shed_util.get_tool_dependency_ids( as_string=False, **kwd ) + tool_dependency_ids = tool_dependency_util.get_tool_dependency_ids( as_string=False, **kwd ) if tool_dependency_ids: clause_list = [] for tool_dependency_id in tool_dependency_ids: diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -6,11 +6,9 @@ from galaxy import util from galaxy.tools import ToolSection from galaxy.util.json import from_json_string, to_json_string -import tool_shed.util.shed_util as shed_util import tool_shed.util.shed_util_common as suc -import tool_shed.util.metadata_util as metadata_util +from tool_shed.util import common_install_util, common_util, datatype_util, metadata_util, tool_dependency_util, tool_util from galaxy.util.odict import odict -from tool_shed.util import common_util class InstallManager( object ): def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ): @@ -174,7 +172,7 @@ # See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems. is_displayed, tool_sections = self.get_containing_tool_sections( tool_config ) if is_displayed: - tool_panel_dict_for_tool_config = shed_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections ) + tool_panel_dict_for_tool_config = tool_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections ) for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[ k ] = v else: @@ -195,43 +193,43 @@ self.app.sa_session.flush() if 'tool_dependencies' in metadata_dict: # All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed. - tool_dependencies = shed_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True ) + tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True ) else: tool_dependencies = None if 'tools' in metadata_dict: sample_files = metadata_dict.get( 'sample_files', [] ) sample_files = [ str( s ) for s in sample_files ] - tool_index_sample_files = shed_util.get_tool_index_sample_files( sample_files ) - shed_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=self.tool_path ) + tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) + tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=self.tool_path ) sample_files_copied = [ s for s in tool_index_sample_files ] repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = shed_util.handle_missing_data_table_entry( self.app, relative_install_dir, self.tool_path, repository_tools_tups ) + repository_tools_tups = tool_util.handle_missing_data_table_entry( self.app, relative_install_dir, self.tool_path, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups, sample_files_copied = shed_util.handle_missing_index_file( self.app, + repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups, sample_files_copied ) # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. - shed_util.copy_sample_files( self.app, sample_files, tool_path=self.tool_path, sample_files_copied=sample_files_copied ) + tool_util.copy_sample_files( self.app, sample_files, tool_path=self.tool_path, sample_files_copied=sample_files_copied ) if install_dependencies and tool_dependencies and 'tool_dependencies' in metadata_dict: # Install tool dependencies. - shed_util.update_tool_shed_repository_status( self.app, - tool_shed_repository, - self.app.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + suc.update_tool_shed_repository_status( self.app, + tool_shed_repository, + self.app.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from disk. tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir ) - installed_tool_dependencies = shed_util.handle_tool_dependencies( app=self.app, - tool_shed_repository=tool_shed_repository, - tool_dependencies_config=tool_dependencies_config, - tool_dependencies=tool_dependencies ) + installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=self.app, + tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_dependencies ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == self.app.model.ToolDependency.installation_status.ERROR: print '\nThe following error occurred from the InstallManager while installing tool dependency ', installed_tool_dependency.name, ':' print installed_tool_dependency.error_message, '\n\n' - shed_util.add_to_tool_panel( self.app, + tool_util.add_to_tool_panel( self.app, tool_shed_repository.name, repository_clone_url, tool_shed_repository.installed_changeset_revision, @@ -250,16 +248,16 @@ datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', repo_install_dir ) # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started # after this installation completes. - converter_path, display_path = shed_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir + converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir if converter_path or display_path: # Create a dictionary of tool shed repository related information. - repository_dict = shed_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed, - name=tool_shed_repository.name, - owner=self.repository_owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) + repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed, + name=tool_shed_repository.name, + owner=self.repository_owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) if converter_path: # Load proprietary datatype converters self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict ) @@ -297,7 +295,7 @@ current_changeset_revision=None, owner=self.repository_owner, dist_to_shed=True ) - shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.CLONING ) + suc.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.CLONING ) cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok: self.handle_repository_contents( tool_shed_repository=tool_shed_repository, @@ -308,9 +306,9 @@ self.app.sa_session.refresh( tool_shed_repository ) metadata_dict = tool_shed_repository.metadata if 'tools' in metadata_dict: - shed_util.update_tool_shed_repository_status( self.app, - tool_shed_repository, - self.app.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) + suc.update_tool_shed_repository_status( self.app, + tool_shed_repository, + self.app.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) # Get the tool_versions from the tool shed for each tool in the installed change set. url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ ( tool_shed_url, tool_shed_repository.name, self.repository_owner, installed_changeset_revision ) @@ -319,7 +317,7 @@ response.close() if text: tool_version_dicts = from_json_string( text ) - shed_util.handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) + tool_util.handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) else: # Set the tool versions since they seem to be missing for this repository in the tool shed. # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. @@ -328,8 +326,8 @@ tool_id = tool_dict[ 'guid' ] old_tool_id = tool_dict[ 'id' ] tool_version = tool_dict[ 'version' ] - tool_version_using_old_id = shed_util.get_tool_version( self.app, old_tool_id ) - tool_version_using_guid = shed_util.get_tool_version( self.app, tool_id ) + tool_version_using_old_id = tool_util.get_tool_version( self.app, old_tool_id ) + tool_version_using_guid = tool_util.get_tool_version( self.app, tool_id ) if not tool_version_using_old_id: tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id, tool_shed_repository=tool_shed_repository ) @@ -341,7 +339,7 @@ self.app.sa_session.add( tool_version_using_guid ) self.app.sa_session.flush() # Associate the two versions as parent / child. - tool_version_association = shed_util.get_tool_version_association( self.app, + tool_version_association = tool_util.get_tool_version_association( self.app, tool_version_using_old_id, tool_version_using_guid ) if not tool_version_association: @@ -349,7 +347,7 @@ parent_id=tool_version_using_old_id.id ) self.app.sa_session.add( tool_version_association ) self.app.sa_session.flush() - shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLED ) + suc.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLED ) @property def non_shed_tool_panel_configs( self ): return common_util.get_non_shed_tool_panel_configs( self.app ) diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -1,89 +1,230 @@ -import tool_shed.util.shed_util as shed_util +import os, logging, threading, urllib2 +from galaxy.web import url_for +from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc -import tool_shed.util.metadata_util as metadata_util +from tool_shed.util import encoding_util, repository_dependency_util, tool_dependency_util, tool_util -def handle_repository_contents( app, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, tool_section=None, - shed_tool_conf=None, reinstalling=False ): +from galaxy import eggs +import pkg_resources + +pkg_resources.require( 'mercurial' ) +from mercurial import hg, ui, commands + +log = logging.getLogger( __name__ ) + +def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, repository=None, + repository_metadata=None, tool_dependencies=None, repository_dependencies=None ): """ - Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy (never the tool shed) - when an admin is installing a new repository or reinstalling an uninstalled repository. + Return a dictionary that includes all of the information needed to install a repository into a local Galaxy instance. The dictionary will also + contain the recursive list of repository dependencies defined for the repository, as well as the defined tool dependencies. + + This method is called from Galaxy under three scenarios: + 1. During the tool shed repository installation process via the tool shed's get_repository_information() method. In this case both the received + repository and repository_metadata will be objects., but tool_dependencies and repository_dependencies will be None + 2. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with no updates available. In this case, both + repository and repository_metadata will be None, but tool_dependencies and repository_dependencies will be objects previously retrieved from the + tool shed if the repository includes definitions for them. + 3. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with updates available. In this case, this + method is reached via the tool shed's get_updated_repository_information() method, and both repository and repository_metadata will be objects + but tool_dependencies and repository_dependencies will be None. """ - sa_session = app.model.context.current - shed_config_dict = app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) - metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict=shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) - tool_shed_repository.metadata = metadata_dict - sa_session.add( tool_shed_repository ) - sa_session.flush() - if 'tool_dependencies' in metadata_dict and not reinstalling: - tool_dependencies = shed_util.create_tool_dependency_objects( app, tool_shed_repository, relative_install_dir, set_status=True ) - if 'tools' in metadata_dict: - tool_panel_dict = shed_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) - sample_files = metadata_dict.get( 'sample_files', [] ) - tool_index_sample_files = shed_util.get_tool_index_sample_files( sample_files ) - shed_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) - sample_files_copied = [ str( s ) for s in tool_index_sample_files ] - repository_tools_tups = suc.get_repository_tools_tups( app, metadata_dict ) - if repository_tools_tups: - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = shed_util.handle_missing_data_table_entry( app, relative_install_dir, tool_path, repository_tools_tups ) - # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups, sample_files_copied = shed_util.handle_missing_index_file( app, - tool_path, - sample_files, - repository_tools_tups, - sample_files_copied ) - # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. - shed_util.copy_sample_files( app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) - shed_util.add_to_tool_panel( app=app, - repository_name=tool_shed_repository.name, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - repository_tools_tups=repository_tools_tups, - owner=tool_shed_repository.owner, - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=True ) - if 'data_manager' in metadata_dict: - new_data_managers = shed_util.install_data_managers( app, - app.config.shed_data_manager_config_file, - metadata_dict, - shed_config_dict, - relative_install_dir, - tool_shed_repository, - repository_tools_tups ) - if 'datatypes' in metadata_dict: - tool_shed_repository.status = app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES - if not tool_shed_repository.includes_datatypes: - tool_shed_repository.includes_datatypes = True - sa_session.add( tool_shed_repository ) - sa_session.flush() - files_dir = relative_install_dir - if shed_config_dict.get( 'tool_path' ): - files_dir = os.path.join( shed_config_dict['tool_path'], files_dir ) - datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', files_dir ) - # Load data types required by tools. - converter_path, display_path = shed_util.alter_config_and_load_prorietary_datatypes( app, datatypes_config, files_dir, override=False ) - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = shed_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) - if converter_path: - # Load proprietary datatype converters - app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=repository_dict ) - if display_path: - # Load proprietary datatype display applications - app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + repo_info_dict = {} + repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) + if trans.webapp.name == 'tool_shed': + # We're in the tool shed. + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), changeset_revision ) + if repository_metadata: + metadata = repository_metadata.metadata + if metadata: + # Get a dictionary of all repositories upon which the contents of the received repository depends. + repository_dependencies = \ + repository_dependency_util.get_repository_dependencies_for_changeset_revision( trans=trans, + repository=repository, + repository_metadata=repository_metadata, + toolshed_base_url=str( url_for( '/', qualified=True ) ).rstrip( '/' ), + key_rd_dicts_to_be_processed=None, + all_repository_dependencies=None, + handled_key_rd_dicts=None, + circular_repository_dependencies=None ) + tool_dependencies = metadata.get( 'tool_dependencies', None ) + if tool_dependencies: + new_tool_dependencies = {} + for dependency_key, requirements_dict in tool_dependencies.items(): + if dependency_key in [ 'set_environment' ]: + new_set_environment_dict_list = [] + for set_environment_dict in requirements_dict: + set_environment_dict[ 'repository_name' ] = repository_name + set_environment_dict[ 'repository_owner' ] = repository_owner + set_environment_dict[ 'changeset_revision' ] = changeset_revision + new_set_environment_dict_list.append( set_environment_dict ) + new_tool_dependencies[ dependency_key ] = new_set_environment_dict_list + else: + requirements_dict[ 'repository_name' ] = repository_name + requirements_dict[ 'repository_owner' ] = repository_owner + requirements_dict[ 'changeset_revision' ] = changeset_revision + new_tool_dependencies[ dependency_key ] = requirements_dict + tool_dependencies = new_tool_dependencies + # Cast unicode to string. + repo_info_dict[ str( repository.name ) ] = ( str( repository.description ), + str( repository_clone_url ), + str( changeset_revision ), + str( ctx_rev ), + str( repository_owner ), + repository_dependencies, + tool_dependencies ) + return repo_info_dict + +def get_update_to_changeset_revision_and_ctx_rev( trans, repository ): + """Return the changeset revision hash to which the repository can be updated.""" + changeset_revision_dict = {} + tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) + url = suc.url_join( tool_shed_url, 'repository/get_changeset_revision_and_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % \ + ( repository.name, repository.owner, repository.installed_changeset_revision ) ) + try: + response = urllib2.urlopen( url ) + encoded_update_dict = response.read() + if encoded_update_dict: + update_dict = encoding_util.tool_shed_decode( encoded_update_dict ) + includes_data_managers = update_dict.get( 'includes_data_managers', False ) + includes_datatypes = update_dict.get( 'includes_datatypes', False ) + includes_tools = update_dict.get( 'includes_tools', False ) + includes_tools_for_display_in_tool_panel = update_dict.get( 'includes_tools_for_display_in_tool_panel', False ) + includes_tool_dependencies = update_dict.get( 'includes_tool_dependencies', False ) + includes_workflows = update_dict.get( 'includes_workflows', False ) + has_repository_dependencies = update_dict.get( 'has_repository_dependencies', False ) + changeset_revision = update_dict.get( 'changeset_revision', None ) + ctx_rev = update_dict.get( 'ctx_rev', None ) + response.close() + changeset_revision_dict[ 'includes_data_managers' ] = includes_data_managers + changeset_revision_dict[ 'includes_datatypes' ] = includes_datatypes + changeset_revision_dict[ 'includes_tools' ] = includes_tools + changeset_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] = includes_tools_for_display_in_tool_panel + changeset_revision_dict[ 'includes_tool_dependencies' ] = includes_tool_dependencies + changeset_revision_dict[ 'includes_workflows' ] = includes_workflows + changeset_revision_dict[ 'has_repository_dependencies' ] = has_repository_dependencies + changeset_revision_dict[ 'changeset_revision' ] = changeset_revision + changeset_revision_dict[ 'ctx_rev' ] = ctx_rev + except Exception, e: + log.debug( "Error getting change set revision for update from the tool shed for repository '%s': %s" % ( repository.name, str( e ) ) ) + changeset_revision_dict[ 'includes_data_managers' ] = False + changeset_revision_dict[ 'includes_datatypes' ] = False + changeset_revision_dict[ 'includes_tools' ] = False + changeset_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] = False + changeset_revision_dict[ 'includes_tool_dependencies' ] = False + changeset_revision_dict[ 'includes_workflows' ] = False + changeset_revision_dict[ 'has_repository_dependencies' ] = False + changeset_revision_dict[ 'changeset_revision' ] = None + changeset_revision_dict[ 'ctx_rev' ] = None + return changeset_revision_dict + +def merge_containers_dicts_for_new_install( containers_dicts ): + """ + When installing one or more tool shed repositories for the first time, the received list of containers_dicts contains a containers_dict for + each repository being installed. Since the repositories are being installed for the first time, all entries are None except the repository + dependencies and tool dependencies. The entries for missing dependencies are all None since they have previously been merged into the installed + dependencies. This method will merge the dependencies entries into a single container and return it for display. + """ + new_containers_dict = dict( readme_files=None, + datatypes=None, + missing_repository_dependencies=None, + repository_dependencies=None, + missing_tool_dependencies=None, + tool_dependencies=None, + invalid_tools=None, + valid_tools=None, + workflows=None ) + if containers_dicts: + lock = threading.Lock() + lock.acquire( True ) + try: + repository_dependencies_root_folder = None + tool_dependencies_root_folder = None + # Use a unique folder id (hopefully the following is). + folder_id = 867 + for old_container_dict in containers_dicts: + # Merge repository_dependencies. + old_container_repository_dependencies_root = old_container_dict[ 'repository_dependencies' ] + if old_container_repository_dependencies_root: + if repository_dependencies_root_folder is None: + repository_dependencies_root_folder = container_util.Folder( id=folder_id, key='root', label='root', parent=None ) + folder_id += 1 + repository_dependencies_folder = container_util.Folder( id=folder_id, + key='merged', + label='Repository dependencies', + parent=repository_dependencies_root_folder ) + folder_id += 1 + # The old_container_repository_dependencies_root will be a root folder containing a single sub_folder. + old_container_repository_dependencies_folder = old_container_repository_dependencies_root.folders[ 0 ] + # Change the folder id so it won't confict with others being merged. + old_container_repository_dependencies_folder.id = folder_id + folder_id += 1 + # Generate the label by retrieving the repository name. + toolshed, name, owner, changeset_revision = container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) + old_container_repository_dependencies_folder.label = str( name ) + repository_dependencies_folder.folders.append( old_container_repository_dependencies_folder ) + # Merge tool_dependencies. + old_container_tool_dependencies_root = old_container_dict[ 'tool_dependencies' ] + if old_container_tool_dependencies_root: + if tool_dependencies_root_folder is None: + tool_dependencies_root_folder = container_util.Folder( id=folder_id, key='root', label='root', parent=None ) + folder_id += 1 + tool_dependencies_folder = container_util.Folder( id=folder_id, + key='merged', + label='Tool dependencies', + parent=tool_dependencies_root_folder ) + folder_id += 1 + else: + td_list = [ td.listify for td in tool_dependencies_folder.tool_dependencies ] + # The old_container_tool_dependencies_root will be a root folder containing a single sub_folder. + old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[ 0 ] + for td in old_container_tool_dependencies_folder.tool_dependencies: + if td.listify not in td_list: + tool_dependencies_folder.tool_dependencies.append( td ) + if repository_dependencies_root_folder: + repository_dependencies_root_folder.folders.append( repository_dependencies_folder ) + new_containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder + if tool_dependencies_root_folder: + tool_dependencies_root_folder.folders.append( tool_dependencies_folder ) + new_containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder + except Exception, e: + log.debug( "Exception in merge_containers_dicts_for_new_install: %s" % str( e ) ) + finally: + lock.release() + return new_containers_dict + +def populate_containers_dict_for_new_install( trans, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies, missing_repository_dependencies, + installed_tool_dependencies, missing_tool_dependencies ): + """Return the populated containers for a repository being installed for the first time.""" + installed_tool_dependencies, missing_tool_dependencies = \ + tool_dependency_util.populate_tool_dependencies_dicts( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + repository_installed_tool_dependencies=installed_tool_dependencies, + repository_missing_tool_dependencies=missing_tool_dependencies, + required_repo_info_dicts=None ) + # Since we are installing a new repository, most of the repository contents are set to None since we don't yet know what they are. + containers_dict = suc.build_repository_containers_for_galaxy( trans=trans, + repository=None, + datatypes=None, + invalid_tools=None, + missing_repository_dependencies=missing_repository_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + readme_files_dict=readme_files_dict, + repository_dependencies=installed_repository_dependencies, + tool_dependencies=installed_tool_dependencies, + valid_tools=None, + workflows=None, + valid_data_managers=None, + invalid_data_managers=None, + data_managers_errors=None, + new_install=True, + reinstalling=False ) + # Merge the missing_repository_dependencies container contents to the installed_repository_dependencies container. + containers_dict = repository_dependency_util.merge_missing_repository_dependencies_to_installed_container( containers_dict ) + # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. + containers_dict = tool_dependency_util.merge_missing_tool_dependencies_to_installed_container( containers_dict ) + return containers_dict + +def pull_repository( repo, repository_clone_url, ctx_rev ): + """Pull changes from a remote repository to a local one.""" + commands.pull( suc.get_configured_ui(), repo, source=repository_clone_url, rev=[ ctx_rev ] ) diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -1,7 +1,7 @@ import sys, os, subprocess, tempfile, urllib2 import common_util import fabric_util -from tool_shed.util import encoding_util +from tool_shed.util import encoding_util, tool_dependency_util from galaxy.model.orm import and_ from galaxy.web import url_for @@ -16,24 +16,6 @@ protocol, base = base_url.split( '://' ) return base.rstrip( '/' ) -def create_or_update_tool_dependency( app, tool_shed_repository, name, version, type, status, set_status=True ): - # Called from Galaxy (never the tool shed) when a new repository is being installed or when an uninstalled repository is being reinstalled. - sa_session = app.model.context.current - # First see if an appropriate tool_dependency record exists for the received tool_shed_repository. - if version: - tool_dependency = get_tool_dependency_by_name_version_type_repository( app, tool_shed_repository, name, version, type ) - else: - tool_dependency = get_tool_dependency_by_name_type_repository( app, tool_shed_repository, name, type ) - if tool_dependency: - if set_status: - tool_dependency.status = status - else: - # Create a new tool_dependency record for the tool_shed_repository. - tool_dependency = app.model.ToolDependency( tool_shed_repository.id, name, version, type, status ) - sa_session.add( tool_dependency ) - sa_session.flush() - return tool_dependency - def create_temporary_tool_dependencies_config( tool_shed_url, name, owner, changeset_revision ): """Make a call to the tool shed to get the required repository's tool_dependencies.xml file.""" url = url_join( tool_shed_url, @@ -95,23 +77,6 @@ return tool_shed_repository return None -def get_tool_dependency_by_name_type_repository( app, repository, name, type ): - sa_session = app.model.context.current - return sa_session.query( app.model.ToolDependency ) \ - .filter( and_( app.model.ToolDependency.table.c.tool_shed_repository_id == repository.id, - app.model.ToolDependency.table.c.name == name, - app.model.ToolDependency.table.c.type == type ) ) \ - .first() - -def get_tool_dependency_by_name_version_type_repository( app, repository, name, version, type ): - sa_session = app.model.context.current - return sa_session.query( app.model.ToolDependency ) \ - .filter( and_( app.model.ToolDependency.table.c.tool_shed_repository_id == repository.id, - app.model.ToolDependency.table.c.name == name, - app.model.ToolDependency.table.c.version == version, - app.model.ToolDependency.table.c.type == type ) ) \ - .first() - def get_tool_dependency_install_dir( app, repository_name, repository_owner, repository_changeset_revision, tool_dependency_type, tool_dependency_name, tool_dependency_version ): if tool_dependency_type == 'package': @@ -147,13 +112,13 @@ for package_elem in elem: if package_elem.tag == 'install': # Create the tool_dependency record in the database. - tool_dependency = create_or_update_tool_dependency( app=app, - tool_shed_repository=tool_shed_repository, - name=package_name, - version=package_version, - type='package', - status=app.model.ToolDependency.installation_status.INSTALLING, - set_status=True ) + tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, + tool_shed_repository=tool_shed_repository, + name=package_name, + version=package_version, + type='package', + status=app.model.ToolDependency.installation_status.INSTALLING, + set_status=True ) # Get the installation method version from a tag like: <install version="1.0"> package_install_version = package_elem.get( 'version', '1.0' ) if package_install_version == '1.0': @@ -296,13 +261,13 @@ elif package_elem.tag == 'install': # <install version="1.0"> package_install_version = package_elem.get( 'version', '1.0' ) - tool_dependency = create_or_update_tool_dependency( app=app, - tool_shed_repository=tool_shed_repository, - name=package_name, - version=package_version, - type='package', - status=app.model.ToolDependency.installation_status.INSTALLING, - set_status=True ) + tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, + tool_shed_repository=tool_shed_repository, + name=package_name, + version=package_version, + type='package', + status=app.model.ToolDependency.installation_status.INSTALLING, + set_status=True ) if package_install_version == '1.0': # Handle tool dependency installation using a fabric method included in the Galaxy framework. for actions_elem in package_elem: @@ -324,7 +289,11 @@ # print 'Installing tool dependencies via fabric script ', proprietary_fabfile_path else: print '\nSkipping installation of tool dependency', package_name, 'version', package_version, 'since it is installed in', install_dir, '\n' - tool_dependency = get_tool_dependency_by_name_version_type_repository( app, tool_shed_repository, package_name, package_version, 'package' ) + tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( app, + tool_shed_repository, + package_name, + package_version, + 'package' ) tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add( tool_dependency ) sa_session.flush() @@ -550,13 +519,13 @@ if env_var_dict: if not os.path.exists( install_dir ): os.makedirs( install_dir ) - tool_dependency = create_or_update_tool_dependency( app=app, - tool_shed_repository=tool_shed_repository, - name=env_var_name, - version=None, - type='set_environment', - status=app.model.ToolDependency.installation_status.INSTALLING, - set_status=True ) + tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, + tool_shed_repository=tool_shed_repository, + name=env_var_name, + version=None, + type='set_environment', + status=app.model.ToolDependency.installation_status.INSTALLING, + set_status=True ) cmd = common_util.create_or_update_env_shell_file( install_dir, env_var_dict ) if env_var_version == '1.0': # Handle setting environment variables using a fabric method. diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/tool_shed/galaxy_install/update_manager.py --- a/lib/tool_shed/galaxy_install/update_manager.py +++ b/lib/tool_shed/galaxy_install/update_manager.py @@ -3,7 +3,6 @@ """ import threading, urllib2, logging from galaxy.util import string_as_bool -import tool_shed.util.shed_util as shed_util import tool_shed.util.shed_util_common as suc from galaxy.model.orm import and_ diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -5,6 +5,7 @@ from galaxy.util import json import tool_shed.util.shed_util_common as suc import tool_shed.grids.util as grids_util +from tool_shed.util import metadata_util from galaxy import eggs eggs.require('markupsafe') @@ -776,12 +777,16 @@ required_repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) if required_repository: required_repository_id = trans.security.encode_id( required_repository.id ) - required_repository_metadata = suc.get_repository_metadata_by_repository_id_changeset_revision( trans, required_repository_id, changeset_revision ) + required_repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, + required_repository_id, + changeset_revision ) if not required_repository_metadata: repo_dir = required_repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) updated_changeset_revision = suc.get_next_downloadable_changeset_revision( required_repository, repo, changeset_revision ) - required_repository_metadata = suc.get_repository_metadata_by_repository_id_changeset_revision( trans, required_repository_id, updated_changeset_revision ) + required_repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, + required_repository_id, + updated_changeset_revision ) required_repository_metadata_id = trans.security.encode_id( required_repository_metadata.id ) rd_str += '<a href="browse_repository_dependencies?operation=view_or_manage_repository&id=%s">' % ( required_repository_metadata_id ) rd_str += 'Repository %s revision %s owned by %s' % ( escape_html( rd_tup[ 1 ] ), escape_html( rd_tup[ 3 ] ), escape_html( rd_tup[ 2 ] ) ) diff -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 -r 320c9b7f08c42c16ac942f05329c134dcdbb2770 lib/tool_shed/grids/repository_review_grids.py --- a/lib/tool_shed/grids/repository_review_grids.py +++ b/lib/tool_shed/grids/repository_review_grids.py @@ -4,7 +4,7 @@ from galaxy.model.orm import and_, or_ from tool_shed.grids.repository_grids import RepositoryGrid import tool_shed.util.shed_util_common as suc -import tool_shed.util.metadata_util as metadata_util +from tool_shed.util import metadata_util from galaxy import eggs eggs.require('mercurial') @@ -73,7 +73,7 @@ class WithoutReviewsRevisionColumn( grids.GridColumn ): def get_value( self, trans, grid, repository ): # Restrict the options to revisions that have not yet been reviewed. - repository_metadata_revisions = suc.get_repository_metadata_revisions_for_review( repository, reviewed=False ) + repository_metadata_revisions = metadata_util.get_repository_metadata_revisions_for_review( repository, reviewed=False ) if repository_metadata_revisions: rval = '' for repository_metadata in repository_metadata_revisions: This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Also require and pass-through authentication for the 'pushkey' mercurial command when pushing changes to a tool shed repository.
by commits-noreply＠bitbucket.org 15 Mar '13

15 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/62bd6fe0e9aa/ changeset: 62bd6fe0e9aa user: inithello date: 2013-03-15 20:11:04 summary: Also require and pass-through authentication for the 'pushkey' mercurial command when pushing changes to a tool shed repository. affected #: 1 file diff -r 750222a21eff68b96069d853a4744d6e3682508c -r 62bd6fe0e9aa297b4448dfbcae32d27c8e313c38 lib/galaxy/webapps/tool_shed/framework/middleware/hg.py --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -56,12 +56,12 @@ times_downloaded += 1 connection.execute( "update repository set times_downloaded = %d where user_id = %d and name = '%s'" % ( times_downloaded, user_id, name.lower() ) ) connection.close() - if cmd == 'unbundle': + if cmd in [ 'unbundle', 'pushkey' ]: # This is an hg push from the command line. When doing this, the following commands, in order, # will be retrieved from environ (see the docs at http://mercurial.selenic.com/wiki/WireProtocol) - # # If mercurial version >= '2.2.3': capabilities -> batch -> branchmap -> unbundle -> listkeys -> pushkey + # # If mercurial version >= '2.2.3': capabilities -> batch -> branchmap -> unbundle -> listkeys -> pushkey -> listkeys # - # The mercurial API unbundle() ( i.e., hg push ) method ultimately requires authorization. + # The mercurial API unbundle() ( i.e., hg push ) and pushkey() methods ultimately require authorization. # We'll force password entry every time a change set is pushed. # # When a user executes hg commit, it is not guaranteed to succeed. Mercurial records your name Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Fixed: No confirm on delete permanently (footer button), saved histories
by commits-noreply＠bitbucket.org 15 Mar '13

15 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/750222a21eff/ changeset: 750222a21eff user: guerler date: 2013-03-15 19:32:30 summary: Fixed: No confirm on delete permanently (footer button), saved histories affected #: 3 files diff -r f1bd817063e092711a9b13f60e569abbce877fba -r 750222a21eff68b96069d853a4744d6e3682508c static/scripts/galaxy.grids.js --- a/static/scripts/galaxy.grids.js +++ b/static/scripts/galaxy.grids.js @@ -541,4 +541,28 @@ } } init_grid_elements(); +} + + +// confirmation/submission of operation request +function submit_operation(selected_button, confirmation_text) +{ + // verify in any item is selected + var number_of_checked_ids = $('input[name="id"]:checked').length; + if (!number_of_checked_ids > 0) + return false; + + // show confirmation box + if (confirmation_text != 'None' && confirmation_text != '') + if(!confirm(confirmation_text)) + return false; + + // set up hidden field to parse the command/operation to controller + $('#operation').val(selected_button.value); + + // submit form + selected_button.form.submit(); + + // return + return true; } \ No newline at end of file diff -r f1bd817063e092711a9b13f60e569abbce877fba -r 750222a21eff68b96069d853a4744d6e3682508c static/scripts/packed/galaxy.grids.js --- a/static/scripts/packed/galaxy.grids.js +++ b/static/scripts/packed/galaxy.grids.js @@ -1,1 +1,1 @@ -jQuery.ajaxSettings.traditional=true;var Grid=Backbone.Model.extend({defaults:{url_base:"",async:false,async_ops:[],categorical_filters:[],filters:{},sort_key:null,show_item_checkboxes:false,cur_page:1,num_pages:1,operation:undefined,item_ids:undefined},can_async_op:function(a){return _.indexOf(this.attributes.async_ops,a)!==-1},add_filter:function(e,f,b){if(b){var c=this.attributes.filters[e],a;if(c===null||c===undefined){a=f}else{if(typeof(c)=="string"){if(c=="All"){a=f}else{var d=[];d[0]=c;d[1]=f;a=d}}else{a=c;a.push(f)}}this.attributes.filters[e]=a}else{this.attributes.filters[e]=f}},remove_filter:function(b,e){var a=this.attributes.filters[b];if(a===null||a===undefined){return false}var d=true;if(typeof(a)==="string"){if(a=="All"){d=false}else{delete this.attributes.filters[b]}}else{var c=_.indexOf(a,e);if(c!==-1){a.splice(c,1)}else{d=false}}return d},get_url_data:function(){var a={async:this.attributes.async,sort:this.attributes.sort_key,page:this.attributes.cur_page,show_item_checkboxes:this.attributes.show_item_checkboxes};if(this.attributes.operation){a.operation=this.attributes.operation}if(this.attributes.item_ids){a.id=this.attributes.item_ids}var b=this;_.each(_.keys(b.attributes.filters),function(c){a["f-"+c]=b.attributes.filters[c]});return a}});function init_operation_buttons(){$("input[name=operation]:submit").each(function(){$(this).click(function(){var b=$(this).val();var a=[];$("input[name=id]:checked").each(function(){a.push($(this).val())});do_operation(b,a)})})}function init_grid_controls(){init_operation_buttons();$(".submit-image").each(function(){$(this).mousedown(function(){$(this).addClass("gray-background")});$(this).mouseup(function(){$(this).removeClass("gray-background")})});$(".sort-link").each(function(){$(this).click(function(){set_sort_condition($(this).attr("sort_key"));return false})});$(".page-link > a").each(function(){$(this).click(function(){set_page($(this).attr("page_num"));return false})});$(".categorical-filter > a").each(function(){$(this).click(function(){set_categorical_filter($(this).attr("filter_key"),$(this).attr("filter_val"));return false})});$(".text-filter-form").each(function(){$(this).submit(function(){var d=$(this).attr("column_key");var c=$("#input-"+d+"-filter");var e=c.val();c.val("");add_filter_condition(d,e,true);return false})});var a=$("#input-tags-filter");if(a.length){a.autocomplete(history_tag_autocomplete_url,{selectFirst:false,autoFill:false,highlight:false,mustMatch:false})}var b=$("#input-name-filter");if(b.length){b.autocomplete(history_name_autocomplete_url,{selectFirst:false,autoFill:false,highlight:false,mustMatch:false})}$(".advanced-search-toggle").each(function(){$(this).click(function(){$("#standard-search").slideToggle("fast");$("#advanced-search").slideToggle("fast");return false})})}function init_grid_elements(){$(".grid").each(function(){var b=$(this).find("input.grid-row-select-checkbox");var a=$(this).find("span.grid-selected-count");var c=function(){a.text($(b).filter(":checked").length)};$(b).each(function(){$(this).change(c)});c()});$(".label").each(function(){var a=$(this).attr("href");if(a!==undefined&&a.indexOf("operation=")!=-1){$(this).click(function(){do_operation_from_href($(this).attr("href"));return false})}});$(".community_rating_star").rating({});make_popup_menus()}function go_page_one(){var a=grid.get("cur_page");if(a!==null&&a!==undefined&&a!=="all"){grid.set("cur_page",1)}}function add_filter_condition(c,e,a){if(e===""){return false}grid.add_filter(c,e,a);var d=$(""+e+"<a href='javascript:void(0);'></a>");d.addClass("text-filter-val");d.click(function(){grid.remove_filter(c,e);$(this).remove();go_page_one();update_grid()});var b=$("#"+c+"-filtering-criteria");b.append(d);go_page_one();update_grid()}function add_tag_to_grid_filter(c,b){var a=c+(b!==undefined&&b!==""?":"+b:"");$("#advanced-search").show("fast");add_filter_condition("tags",a,true)}function set_sort_condition(f){var e=grid.get("sort_key");var d=f;if(e.indexOf(f)!==-1){if(e.substring(0,1)!=="-"){d="-"+f}else{}}$(".sort-arrow").remove();var c=(d.substring(0,1)=="-")?"↑":"↓";var a=$(""+c+"").addClass("sort-arrow");var b=$("#"+f+"-header");b.append(a);grid.set("sort_key",d);go_page_one();update_grid()}function set_categorical_filter(b,d){var a=grid.get("categorical_filters")[b],c=grid.get("filters")[b];$("."+b+"-filter").each(function(){var h=$.trim($(this).text());var f=a[h];var g=f[b];if(g==d){$(this).empty();$(this).addClass("current-filter");$(this).append(h)}else{if(g==c){$(this).empty();var e=$("<a href='#'>"+h+"</a>");e.click(function(){set_categorical_filter(b,g)});$(this).removeClass("current-filter");$(this).append(e)}}});grid.add_filter(b,d);go_page_one();update_grid()}function set_page(a){$(".page-link").each(function(){var g=$(this).attr("id"),e=parseInt(g.split("-")[2],10),c=grid.get("cur_page"),f;if(e===a){f=$(this).children().text();$(this).empty();$(this).addClass("inactive-link");$(this).text(f)}else{if(e===c){f=$(this).text();$(this).empty();$(this).removeClass("inactive-link");var d=$("<a href='#'>"+f+"</a>");d.click(function(){set_page(e)});$(this).append(d)}}});var b=true;if(a==="all"){grid.set("cur_page",a);b=false}else{grid.set("cur_page",parseInt(a,10))}update_grid(b)}function do_operation(b,a){b=b.toLowerCase();grid.set({operation:b,item_ids:a});if(grid.can_async_op(b)){update_grid(true)}else{go_to_URL()}}function do_operation_from_href(c){var f=c.split("?");if(f.length>1){var a=f[1];var e=a.split("&");var b=null;var g=-1;for(var d=0;d<e.length;d++){if(e[d].indexOf("operation")!=-1){b=e[d].split("=")[1]}else{if(e[d].indexOf("id")!=-1){g=e[d].split("=")[1]}}}do_operation(b,g);return false}}function go_to_URL(){grid.set("async",false);window.location=grid.get("url_base")+"?"+$.param(grid.get_url_data())}function update_grid(a){if(!grid.get("async")){go_to_URL();return}var b=(grid.get("operation")?"POST":"GET");$(".loading-elt-overlay").show();$.ajax({type:b,url:grid.get("url_base"),data:grid.get_url_data(),error:function(){alert("Grid refresh failed")},success:function(d){var c=d.split("*****");$("#grid-table-body").html(c[0]);$("#grid-table-footer").html(c[1]);$("#grid-table-body").trigger("update");init_grid_elements();init_operation_buttons();make_popup_menus();$(".loading-elt-overlay").hide();var e=$.trim(c[2]);if(e!==""){$("#grid-message").html(e).show();setTimeout(function(){$("#grid-message").hide()},5000)}},complete:function(){grid.set({operation:undefined,item_ids:undefined})}})}function check_all_items(){var a=document.getElementById("check_all"),b=document.getElementsByTagName("input"),d=0,c;if(a.checked===true){for(c=0;c<b.length;c++){if(b[c].name.indexOf("id")!==-1){b[c].checked=true;d++}}}else{for(c=0;c<b.length;c++){if(b[c].name.indexOf("id")!==-1){b[c].checked=false}}}init_grid_elements()}; \ No newline at end of file +jQuery.ajaxSettings.traditional=true;var Grid=Backbone.Model.extend({defaults:{url_base:"",async:false,async_ops:[],categorical_filters:[],filters:{},sort_key:null,show_item_checkboxes:false,cur_page:1,num_pages:1,operation:undefined,item_ids:undefined},can_async_op:function(a){return _.indexOf(this.attributes.async_ops,a)!==-1},add_filter:function(e,f,b){if(b){var c=this.attributes.filters[e],a;if(c===null||c===undefined){a=f}else{if(typeof(c)=="string"){if(c=="All"){a=f}else{var d=[];d[0]=c;d[1]=f;a=d}}else{a=c;a.push(f)}}this.attributes.filters[e]=a}else{this.attributes.filters[e]=f}},remove_filter:function(b,e){var a=this.attributes.filters[b];if(a===null||a===undefined){return false}var d=true;if(typeof(a)==="string"){if(a=="All"){d=false}else{delete this.attributes.filters[b]}}else{var c=_.indexOf(a,e);if(c!==-1){a.splice(c,1)}else{d=false}}return d},get_url_data:function(){var a={async:this.attributes.async,sort:this.attributes.sort_key,page:this.attributes.cur_page,show_item_checkboxes:this.attributes.show_item_checkboxes};if(this.attributes.operation){a.operation=this.attributes.operation}if(this.attributes.item_ids){a.id=this.attributes.item_ids}var b=this;_.each(_.keys(b.attributes.filters),function(c){a["f-"+c]=b.attributes.filters[c]});return a}});function init_operation_buttons(){$("input[name=operation]:submit").each(function(){$(this).click(function(){var b=$(this).val();var a=[];$("input[name=id]:checked").each(function(){a.push($(this).val())});do_operation(b,a)})})}function init_grid_controls(){init_operation_buttons();$(".submit-image").each(function(){$(this).mousedown(function(){$(this).addClass("gray-background")});$(this).mouseup(function(){$(this).removeClass("gray-background")})});$(".sort-link").each(function(){$(this).click(function(){set_sort_condition($(this).attr("sort_key"));return false})});$(".page-link > a").each(function(){$(this).click(function(){set_page($(this).attr("page_num"));return false})});$(".categorical-filter > a").each(function(){$(this).click(function(){set_categorical_filter($(this).attr("filter_key"),$(this).attr("filter_val"));return false})});$(".text-filter-form").each(function(){$(this).submit(function(){var d=$(this).attr("column_key");var c=$("#input-"+d+"-filter");var e=c.val();c.val("");add_filter_condition(d,e,true);return false})});var a=$("#input-tags-filter");if(a.length){a.autocomplete(history_tag_autocomplete_url,{selectFirst:false,autoFill:false,highlight:false,mustMatch:false})}var b=$("#input-name-filter");if(b.length){b.autocomplete(history_name_autocomplete_url,{selectFirst:false,autoFill:false,highlight:false,mustMatch:false})}$(".advanced-search-toggle").each(function(){$(this).click(function(){$("#standard-search").slideToggle("fast");$("#advanced-search").slideToggle("fast");return false})})}function init_grid_elements(){$(".grid").each(function(){var b=$(this).find("input.grid-row-select-checkbox");var a=$(this).find("span.grid-selected-count");var c=function(){a.text($(b).filter(":checked").length)};$(b).each(function(){$(this).change(c)});c()});$(".label").each(function(){var a=$(this).attr("href");if(a!==undefined&&a.indexOf("operation=")!=-1){$(this).click(function(){do_operation_from_href($(this).attr("href"));return false})}});$(".community_rating_star").rating({});make_popup_menus()}function go_page_one(){var a=grid.get("cur_page");if(a!==null&&a!==undefined&&a!=="all"){grid.set("cur_page",1)}}function add_filter_condition(c,e,a){if(e===""){return false}grid.add_filter(c,e,a);var d=$(""+e+"<a href='javascript:void(0);'></a>");d.addClass("text-filter-val");d.click(function(){grid.remove_filter(c,e);$(this).remove();go_page_one();update_grid()});var b=$("#"+c+"-filtering-criteria");b.append(d);go_page_one();update_grid()}function add_tag_to_grid_filter(c,b){var a=c+(b!==undefined&&b!==""?":"+b:"");$("#advanced-search").show("fast");add_filter_condition("tags",a,true)}function set_sort_condition(f){var e=grid.get("sort_key");var d=f;if(e.indexOf(f)!==-1){if(e.substring(0,1)!=="-"){d="-"+f}else{}}$(".sort-arrow").remove();var c=(d.substring(0,1)=="-")?"↑":"↓";var a=$(""+c+"").addClass("sort-arrow");var b=$("#"+f+"-header");b.append(a);grid.set("sort_key",d);go_page_one();update_grid()}function set_categorical_filter(b,d){var a=grid.get("categorical_filters")[b],c=grid.get("filters")[b];$("."+b+"-filter").each(function(){var h=$.trim($(this).text());var f=a[h];var g=f[b];if(g==d){$(this).empty();$(this).addClass("current-filter");$(this).append(h)}else{if(g==c){$(this).empty();var e=$("<a href='#'>"+h+"</a>");e.click(function(){set_categorical_filter(b,g)});$(this).removeClass("current-filter");$(this).append(e)}}});grid.add_filter(b,d);go_page_one();update_grid()}function set_page(a){$(".page-link").each(function(){var g=$(this).attr("id"),e=parseInt(g.split("-")[2],10),c=grid.get("cur_page"),f;if(e===a){f=$(this).children().text();$(this).empty();$(this).addClass("inactive-link");$(this).text(f)}else{if(e===c){f=$(this).text();$(this).empty();$(this).removeClass("inactive-link");var d=$("<a href='#'>"+f+"</a>");d.click(function(){set_page(e)});$(this).append(d)}}});var b=true;if(a==="all"){grid.set("cur_page",a);b=false}else{grid.set("cur_page",parseInt(a,10))}update_grid(b)}function do_operation(b,a){b=b.toLowerCase();grid.set({operation:b,item_ids:a});if(grid.can_async_op(b)){update_grid(true)}else{go_to_URL()}}function do_operation_from_href(c){var f=c.split("?");if(f.length>1){var a=f[1];var e=a.split("&");var b=null;var g=-1;for(var d=0;d<e.length;d++){if(e[d].indexOf("operation")!=-1){b=e[d].split("=")[1]}else{if(e[d].indexOf("id")!=-1){g=e[d].split("=")[1]}}}do_operation(b,g);return false}}function go_to_URL(){grid.set("async",false);window.location=grid.get("url_base")+"?"+$.param(grid.get_url_data())}function update_grid(a){if(!grid.get("async")){go_to_URL();return}var b=(grid.get("operation")?"POST":"GET");$(".loading-elt-overlay").show();$.ajax({type:b,url:grid.get("url_base"),data:grid.get_url_data(),error:function(){alert("Grid refresh failed")},success:function(d){var c=d.split("*****");$("#grid-table-body").html(c[0]);$("#grid-table-footer").html(c[1]);$("#grid-table-body").trigger("update");init_grid_elements();init_operation_buttons();make_popup_menus();$(".loading-elt-overlay").hide();var e=$.trim(c[2]);if(e!==""){$("#grid-message").html(e).show();setTimeout(function(){$("#grid-message").hide()},5000)}},complete:function(){grid.set({operation:undefined,item_ids:undefined})}})}function check_all_items(){var a=document.getElementById("check_all"),b=document.getElementsByTagName("input"),d=0,c;if(a.checked===true){for(c=0;c<b.length;c++){if(b[c].name.indexOf("id")!==-1){b[c].checked=true;d++}}}else{for(c=0;c<b.length;c++){if(b[c].name.indexOf("id")!==-1){b[c].checked=false}}}init_grid_elements()}function submit_operation(b,c){var a=$('input[name="id"]:checked').length;if(!a>0){return false}if(c!="None"&&c!=""){if(!confirm(c)){return false}}$("#operation").val(b.value);b.form.submit();return true}; \ No newline at end of file diff -r f1bd817063e092711a9b13f60e569abbce877fba -r 750222a21eff68b96069d853a4744d6e3682508c templates/grid_base.mako --- a/templates/grid_base.mako +++ b/templates/grid_base.mako @@ -459,12 +459,14 @@ ## Grid operations for multiple items. %if show_item_checkboxes: <tr> + ## place holder for multiple operation commands + <input type="hidden" id="operation" name="operation" value=""><td></td><td colspan="100"> For selected ${items_plural}: %for operation in grid.operations: %if operation.allow_multiple: - <input type="submit" name="operation" value="${operation.label}" class="action-button"> + <input type="button" value="${operation.label}" class="action-button" onclick="submit_operation(this, '${operation.confirm}')"> %endif %endfor </td> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: browser tests: cleanup, unify datasource format, add more documentation
by commits-noreply＠bitbucket.org 13 Mar '13

13 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f1bd817063e0/ changeset: f1bd817063e0 user: carlfeberhard date: 2013-03-13 23:58:42 summary: browser tests: cleanup, unify datasource format, add more documentation affected #: 7 files diff -r c2f310489973fc2ba42e5289ccec3ed88a42aabc -r f1bd817063e092711a9b13f60e569abbce877fba test/casperjs/anon-history-tests.js --- a/test/casperjs/anon-history-tests.js +++ b/test/casperjs/anon-history-tests.js @@ -39,12 +39,9 @@ spaceghost.info( 'Will use fixtureData.testUser: ' + email ); } -var tooltipSelector = '.bs-tooltip', - - editableTextClass = 'editable-text', - editableTextInputSelector = 'input#renaming-active', - - galaxyCookieName = 'galaxysession'; +var tooltipSelector = spaceghost.data.selectors.tooltipBalloon, + editableTextClass = spaceghost.data.selectors.editableText, + editableTextInput = spaceghost.data.selectors.editableTextInput, unnamedName = spaceghost.historypanel.data.text.history.newName, nameSelector = spaceghost.historypanel.data.selectors.history.name, @@ -57,6 +54,7 @@ anonNameTooltip = spaceghost.historypanel.data.text.anonymous.tooltips.name; var historyFrameInfo = {}, + filepathToUpload = '../../test-data/1.txt', testUploadInfo = {}; @@ -69,18 +67,6 @@ if( loggedInAs ){ this.logout(); } }); -// ------------------------------------------------------------------- check anon cookies -spaceghost.then( function testAnonCookies(){ - this.test.comment( 'session cookie for anon-user should be present and well formed' ); - var cookies = this.page.cookies; - this.debug( this.jsonStr( this.page.cookies ) ); - //??: what are 'well formed' values? - this.test.assert( cookies.length === 1, "Has one cookie" ); - var galaxyCookie = cookies[0]; - this.test.assert( galaxyCookie.name === galaxyCookieName, "Cookie named: " + galaxyCookieName ); - this.test.assert( !galaxyCookie.secure, "Cookie.secure is false" ); -}); - // ------------------------------------------------------------------- check the empty history for well formedness // grab the history frame bounds for mouse later tests spaceghost.then( function(){ @@ -90,7 +76,7 @@ spaceghost.thenOpen( spaceghost.baseUrl, function testPanelStructure(){ this.test.comment( 'history panel for anonymous user, new history' ); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.test.comment( "frame should have proper url and title: 'History'" ); this.test.assertMatch( this.getCurrentUrl(), /\/history/, 'Found history frame url' ); this.test.assertTitle( this.getTitle(), 'History', 'Found history frame title' ); @@ -118,33 +104,29 @@ this.test.comment( 'name should have a tooltip with info on anon-user name editing' ); // mouse over to find tooltip - //NOTE!!: bounds are returned relative to containing frame - need to adjust using historyFrameInfo - //TODO: into conv. fn - var nameInfo = this.getElementInfo( nameSelector ); - //this.debug( 'nameInfo:' + this.jsonStr( nameInfo ) ); - this.page.sendEvent( 'mousemove', - historyFrameInfo.x + nameInfo.x + 1, historyFrameInfo.y + nameInfo.y + 1 ); - this.test.assertExists( tooltipSelector, "Found tooltip after name hover" ); - this.test.assertSelectorHasText( tooltipSelector, anonNameTooltip ); + this.historypanel.hoverOver( nameSelector, function testingHover(){ + this.test.assertExists( tooltipSelector, "Found tooltip after name hover" ); + this.test.assertSelectorHasText( tooltipSelector, anonNameTooltip ); + }, historyFrameInfo ); this.test.comment( 'name should NOT be editable when clicked by anon-user' ); - this.test.assert( nameInfo.attributes[ 'class' ].indexOf( editableTextClass ) === -1, - "Name field is not class for editable text" ); + this.assertDoesntHaveClass( nameSelector, editableTextClass, + "Name field is not classed as editable text" ); this.click( nameSelector ); - this.test.assertDoesntExist( editableTextInputSelector, "Clicking on name does not create an input" ); + this.test.assertDoesntExist( editableTextInput, "Clicking on name does not create an input" ); }); }); // ------------------------------------------------------------------- anon user can upload file spaceghost.then( function testAnonUpload(){ this.test.comment( 'anon-user should be able to upload files' ); - spaceghost.tools.uploadFile( '../../test-data/1.txt', function uploadCallback( _uploadInfo ){ + spaceghost.tools.uploadFile( filepathToUpload, function uploadCallback( _uploadInfo ){ this.debug( 'uploaded HDA info: ' + this.jsonStr( _uploadInfo ) ); var hasHda = _uploadInfo.hdaElement, hasClass = _uploadInfo.hdaElement.attributes[ 'class' ], hasOkClass = _uploadInfo.hdaElement.attributes[ 'class' ].indexOf( 'historyItem-ok' ) !== -1; this.test.assert( ( hasHda && hasClass && hasOkClass ), "Uploaded file: " + _uploadInfo.name ); - uploadInfo = _uploadInfo; + testUploadInfo = _uploadInfo; }); }); spaceghost.then( function testAnonUpload(){ @@ -157,18 +139,17 @@ // ------------------------------------------------------------------- anon user registers/logs in -> same history spaceghost.user.loginOrRegisterUser( email, password ); -//??: why is a reload needed here? If we don't, loggedInAs === '' ... spaceghost.thenOpen( spaceghost.baseUrl, function(){ this.test.comment( 'anon-user should login and be associated with previous history' ); var loggedInAs = spaceghost.user.loggedInAs(); this.test.assert( loggedInAs === email, 'loggedInAs() matches email: "' + loggedInAs + '"' ); - this.withFrame( this.selectors.frames.history, function(){ - var hdaInfo = this.historypanel.hdaElementInfoByTitle( uploadInfo.name, uploadInfo.hid ); + this.withFrame( spaceghost.data.selectors.frames.history, function(){ + var hdaInfo = this.historypanel.hdaElementInfoByTitle( testUploadInfo.name, testUploadInfo.hid ); this.test.assert( hdaInfo !== null, "After logging in - found a matching hda by name and hid" ); if( hdaInfo ){ - this.test.assert( uploadInfo.hdaElement.attributes.id === hdaInfo.attributes.id, + this.test.assert( testUploadInfo.hdaElement.attributes.id === hdaInfo.attributes.id, "After logging in - found a matching hda by hda view id: " + hdaInfo.attributes.id ); } }); @@ -177,8 +158,7 @@ spaceghost.user.logout(); spaceghost.thenOpen( spaceghost.baseUrl, function(){ this.test.comment( 'logging out should create a new, anonymous history' ); - - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.test.assertSelectorHasText( nameSelector, unnamedName, 'History name is ' + unnamedName ); this.test.assertSelectorHasText( emptyMsgSelector, emptyMsgStr, 'Message contains "' + emptyMsgStr + '"' ); diff -r c2f310489973fc2ba42e5289ccec3ed88a42aabc -r f1bd817063e092711a9b13f60e569abbce877fba test/casperjs/hda-state-tests.js --- a/test/casperjs/hda-state-tests.js +++ b/test/casperjs/hda-state-tests.js @@ -40,7 +40,7 @@ spaceghost.info( 'Will use fixtureData.testUser: ' + email ); } -var tooltipSelector = '.bs-tooltip'; +var tooltipSelector = spaceghost.data.selectors.tooltipBalloon; var utils = require( 'utils' ), historyFrameInfo = {}, @@ -70,8 +70,7 @@ }); -// =================================================================== TESTS -// ------------------------------------------------------------------- helpers +// =================================================================== TEST HELPERS //NOTE: to be called with fn.call( spaceghost, ... ) function testTitle( hdaSelector, hid, name ){ @@ -238,19 +237,18 @@ testPeek.call( this, hdaSelector, peekShouldBeArray ); } +// =================================================================== TESTS // ------------------------------------------------------------------- ok state spaceghost.then( function checkOkState(){ this.test.comment( 'HDAs in the "ok" state should be well formed' ); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ var uploadSelector = '#' + testUploadInfo.hdaElement.attributes.id; this.test.assertVisible( uploadSelector, 'HDA is visible' ); this.test.comment( 'should have the proper state class' ); - var okStateClass = this.historypanel.data.selectors.hda.wrapper.stateClasses.ok, - uploadElement = this.getElementInfo( uploadSelector ); - this.test.assert( uploadElement.attributes['class'].indexOf( okStateClass ) !== -1, - 'HDA has "ok" state class' ); + this.assertHasClass( uploadSelector, this.historypanel.data.selectors.hda.wrapper.stateClasses.ok, + 'HDA has ok state class' ); // since we're using css there's no great way to test state icon (.state-icon is empty) @@ -273,11 +271,27 @@ }); }); -/* +// restore to collapsed +spaceghost.then( function collapseOkState(){ + this.test.comment( "Collapsing hda in 'ok' state should hide body again" ); + this.withFrame( spaceghost.data.selectors.frames.history, function(){ + var uploadSelector = '#' + testUploadInfo.hdaElement.attributes.id, + hdaTitle = uploadSelector + ' ' + this.historypanel.data.selectors.hda.title; + body = uploadSelector + ' ' + this.historypanel.data.selectors.hda.body; + + this.click( hdaTitle ); + this.wait( 500, function(){ + this.test.assertNotVisible( body, 'body is not visible' ); + }); + }); +}); + + +// ------------------------------------------------------------------- new state spaceghost.then( function checkNewState(){ this.test.comment( 'HDAs in the "new" state should be well formed' ); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ // set state directly through model //TODO: not ideal this.evaluate( function(){ @@ -291,37 +305,45 @@ // should have proper title and hid testTitle.call( spaceghost, uploadSelector, testUploadInfo.hid, testUploadInfo.name ); - // should have the new state class - var newStateClass = this.historypanel.data.selectors.hda.wrapper.stateClasses['new']; - uploadElement = this.getElementInfo( uploadSelector ); - this.test.assert( uploadElement.attributes['class'].indexOf( newStateClass ) !== -1, + this.test.comment( 'new HDA should have the new state class' ); + this.assertHasClass( uploadSelector, this.historypanel.data.selectors.hda.wrapper.stateClasses['new'], 'HDA has new state class' ); - // since we're using css there's no great way to test this - //var stateIconSelector = uploadSelector + ' .state-icon'; - //this.test.assertVisible( stateIconSelector, 'HDA has proper hid' ); - - // should NOT have any of the three, main buttons + this.test.comment( 'new HDA should NOT have any of the three, main buttons' ); var buttonSelector = uploadSelector + ' ' + this.historypanel.data.selectors.hda.titleButtons + ' a'; this.test.assertDoesntExist( buttonSelector, 'No display, edit, or delete buttons' ); - // expand and check the body - this.click( titleSelector ); + this.test.comment( 'clicking the title of the new HDA will expand the body' ); + var hdaTitle = uploadSelector + ' ' + this.historypanel.data.selectors.hda.title; + this.click( hdaTitle ); this.wait( 500, function(){ var bodySelector = uploadSelector + ' ' + this.historypanel.data.selectors.hda.body; this.test.assertVisible( bodySelector, 'HDA body is visible (after expanding)' ); var expectedBodyText = 'This is a new dataset'; + this.test.comment( 'the body should have the text: ' + expectedBodyText ); this.test.assertSelectorHasText( bodySelector, expectedBodyText, 'HDA body has text: ' + expectedBodyText ); // restore to collapsed - this.click( titleSelector ); + this.click( hdaTitle ); }); }); }); }); -*/ +// restore state, collapse +spaceghost.then( function revertStateAndCollapse(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ + this.evaluate( function(){ + return Galaxy.currHistoryPanel.model.hdas.at( 0 ).set( 'state', 'ok' ); + }); + this.wait( 500, function(){ + var hdaTitle = '#' + testUploadInfo.hdaElement.attributes.id + + ' ' + this.historypanel.data.selectors.hda.title; + this.click( hdaTitle ); + }); + }); +}); // =================================================================== diff -r c2f310489973fc2ba42e5289ccec3ed88a42aabc -r f1bd817063e092711a9b13f60e569abbce877fba test/casperjs/history-panel-tests.js --- a/test/casperjs/history-panel-tests.js +++ b/test/casperjs/history-panel-tests.js @@ -38,7 +38,11 @@ } // selectors and labels -var nameSelector = spaceghost.historypanel.data.selectors.history.name, +var tooltipSelector = spaceghost.data.selectors.tooltipBalloon, + editableTextClass = spaceghost.data.selectors.editableText, + editableTextInput = spaceghost.data.selectors.editableTextInput, + + nameSelector = spaceghost.historypanel.data.selectors.history.name, subtitleSelector = spaceghost.historypanel.data.selectors.history.subtitle, unnamedName = spaceghost.historypanel.data.text.history.newName, initialSizeStr = spaceghost.historypanel.data.text.history.newSize, @@ -51,14 +55,9 @@ annoAreaSelector = spaceghost.historypanel.data.selectors.history.annoArea, nameTooltip = spaceghost.historypanel.data.text.history.tooltips.name, - tooltipSelector = '.bs-tooltip', - - editableTextClass = 'editable-text', - editableTextInputSelector = 'input#renaming-active', - - refreshButtonSelector = 'a#history-refresh-button', - refreshButtonIconSelector = 'span.fa-icon-refresh', - refreshButtonHref = '/history', + refreshButtonSelector = 'a#history-refresh-button', + refreshButtonIconSelector = 'span.fa-icon-refresh', + refreshButtonHref = '/history', includeDeletedOptionsLabel = spaceghost.historyoptions.data.labels.options.includeDeleted; @@ -66,7 +65,7 @@ var newHistoryName = "Test History", filepathToUpload = '../../test-data/1.txt', historyFrameInfo = {}, - uploadInfo = {}; + testUploadInfo = {}; // =================================================================== TESTS @@ -88,7 +87,7 @@ // ------------------------------------------------------------------- check structure of empty history spaceghost.thenOpen( spaceghost.baseUrl, function testPanelStructure(){ this.test.comment( 'history panel, new history' ); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.test.comment( "frame should have proper url and title: 'History'" ); this.test.assertMatch( this.getCurrentUrl(), /\/history/, 'Found history frame url' ); this.test.assertTitle( this.getTitle(), 'History', 'Found history frame title' ); @@ -98,7 +97,7 @@ this.test.assertVisible( nameSelector, 'History name is visible' ); this.test.assertSelectorHasText( nameSelector, unnamedName, 'History name is ' + unnamedName ); - this.test.comment( "history subtitle should display size and size should be 0 bytes" ); + this.test.comment( "history subtitle should display size and size should be: " + initialSizeStr ); this.test.assertExists( subtitleSelector, 'Found ' + subtitleSelector ); this.test.assertVisible( subtitleSelector, 'History subtitle is visible' ); this.test.assertSelectorHasText( subtitleSelector, initialSizeStr, @@ -119,50 +118,46 @@ // ------------------------------------------------------------------- name editing spaceghost.then( function(){ this.test.comment( 'history panel, editing the history name' ); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.test.comment( 'name should have a tooltip with proper info on name editing' ); var nameInfo = this.getElementInfo( nameSelector ); - this.page.sendEvent( 'mousemove', - historyFrameInfo.x + nameInfo.x + 1, historyFrameInfo.y + nameInfo.y + 1 ); + this.page.sendEvent( 'mousemove', historyFrameInfo.x + nameInfo.x + 1, historyFrameInfo.y + nameInfo.y + 1 ); this.test.assertExists( tooltipSelector, "Found tooltip after name hover" ); this.test.assertSelectorHasText( tooltipSelector, nameTooltip ); this.test.comment( 'name should be create an input when clicked' ); - this.test.assert( nameInfo.attributes[ 'class' ].indexOf( editableTextClass ) !== -1, - "Name field classed for editable text" ); + this.assertHasClass( nameSelector, editableTextClass, "Name field classed for editable text" ); this.click( nameSelector ); - this.test.assertExists( editableTextInputSelector, "Clicking on name creates an input" ); + this.test.assertExists( editableTextInput, "Clicking on name creates an input" ); this.test.comment( 'name should be editable by entering keys and pressing enter' ); //NOTE: casperjs.sendKeys adds a click before and a selector.blur after sending - won't work here - //TODO: to conv. fn this.page.sendEvent( 'keypress', newHistoryName ); this.page.sendEvent( 'keypress', this.page.event.key.Enter ); this.wait( 1000, function(){ this.test.assertSelectorHasText( nameSelector, newHistoryName, 'History name is ' + newHistoryName ); - this.test.assertDoesntExist( editableTextInputSelector, "Input disappears after pressing enter" ); + this.test.assertDoesntExist( editableTextInput, "Input disappears after pressing enter" ); }); }); }); spaceghost.then( function(){ - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.test.comment( 'name should revert if user clicks away while editing' ); this.click( nameSelector ); this.page.sendEvent( 'keypress', "Woodchipper metagenomics, Fargo, ND" ); // click above the name input element - var inputInfo = this.getElementInfo( editableTextInputSelector ); - this.page.sendEvent( 'mousedown', - historyFrameInfo.x + inputInfo.x + 1, historyFrameInfo.y + inputInfo.y - 5 ); + var inputInfo = this.getElementInfo( editableTextInput ); + this.page.sendEvent( 'mousedown', historyFrameInfo.x + inputInfo.x + 1, historyFrameInfo.y + inputInfo.y - 5 ); this.wait( 1000, function(){ this.test.assertSelectorHasText( nameSelector, newHistoryName, 'History name is STILL ' + newHistoryName ); - this.test.assertDoesntExist( editableTextInputSelector, "Input disappears after clicking away" ); + this.test.assertDoesntExist( editableTextInput, "Input disappears after clicking away" ); }); }); }); spaceghost.then( function(){ - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.test.comment( 'name should revert if user hits ESC while editing' ); this.click( nameSelector ); this.page.sendEvent( 'keypress', "Arsenic Bacteria" ); @@ -170,7 +165,7 @@ this.page.sendEvent( 'keypress', this.page.event.key.Escape ); this.wait( 1000, function(){ this.test.assertSelectorHasText( nameSelector, newHistoryName, 'History name is STILL ' + newHistoryName ); - this.test.assertDoesntExist( editableTextInputSelector, "Input disappears after hitting ESC" ); + this.test.assertDoesntExist( editableTextInput, "Input disappears after hitting ESC" ); }); }); }); @@ -185,14 +180,14 @@ hasClass = _uploadInfo.hdaElement.attributes[ 'class' ], hasOkClass = _uploadInfo.hdaElement.attributes[ 'class' ].indexOf( wrapperOkClassName ) !== -1; this.test.assert( ( hasHda && hasClass && hasOkClass ), "Uploaded file: " + _uploadInfo.name ); - uploadInfo = _uploadInfo; + testUploadInfo = _uploadInfo; }); }); spaceghost.then( function checkPanelStructure(){ this.test.comment( 'checking structure of non-empty panel' ); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.test.comment( "history name should exist, be visible, and have text " + unnamedName ); this.test.assertExists( nameSelector, nameSelector + ' exists' ); this.test.assertVisible( nameSelector, 'History name is visible' ); @@ -221,8 +216,7 @@ //TODO: check tooltips spaceghost.then( function openTags(){ this.test.comment( 'tag area should open when the history panel tag icon is clicked' ); - this.withFrame( this.selectors.frames.history, function(){ - this.capture( 'tag-area.png' ); + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.mouseEvent( 'click', tagIconSelector ); this.wait( 1000, function(){ this.test.assertVisible( tagAreaSelector, 'Tag area is now displayed' ); @@ -235,7 +229,7 @@ //TODO: check tooltips spaceghost.then( function openAnnotation(){ this.test.comment( 'annotation area should open when the history panel annotation icon is clicked' ); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.mouseEvent( 'click', annoIconSelector ); this.wait( 1000, function(){ this.test.assertVisible( annoAreaSelector, 'Annotation area is now displayed' ); @@ -244,7 +238,7 @@ }); spaceghost.then( function closeAnnotation(){ this.test.comment( 'annotation area should close when the history panel tag icon is clicked again' ); - this.withFrame( this.selectors.frames.history, function bler(){ + this.withFrame( spaceghost.data.selectors.frames.history, function bler(){ this.mouseEvent( 'click', annoIconSelector ); this.wait( 1000, function(){ this.test.assertNotVisible( annoAreaSelector, 'Tag area is now hidden' ); @@ -296,8 +290,9 @@ spaceghost.then( function(){ this.test.comment( 'deleted hdas shouldn\'t be in the history panel DOM' ); - this.historypanel.deleteHda( '#' + uploadInfo.hdaElement.attributes.id, function(){ - this.test.assertDoesntExist( '#' + uploadInfo.hdaElement.attributes.id, "Deleted HDA is not in the DOM" ); + this.historypanel.deleteHda( '#' + testUploadInfo.hdaElement.attributes.id, function(){ + this.test.assertDoesntExist( '#' + testUploadInfo.hdaElement.attributes.id, + "Deleted HDA is not in the DOM" ); }); }); @@ -306,11 +301,11 @@ this.test.comment( 'History options->' + includeDeletedOptionsLabel + ' shows deleted datasets' ); this.historyoptions.includeDeleted(); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.waitForSelector( nameSelector, function(){ - this.test.assertExists( '#' + uploadInfo.hdaElement.attributes.id, + this.test.assertExists( '#' + testUploadInfo.hdaElement.attributes.id, "Deleted HDA is in the DOM (using history options -> " + includeDeletedOptionsLabel + ")" ); - this.test.assertVisible( '#' + uploadInfo.hdaElement.attributes.id, + this.test.assertVisible( '#' + testUploadInfo.hdaElement.attributes.id, "Deleted HDA is visible again (using history options -> " + includeDeletedOptionsLabel + ")" ); }); }); @@ -320,9 +315,9 @@ this.test.comment( 'History options->' + includeDeletedOptionsLabel + ' (again) re-hides deleted datasets' ); this.historyoptions.includeDeleted(); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.waitForSelector( nameSelector, function(){ - this.test.assertDoesntExist( '#' + uploadInfo.hdaElement.attributes.id, + this.test.assertDoesntExist( '#' + testUploadInfo.hdaElement.attributes.id, "Deleted HDA is not in the DOM (using history options -> " + includeDeletedOptionsLabel + ")" ); }); }); @@ -331,10 +326,10 @@ // undelete the uploaded file spaceghost.then( function(){ this.historyoptions.includeDeleted(); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.waitForSelector( nameSelector, function(){ //TODO: to conv. fn - this.click( '#' + uploadInfo.hdaElement.attributes.id + this.click( '#' + testUploadInfo.hdaElement.attributes.id + ' ' + this.historypanel.data.selectors.history.undeleteLink ); }); }); @@ -348,16 +343,13 @@ // broken in webkit w/ jq 1.7 spaceghost.then( function(){ this.test.comment( 'HDAs can be expanded by clicking on the name' ); - var uploadedSelector = '#' + uploadInfo.hdaElement.attributes.id; + var uploadedSelector = '#' + testUploadInfo.hdaElement.attributes.id; - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.click( uploadedSelector + ' .historyItemTitle' ); - this.debug( 'title: ' + this.debugElement( uploadedSelector + ' .historyItemTitle' ) ); - this.debug( 'wrapper: ' + this.debugElement( uploadedSelector ) ); - this.wait( 1000, function(){ - this.test.assertExists( uploadedSelector + ' .historyItemBody', "Body for uploaded file is found" ); - this.test.assertVisible( uploadedSelector + ' .hda-summary', "hda-summary is visible" ); + this.test.assertVisible( uploadedSelector + ' ' + this.historypanel.data.selectors.hda.body, + "Body for uploaded file is visible" ); }); }); }); @@ -365,13 +357,13 @@ // ------------------------------------------------------------------- expanded hdas are still expanded after a refresh spaceghost.then( function(){ this.test.comment( 'Expanded hdas are still expanded after a refresh' ); - var uploadedSelector = '#' + uploadInfo.hdaElement.attributes.id; + var uploadedSelector = '#' + testUploadInfo.hdaElement.attributes.id; this.click( refreshButtonSelector ); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.waitForSelector( nameSelector, function(){ - this.test.assertExists( uploadedSelector + ' .historyItemBody', "Body for uploaded file is found" ); - this.test.assertVisible( uploadedSelector + ' .hda-summary', "hda-summary is visible" ); + this.test.assertVisible( uploadedSelector + ' ' + this.historypanel.data.selectors.hda.body, + "Body for uploaded file is visible" ); }); }); // this will break: webkit + jq 1.7 @@ -380,26 +372,27 @@ // ------------------------------------------------------------------- expanded hdas collapse by clicking name again spaceghost.then( function(){ this.test.comment( 'Expanded hdas collapse by clicking name again' ); - var uploadedSelector = '#' + uploadInfo.hdaElement.attributes.id; + var uploadedSelector = '#' + testUploadInfo.hdaElement.attributes.id; - this.withFrame( this.selectors.frames.history, function(){ - this.click( uploadedSelector + ' .historyItemTitle' ); - + this.withFrame( spaceghost.data.selectors.frames.history, function(){ + this.click( uploadedSelector + ' ' + this.historypanel.data.selectors.hda.title ); this.wait( 500, function(){ - this.test.assertNotVisible( uploadedSelector + ' .hda-summary', "hda-summary is not visible" ); + this.test.assertNotVisible( uploadedSelector + ' ' + this.historypanel.data.selectors.hda.body, + "Body for uploaded file is not visible" ); }); }); }); -// ------------------------------------------------------------------- collapsed hdas are still collapsed after a refresh +// ------------------------------------------------------------------- collapsed hdas still collapsed after a refresh spaceghost.then( function(){ - this.test.comment( 'Expanded hdas are still expanded after a refresh' ); - var uploadedSelector = '#' + uploadInfo.hdaElement.attributes.id; + this.test.comment( 'collapsed hdas still collapsed after a refresh' ); + var uploadedSelector = '#' + testUploadInfo.hdaElement.attributes.id; this.click( refreshButtonSelector ); - this.withFrame( this.selectors.frames.history, function(){ + this.withFrame( spaceghost.data.selectors.frames.history, function(){ this.waitForSelector( nameSelector, function(){ - this.test.assertNotVisible( uploadedSelector + ' .hda-summary', "hda-summary is not visible" ); + this.test.assertNotVisible( uploadedSelector + ' ' + this.historypanel.data.selectors.hda.body, + "Body for uploaded file is not visible" ); }); }); }); @@ -407,19 +400,20 @@ // ------------------------------------------------------------------- history options collapses all expanded hdas spaceghost.then( function(){ // expand again - this.withFrame( this.selectors.frames.history, function(){ - this.click( '#' + uploadInfo.hdaElement.attributes.id + ' .historyItemTitle' ); + this.withFrame( spaceghost.data.selectors.frames.history, function(){ + this.click( '#' + testUploadInfo.hdaElement.attributes.id + ' ' + this.historypanel.data.selectors.hda.title ); this.wait( 500, function(){}); }); }); spaceghost.then( function(){ this.test.comment( 'History option collapses all expanded hdas' ); - var uploadedSelector = '#' + uploadInfo.hdaElement.attributes.id; + var uploadedSelector = '#' + testUploadInfo.hdaElement.attributes.id; this.historyoptions.collapseExpanded(); this.wait( 500, function(){ - this.withFrame( this.selectors.frames.history, function(){ - this.test.assertNotVisible( uploadedSelector + ' .hda-summary', "hda-summary is not visible" ); + this.withFrame( spaceghost.data.selectors.frames.history, function(){ + this.test.assertNotVisible( uploadedSelector + ' ' + this.historypanel.data.selectors.hda.body, + "Body for uploaded file is not visible" ); }); }); }); diff -r c2f310489973fc2ba42e5289ccec3ed88a42aabc -r f1bd817063e092711a9b13f60e569abbce877fba test/casperjs/modules/historypanel.js --- a/test/casperjs/modules/historypanel.js +++ b/test/casperjs/modules/historypanel.js @@ -73,7 +73,7 @@ finalStateClass = '.historyItem-' + finalState; spaceghost.then( function(){ - spaceghost.withFrame( spaceghost.selectors.frames.history, function(){ + spaceghost.withFrame( spaceghost.data.selectors.frames.history, function(){ // save the old time out var oldWaitTimeout = spaceghost.options.waitTimeout, @@ -157,7 +157,7 @@ whenDeletedFn = whenDeletedFn || function(){}; var spaceghost = this.spaceghost; - spaceghost.withFrame( spaceghost.selectors.frames.history, function deletingHda(){ + spaceghost.withFrame( spaceghost.data.selectors.frames.history, function deletingHda(){ //precondition: historyItemWrapper's (hda dom elements) should have an id // we could use the selector directly, but better if it errors before an attempted delete var hdaId = spaceghost.getElementInfo( hdaSelector ).attributes.id; @@ -192,7 +192,7 @@ var spaceghost = this.spaceghost, historyFrameInfo = spaceghost.getElementInfo( 'iframe[name="galaxy_history"]' ); - spaceghost.withFrame( spaceghost.selectors.frames.history, function expandingHda(){ + spaceghost.withFrame( spaceghost.data.selectors.frames.history, function expandingHda(){ var titleInfo = spaceghost.getElementInfo( hdaSelector + ' .historyItemTitle' ); spaceghost.page.sendEvent( 'mousedown', historyFrameInfo.x + titleInfo.x + 1, historyFrameInfo.y + titleInfo.y - 5 ); @@ -233,7 +233,7 @@ if( !historyFrameInfo ){ //TODO: move selector to data (use selectors.frames? ) historyFrameInfo = spaceghost.getElementInfo( 'iframe[name="galaxy_history"]' ); - spaceghost.withFrame( spaceghost.selectors.frames.history, function inHistoryPanel(){ + spaceghost.withFrame( spaceghost.data.selectors.frames.history, function inHistoryPanel(){ hoverAndCallback.call( spaceghost, historyFrameInfo, selector, whenHovering ); }); diff -r c2f310489973fc2ba42e5289ccec3ed88a42aabc -r f1bd817063e092711a9b13f60e569abbce877fba test/casperjs/modules/tools.js --- a/test/casperjs/modules/tools.js +++ b/test/casperjs/modules/tools.js @@ -22,7 +22,11 @@ return this.spaceghost + '.Tools'; }; +// ------------------------------------------------------------------- +/* TODO: + move selectors from sg to here +*/ // =================================================================== INTERNAL /** Tests uploading a file. * NOTE: this version does NOT throw an error on a bad upload. @@ -35,52 +39,34 @@ uploadInfo = {}; //TODO: check file exists using phantom.fs //TODO: pull from test data - uploadInfo[ spaceghost.selectors.tools.upload.fileInput ] = filepath; + uploadInfo[ spaceghost.data.selectors.tools.upload.fileInput ] = filepath; spaceghost.debug( 'uploading file: ' + filepath ); spaceghost.then( function(){ - spaceghost.withFrame( spaceghost.selectors.frames.tools, function(){ - spaceghost.clickLabel( spaceghost.labels.tools.upload.panelLabel ); + spaceghost.withFrame( spaceghost.data.selectors.frames.tools, function(){ + spaceghost.clickLabel( spaceghost.data.labels.tools.upload.panelLabel ); }); }); spaceghost.then( function beginUpload(){ - spaceghost.withFrame( spaceghost.selectors.frames.main, function(){ - spaceghost.fill( spaceghost.selectors.tools.general.form, uploadInfo, false ); + spaceghost.withFrame( spaceghost.data.selectors.frames.main, function(){ + spaceghost.fill( spaceghost.data.selectors.tools.general.form, uploadInfo, false ); // the following throws: // [error] [remote] Failed dispatching clickmouse event on xpath selector: //input[@value="Execute"]: // PageError: TypeError: 'undefined' is not a function (evaluating '$(spaceghost).formSerialize()') // ...and yet the upload still seems to work - spaceghost.click( xpath( spaceghost.selectors.tools.general.executeButton_xpath ) ); + spaceghost.click( xpath( spaceghost.data.selectors.tools.general.executeButton_xpath ) ); }); }); // debugging - spaceghost.withFrame( spaceghost.selectors.frames.main, function afterUpload(){ - var messageInfo = spaceghost.elementInfoOrNull( spaceghost.selectors.messages.all ); + spaceghost.withFrame( spaceghost.data.selectors.frames.main, function afterUpload(){ + var messageInfo = spaceghost.elementInfoOrNull( spaceghost.data.selectors.messages.all ); spaceghost.debug( 'post upload message:\n' + spaceghost.jsonStr( messageInfo ) ); }); }; -/** Uploads a file. - * @param {String} filepath the local filesystem path of the file to upload (absolute (?)) - */ -//Tools.prototype.uploadFile = function uploadFile( filepath ){ -// this._uploadFile( filepath ); -// this.then( function(){ -// this.withFrame( this.selectors.frames.main, function mainAfterUpload(){ -// var messageInfo = this.elementInfoOrNull( this.selectors.messages.all ); -// if( ( !messageInfo ) -// || ( messageInfo.attributes[ 'class' ] !== 'donemessagelarge' ) -// || ( messageInfo.text.indexOf( this.text.upload.success ) === -1 ) ){ -// throw new GalaxyError( 'UploadError: ' + this.jsonStr( messageInfo ) ); -// } -// }); -// }); -// return this; -//}; - /** Parses the hid and name of a newly uploaded file from the tool execution donemessagelarge * @param {String} doneMsgText the text extracted from the donemessagelarge after a tool execution */ @@ -159,9 +145,9 @@ // upload the file erroring if a done message is not displayed, aggregate info about upload spaceghost.info( 'uploading file: ' + filepath + ' (timeout after ' + timeoutAfterMs + ')' ); this._uploadFile( filepath ); - spaceghost.withFrame( spaceghost.selectors.frames.main, function toolExecuted(){ + spaceghost.withFrame( spaceghost.data.selectors.frames.main, function toolExecuted(){ spaceghost.debug( 'checking for done message' ); - var doneElementInfo = spaceghost.elementInfoOrNull( spaceghost.selectors.messages.donelarge ); + var doneElementInfo = spaceghost.elementInfoOrNull( spaceghost.data.selectors.messages.donelarge ); if( !doneElementInfo ){ throw new spaceghost.GalaxyError( 'Upload Error: no done message uploading "' + filepath + '"' ); } @@ -178,7 +164,7 @@ spaceghost.then( function getNewHda(){ spaceghost.debug( 'beginning wait for upload file\'s ok state' ); // get the hda view DOM element from the upload name and hid - spaceghost.withFrame( spaceghost.selectors.frames.history, function(){ + spaceghost.withFrame( spaceghost.data.selectors.frames.history, function(){ spaceghost.waitForSelector( '#history-name', function(){ var hdaInfo = spaceghost.historypanel.hdaElementInfoByTitle( uploadInfo.name, uploadInfo.hid ); if( hdaInfo === null ){ diff -r c2f310489973fc2ba42e5289ccec3ed88a42aabc -r f1bd817063e092711a9b13f60e569abbce877fba test/casperjs/modules/user.js --- a/test/casperjs/modules/user.js +++ b/test/casperjs/modules/user.js @@ -43,19 +43,19 @@ spaceghost.debug( 'registering user:\n' + spaceghost.jsonStr( userInfo ) ); spaceghost.thenOpen( spaceghost.baseUrl, function(){ - spaceghost.clickLabel( spaceghost.labels.masthead.menus.user ); - spaceghost.clickLabel( spaceghost.labels.masthead.userMenu.register ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.register ); - spaceghost.withFrame( spaceghost.selectors.frames.main, function mainBeforeRegister(){ + spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainBeforeRegister(){ spaceghost.debug( 'submitting registration... ' + spaceghost.getCurrentUrl() ); - spaceghost.fill( spaceghost.selectors.registrationPage.form, userInfo, false ); + spaceghost.fill( spaceghost.data.selectors.registrationPage.form, userInfo, false ); // need manual submit (not a normal html form) - spaceghost.click( xpath( spaceghost.selectors.registrationPage.submit_xpath ) ); + spaceghost.click( xpath( spaceghost.data.selectors.registrationPage.submit_xpath ) ); }); //// debugging - //spaceghost.withFrame( spaceghost.selectors.frames.main, function mainAfterRegister(){ - // var messageInfo = spaceghost.getElementInfo( spaceghost.selectors.messages.all ); + //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterRegister(){ + // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); // spaceghost.debug( 'post registration message:\n' + spaceghost.jsonStr( messageInfo ) ); //}); }); @@ -78,21 +78,21 @@ spaceghost.thenOpen( spaceghost.baseUrl, function(){ - spaceghost.clickLabel( spaceghost.labels.masthead.menus.user ); - spaceghost.clickLabel( spaceghost.labels.masthead.userMenu.login ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.login ); - spaceghost.withFrame( spaceghost.selectors.frames.main, function mainBeforeLogin(){ + spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainBeforeLogin(){ spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') logging in user:\n' + spaceghost.jsonStr( loginInfo ) ); - spaceghost.fill( spaceghost.selectors.loginPage.form, loginInfo, false ); - spaceghost.click( xpath( spaceghost.selectors.loginPage.submit_xpath ) ); + spaceghost.fill( spaceghost.data.selectors.loginPage.form, loginInfo, false ); + spaceghost.click( xpath( spaceghost.data.selectors.loginPage.submit_xpath ) ); }); //// debugging - //spaceghost.withFrame( spaceghost.selectors.frames.main, function mainAfterLogin(){ + //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterLogin(){ // //TODO: prob. could use a more generalized form of this for url breakdown/checking - // if( spaceghost.getCurrentUrl().search( spaceghost.selectors.loginPage.url_regex ) != -1 ){ - // var messageInfo = spaceghost.getElementInfo( spaceghost.selectors.messages.all ); + // if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) != -1 ){ + // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); // spaceghost.debug( 'post login message:\n' + spaceghost.jsonStr( messageInfo ) ); // } //}); @@ -111,8 +111,8 @@ var spaceghost = this.spaceghost; this._submitRegistration( email, password, username ); spaceghost.then( function(){ - spaceghost.withFrame( spaceghost.selectors.frames.main, function mainAfterRegister(){ - var messageInfo = spaceghost.getElementInfo( spaceghost.selectors.messages.all ); + spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterRegister(){ + var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); spaceghost.debug( 'post registration message:\n' + this.jsonStr( messageInfo ) ); if( messageInfo.attributes[ 'class' ] === 'errormessage' ){ @@ -132,9 +132,9 @@ var spaceghost = this.spaceghost; this._submitLogin( email, password ); spaceghost.then( function(){ - spaceghost.withFrame( spaceghost.selectors.frames.main, function mainAfterLogin(){ - if( spaceghost.getCurrentUrl().search( spaceghost.selectors.loginPage.url_regex ) != -1 ){ - var messageInfo = spaceghost.getElementInfo( spaceghost.selectors.messages.all ); + spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterLogin(){ + if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) != -1 ){ + var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); if( messageInfo && messageInfo.attributes[ 'class' ] === 'errormessage' ){ throw new spaceghost.GalaxyError( 'LoginError: ' + messageInfo.html ); } @@ -154,7 +154,8 @@ var spaceghost = this.spaceghost, userEmail = ''; try { - var loggedInInfo = spaceghost.getElementInfo( xpath( spaceghost.selectors.masthead.userMenu.userEmail_xpath ) ); + var loggedInInfo = spaceghost.getElementInfo( + xpath( spaceghost.data.selectors.masthead.userMenu.userEmail_xpath ) ); userEmail = loggedInInfo.text; } catch( err ){ spaceghost.error( err ); @@ -170,8 +171,8 @@ var spaceghost = this.spaceghost; spaceghost.thenOpen( spaceghost.baseUrl, function(){ //TODO: handle already logged out - spaceghost.clickLabel( spaceghost.labels.masthead.menus.user ); - spaceghost.clickLabel( spaceghost.labels.masthead.userMenu.logout ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.logout ); }); return spaceghost; }; diff -r c2f310489973fc2ba42e5289ccec3ed88a42aabc -r f1bd817063e092711a9b13f60e569abbce877fba test/casperjs/spaceghost.js --- a/test/casperjs/spaceghost.js +++ b/test/casperjs/spaceghost.js @@ -64,11 +64,13 @@ var utils = require( 'utils' ); // ------------------------------------------------------------------- inheritance -/** +/** @class An extension of the Casper object with methods and overrides specifically + * for interacting with a Galaxy web page. + * @augments Casper */ function SpaceGhost(){ SpaceGhost.super_.apply( this, arguments ); - this.init.apply( this, arguments ); + this._init.apply( this, arguments ); } utils.inherits( SpaceGhost, Casper ); @@ -85,6 +87,9 @@ // ------------------------------------------------------------------- error types PageError.prototype = new CasperError(); PageError.prototype.constructor = CasperError; +/** @class Represents a javascript error on the page casper is browsing + * (as opposed to an error in the test script). + */ function PageError(){ CasperError.apply( this, arguments ); this.name = "PageError"; @@ -93,6 +98,7 @@ GalaxyError.prototype = new CasperError(); GalaxyError.prototype.constructor = CasperError; +/** @class Thrown when Galaxy has (gracefully?) indicated pilot error. */ function GalaxyError(){ CasperError.apply( this, arguments ); this.name = "GalaxyError"; @@ -101,6 +107,7 @@ AlertError.prototype = new CasperError(); AlertError.prototype.constructor = CasperError; +/** @class Thrown when Galaxy has displayed a javascript alert. */ function AlertError(){ CasperError.apply( this, arguments ); this.name = "AlertError"; @@ -111,14 +118,17 @@ // ------------------------------------------------------------------- set up /** More initialization: cli, event handlers, etc. * @param {Object} options option hash + * @private */ -SpaceGhost.prototype.init = function init( options ){ +SpaceGhost.prototype._init = function _init( options ){ //console.debug( 'init, options:', JSON.stringify( options, null, 2 ) ); //NOTE: cli will override in-script options this._setOptionsFromCli(); // save errors for later output (needs to go before process CLI) + /** cache of errors that have occurred + * @memberOf SpaceGhost */ this.errors = []; this.on( 'error', function pushErrorToStack( msg, backtrace ){ //this.debug( 'adding error to stack: ' + msg + ', trace:' + JSON.stringify( backtrace, null, 2 ) ); @@ -139,8 +149,9 @@ }; /** Allow CLI arguments to set options if the proper option name is used. - * @example: + * @example * casperjs myscript.js --verbose=true --logLevel=debug + * @private */ SpaceGhost.prototype._setOptionsFromCli = function setOptionsFromCli(){ // get and remove any casper options passed on the command line @@ -154,25 +165,8 @@ }; // ------------------------------------------------------------------- cli args and options -SpaceGhost.prototype._saveHtmlOnErrorHandler = function _saveHtmlOnErrorHandler( msg, backtrace ){ - // needs to output to a file in GALAXY_SAVE - //this.debugHTML(); -}; - -SpaceGhost.prototype._saveTextOnErrorHandler = function _saveTextOnErrorHandler( msg, backtrace ){ - // needs to output to a file in GALAXY_SAVE - //this.debugPage(); -}; - -SpaceGhost.prototype._saveScreenOnErrorHandler = function _saveScreenOnErrorHandler( msg, backtrace ){ - // needs to output to a pic in GALAXY_SAVE - //var filename = ...?? - //?? this.getCurrentUrl(), this.getCurrent - //this.capture( filename ); -}; - - /** Set up any SG specific options passed in on the cli. + * @private */ SpaceGhost.prototype._processCLIArguments = function _processCLIArguments(){ //this.debug( 'cli: ' + this.jsonStr( this.cli ) ); @@ -262,6 +256,7 @@ }; /** Suppress the normal output from the casper object (echo, errors) + * @private */ SpaceGhost.prototype._suppressOutput = function _suppressOutput(){ // currently (1.0) the only way to suppress test pass/fail messages @@ -274,6 +269,7 @@ }; /** Suppress the normal output from the casper object (echo, errors) + * @private */ SpaceGhost.prototype._redirectOutputToStderr = function _redirectOutputToStderr(){ // currently (1.0) the only way to suppress test pass/fail messages @@ -288,7 +284,7 @@ this.removeListener( 'error', this.listeners( 'error' )[1] ); }; -/** Outputs logs, test results and errors in a single JSON formatted object. +/** Outputs logs, test results and errors in a single JSON formatted object to the console. */ SpaceGhost.prototype.outputStateAsJson = function outputStateAsJson(){ var returnedJSON = { @@ -303,18 +299,20 @@ // ------------------------------------------------------------------- event handling //note: using non-anon fns to allow removal if needed -// most of these are stubs (w logging) for later expansion -/** Event handler for failed page loads +/** 'load failed' Event handler for failed page loads that only records to the log + * @private */ SpaceGhost.prototype._loadFailedHandler = function _loadFailedHandler( object ){ this.error( 'load.failed: ' + spaceghost.jsonStr( object ) ); //TODO: throw error? }; -/** Event handler for page errors (js) - throws test scope as PageError +/** 'page.error' Event handler that re-raises as PageError * NOTE: this has some special handling for DOM exc 12 which some casper selectors are throwing * (even tho the selector still works) + * @throws {PageError} (with original's msg and backtrace) + * @private */ SpaceGhost.prototype._pageErrorHandler = function _pageErrorHandler( msg, backtrace ){ // add a page error handler to catch page errors (what we're most interested with here) @@ -337,7 +335,9 @@ } }; -/** Event handler for step/casper timeouts - throws PageError +/** 'timeout' Event handler for step/casper timeouts - raises as PageError + * @throws {PageError} Timeout occurred + * @private */ SpaceGhost.prototype._timeoutHandler = function _timeoutHandler(){ console.debug( 'timeout' ); @@ -345,17 +345,13 @@ throw new PageError( 'Timeout occurred' ); }; -/** Event handler for console messages from the page. - */ -SpaceGhost.prototype._pageConsoleHandler = function _pageConsoleHandler(){ - // remote.message - var DELIM = '-'; - this.debug( this + '(page console) "' + Array.prototype.join.call( arguments, DELIM ) + '"' ); -}; - -/** Event handler for alerts +/** 'alert' Event handler that raises an AlertError with the alert message + * @throws {AlertError} (the alert message) + * @private */ SpaceGhost.prototype._alertHandler = function _alertHandler( message ){ + //TODO: this still isn't working well... + // casper info level already has outputs these //this.warning( this + '(page alert)\n"' + message + '"' ); var ALERT_MARKER = '(page alert) '; @@ -374,13 +370,34 @@ } }; -/** Event handler for navigation requested (loading of frames, redirects(?)) +/** 'error' Event handler that saves html from the errored page. + * @private */ -SpaceGhost.prototype._navHandler = function _navHandler( url, navigationType, navigationLocked, isMainFrame ){ - this.debug( 'navigation.requested: ' + url ); +SpaceGhost.prototype._saveHtmlOnErrorHandler = function _saveHtmlOnErrorHandler( msg, backtrace ){ + // needs to output to a file in GALAXY_SAVE + //this.debugHTML(); }; -/** Set up event handlers. +/** 'error' Event handler that saves text from the errored page. + * @private + */ +SpaceGhost.prototype._saveTextOnErrorHandler = function _saveTextOnErrorHandler( msg, backtrace ){ + // needs to output to a file in GALAXY_SAVE + //this.debugPage(); +}; + +/** 'error' Event handler that saves a screenshot of the errored page. + * @private + */ +SpaceGhost.prototype._saveScreenOnErrorHandler = function _saveScreenOnErrorHandler( msg, backtrace ){ + // needs to output to a pic in GALAXY_SAVE + //var filename = ...?? + //?? this.getCurrentUrl(), this.getCurrent + //this.capture( filename ); +}; + +/** Sets up event handlers. + * @private */ SpaceGhost.prototype._setUpEventHandlers = function _setUpEventHandlers(){ //console.debug( '_setUpEventHandlers' ); @@ -393,18 +410,17 @@ this.on( 'waitFor.timeout', this._timeoutHandler ); // ........................ page info/debugging - // these are already displayed at the casper info level - - //this.on( 'remote.message', this._pageConsoleHandler ); this.on( 'remote.alert', this._alertHandler ); - // these are already displayed at the casper debug level - //this.on( 'navigation.requested', this._navHandler ); - }; // ------------------------------------------------------------------- sub modules /** Load sub modules (similar to casperjs.test) + * @requires User modules/user.js + * @requires Tools modules/tools.js + * @requires HistoryPanel modules/historypanel.js + * @requires HistoryOptions modules/historyoptions.js + * @private */ SpaceGhost.prototype._loadModules = function _loadModules(){ this.user = require( this.options.scriptDir + 'modules/user' ).create( this ); @@ -416,6 +432,7 @@ // =================================================================== PAGE CONTROL /** An override of casper.start for additional set up. * (Currently only used to change viewport) + * @see Casper#start */ SpaceGhost.prototype.start = function start(){ var returned = Casper.prototype.start.apply( this, arguments ); @@ -423,8 +440,9 @@ return returned; }; -/** An override of casper.open specifically for Galaxy. +/** An override of casper.open for additional page control. * (Currently only used to change language headers) + * @see Casper#open */ SpaceGhost.prototype.open = function open(){ //TODO: this can be moved to start (I think...?) @@ -433,13 +451,13 @@ return Casper.prototype.open.apply( this, arguments ); }; -/** An override to provide json output and more informative error codes +/** An override to provide json output and more informative error codes. + * Exits with 2 if a test has failed. + * Exits with 1 if some error has occurred. + * Exits with 0 if all tests passed. + * @see Casper#run run, boy, run (doesn't he fly?) */ SpaceGhost.prototype.run = function run( onComplete, time ){ - // wrap the onComplete to: - // return code 2 on test failure - // 0 on success - // (1 on js error - in error handler) var new_onComplete = function(){ onComplete.call( this ); var returnCode = ( this.test.testResults.failed )?( 2 ):( 0 ); @@ -467,7 +485,6 @@ * @param {Function} catchFn some portion of the correct error msg */ SpaceGhost.prototype.tryStepsCatch = function tryStepsCatch( stepsFn, catchFn ){ - //TODO: * @param {Boolean} removeOtherListeners option to remove other listeners while this fires // create three steps: 1) set up new error handler, 2) try the fn, 3) check for errors and rem. handler var originalExitOnError, originalErrorHandlers = [], @@ -522,11 +539,10 @@ /** Casper has an (undocumented?) skip test feature. This is a conv. wrapper for that. */ SpaceGhost.prototype.skipTest = function(){ - //TODO: does this work? seems to... throw this.test.SKIP_MESSAGE; }; -/** test helper - within frame, assert selector, and assert text in selector +/** Test helper - within frame, assert selector, and assert text in selector * @param {CasperJS selector} selector what element in which to search for the text * @param {String} text what text to search for * @param {String} frame frame selector (gen. name) in which to search for selector (defaults to top) @@ -548,15 +564,17 @@ } } -/** test helper - within frame, assert errormessage, and assert text in errormessage +/** Test helper - within frame, assert errormessage, and assert text in errormessage * *message is a common UI feedback motif in Galaxy (often displayed in the main panel) - * @param {String} message what the message should contain - * @param {String} frame frame selector (gen. name) in which to search for selector (defaults to 'galaxy_main') - * @param {CasperJS selector} messageSelector what element in which to search for the text (defaults to '.errormessage') + * @param {String} message what the message should contain + * @param {String} frame frame selector (gen. name) in which to search for selector + * (defaults to 'galaxy_main') + * @param {CasperJS selector} messageSelector what element in which to search for the text + * (defaults to '.errormessage') */ SpaceGhost.prototype.assertErrorMessage = function assertSelectorAndTextInFrame( message, frame, messageSelector ){ - messageSelector = messageSelector || this.selectors.messages.error; - frame = frame || this.selectors.frames.main; + messageSelector = messageSelector || this.data.selectors.messages.error; + frame = frame || this.data.selectors.frames.main; this.assertSelectorAndTextInFrame( messageSelector, message, frame ); }; @@ -610,9 +628,32 @@ this.test.assert( toSearch.indexOf( searchFor ) !== -1, msg ); }; +/** Assert that a given element has a given class. + * @param {CasperJS selector} selector what element to test + * @param {String} className the class to test for (classes passed in with a leading '.' will have it trimmed) + */ +SpaceGhost.prototype.assertHasClass = function assertHasClass( selector, className, msg ){ + className = ( className[0] == '.' )?( className.slice( 1 ) ):( className ); + msg = msg || 'selector "' + selector + '" has class: "' + className + '"'; + var classes = this.getElementAttribute( selector, 'class' ); + this.test.assert( classes.indexOf( className ) !== -1, msg ); +}; + +/** Assert that a given element doesn't have a given class. + * @param {CasperJS selector} selector what element to test + * @param {String} className the class to test for (classes passed in with a leading '.' will have it trimmed) + */ +SpaceGhost.prototype.assertDoesntHaveClass = function assertDoesntHaveClass( selector, className, msg ){ + className = ( className[0] == '.' )?( className.slice( 1 ) ):( className ); + msg = msg || 'selector "' + selector + '" has class: "' + className + '"'; + var classes = this.getElementAttribute( selector, 'class' ); + this.test.assert( classes.indexOf( className ) === -1, msg ); +}; + // =================================================================== CONVENIENCE /** Wraps casper.getElementInfo in try, returning null if element not found instead of erroring. * @param {String} selector css or xpath selector for the element to find + * @returns {Object|null} element info if found, null if not */ SpaceGhost.prototype.elementInfoOrNull = function elementInfoOrNull( selector ){ var found = null; @@ -622,8 +663,9 @@ return found; }; -/** Wraps casper.click in try, returning true if element found and clicked, false if not instead of erroring. +/** Wraps casper.click in try to prevent error if element isn't found * @param {String} selector css or xpath selector for the element to find + * @returns {Boolean} true if element found and clicked, false if not instead of erroring */ SpaceGhost.prototype.tryClick = function tryClick( selector ){ var done = false; @@ -634,9 +676,9 @@ return done; }; +// =================================================================== GALAXY CONVENIENCE -// =================================================================== GALAXY CONVENIENCE // =================================================================== MISCELAIN /** Override capture to save to environ: GALAXY_TEST_SAVE (or passed in from CLI) * @param {String} filename the image filename @@ -651,6 +693,7 @@ /** Pop all handlers for eventName from casper and return them in order. * @param {String} eventName the name of the event from which to remove handlers + * @returns {Function[]} the array of functions no longer bound to the event */ SpaceGhost.prototype.popAllListeners = function popAllListeners( eventName ){ var returnedListeners = this.listeners( eventName ); @@ -660,7 +703,7 @@ /** Add the given list of handler functions to the listener for eventName in order. * @param {String} eventName the name of the event to which to add handlers - * @param {Array} handlerArray an array of event handler functions to add + * @param {Function[]} handlerArray an array of event handler functions to add */ SpaceGhost.prototype.addListeners = function addListeners( eventName, handlerArray ){ for( var i=0; i<handlerArray.length; i++ ){ @@ -668,14 +711,15 @@ } }; -/** Send message to stderr +/** Send message to stderr using the phantom fs module. + * @param {String} the msg to output */ SpaceGhost.prototype.stderr = function( msg ){ var fs = require( 'fs' ); fs.write( '/dev/stderr', msg + '\n', 'w' ); }; -// convenience logging funcs +// ------------------------------------------------------------------- convenience logging funcs /** log using level = 'debug' and default namespace = 'spaceghost' */ SpaceGhost.prototype.debug = function( msg, namespace ){ @@ -690,14 +734,14 @@ this.log( msg, 'info', namespace ); }; -/** log using level = 'info' and default namespace = 'spaceghost' +/** log using level = 'warning' and default namespace = 'spaceghost' */ SpaceGhost.prototype.warning = function( msg, namespace ){ namespace = namespace || 'spaceghost'; this.log( msg, 'warning', namespace ); }; -/** log using level = 'info' and default namespace = 'spaceghost' +/** log using level = 'error' and default namespace = 'spaceghost' */ SpaceGhost.prototype.error = function( msg, namespace ){ namespace = namespace || 'spaceghost'; @@ -707,6 +751,7 @@ /** log despite logLevel settings, unless returnJsonOnly is set */ SpaceGhost.prototype.out = function( msg, namespace ){ + namespace = namespace || 'spaceghost'; if( !this.options.returnJsonOnly ){ console.debug( msg ); } @@ -718,10 +763,10 @@ return JSON.stringify( obj, null, 2 ); }; -/** output to debug the JSON of the selector (or null if not found) +/** output the JSON of the selector (or null if not found) to debug level */ SpaceGhost.prototype.debugElement = function debugElement( selector ){ - this.debug( this.jsonStr( this.elementInfoOrNull( selector ) ) ); + this.debug( selector + ':\n' + this.jsonStr( this.elementInfoOrNull( selector ) ) ); }; /** Debug SG itself @@ -737,21 +782,6 @@ return this.errors[( this.errors.length - 1 )]; }; -/** Get the last error from an assertRaises test (gen. for the message) - */ -SpaceGhost.prototype.getLastAssertRaisesError = function(){ - // assuming the test passed here... - var testsThatPassed = this.test.testResults.passes; - var test = null; - for( var i=( testsThatPassed.length - 1 ); i>=0; i-- ){ - currTest = testsThatPassed[i]; - if( currTest.type === 'assertRaises' ){ - test = currTest; break; - } - } - return ( ( test && test.values )?( test.values.error ):( undefined ) ); -}; - /** String representation */ SpaceGhost.prototype.toString = function(){ @@ -762,12 +792,105 @@ return 'SpaceGhost(' + currentUrl + ')'; }; +/** Load and parse a JSON file into an object. + * @param filepath filepath relative to the current scriptDir + * @returns the object parsed + */ +SpaceGhost.prototype.loadJSONFile = function loadJSONFile( filepath ){ + //precondition: filepath is relative to script dir + filepath = this.options.scriptDir + filepath; + return JSON.parse( require( 'fs' ).read( filepath ) ); +}; + +/** Load and parse a JSON file into an object. + * @param filepath filepath relative to the current scriptDir + * @param object the object to write + * @param mode 'w' for a new file, 'a' for append + */ +SpaceGhost.prototype.writeJSONFile = function writeJSONFile( filepath, object, mode ){ + mode = mode || 'w'; + //precondition: filepath is relative to script dir + filepath = this.options.scriptDir + filepath; + return require( 'fs' ).write( filepath, this.jsonStr( object ), mode ); +}; + // =================================================================== TEST DATA -// maintain selectors, labels, text here in one central location +/** General use selectors, labels, and text. Kept here to allow a centralized location. + */ +SpaceGhost.prototype.data = { + selectors : { + tooltipBalloon : '.bs-tooltip', + editableText : '.editable-text', + editableTextInput : 'input#renaming-active', + masthead : { + userMenu : { + userEmail : 'a #user-email', + userEmail_xpath : '//a[contains(text(),"Logged in as")]/span["id=#user-email"]' + } + }, + frames : { + main : 'galaxy_main', + tools : 'galaxy_tools', + history : 'galaxy_history' + }, + messages : { + all : '[class*="message"]', + error : '.errormessage', + done : '.donemessage', + donelarge : '.donemessagelarge' + }, + loginPage : { + form : 'form#login', + submit_xpath : "//input[@value='Login']", + url_regex : /\/user\/login/ + }, + registrationPage : { + form : 'form#registration', + submit_xpath : "//input[@value='Submit']" + }, + tools : { + general : { + form : 'form#tool_form', + executeButton_xpath : '//input[@value="Execute"]' + }, + upload : { + fileInput : 'files_0|file_data' // is this general? + } + } + }, + labels : { + masthead : { + menus : { + user : 'User' + }, + userMenu : { + register : 'Register', + login : 'Login', + logout : 'Logout' + } + }, + tools : { + upload : { + panelLabel : 'Upload File' + } + } + }, + text : { + registrationPage : { + badEmailError : 'Enter a real email address' + }, + upload : { + success : 'The following job has been successfully added to the queue' + } + } +}; -//TODO: to data +/* SpaceGhost.prototype.selectors = { + tooltipBalloon : '.bs-tooltip', + editableText : '.editable-text', + editableTextInput : 'input#renaming-active', masthead : { userMenu : { userEmail : 'a #user-email', @@ -832,28 +955,14 @@ success : 'The following job has been successfully added to the queue' } }; - -SpaceGhost.prototype.loadJSONFile = function loadJSONFile( filepath ){ - //precondition: filepath is relative to script dir - filepath = this.options.scriptDir + filepath; - return JSON.parse( require( 'fs' ).read( filepath ) ); -}; - -SpaceGhost.prototype.writeJSONFile = function writeJSONFile( filepath, object, mode ){ - mode = mode || 'w'; - //precondition: filepath is relative to script dir - filepath = this.options.scriptDir + filepath; - return require( 'fs' ).write( filepath, this.jsonStr( object ), mode ); -}; +*/ // =================================================================== EXPORTS -/** - */ exports.SpaceGhost = SpaceGhost; exports.PageError = PageError; exports.GalaxyError = GalaxyError; exports.AlertError = AlertError; -/** +/** creation function */ exports.create = function create(options) { "use strict"; Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: browser tests: add tests for structure of HDA in the 'ok' state
by commits-noreply＠bitbucket.org 13 Mar '13

13 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c2f310489973/ changeset: c2f310489973 user: carlfeberhard date: 2013-03-13 21:03:31 summary: browser tests: add tests for structure of HDA in the 'ok' state affected #: 4 files diff -r 4827bed265e7563c58346654f5a668f5de6d9d68 -r c2f310489973fc2ba42e5289ccec3ed88a42aabc test/casperjs/hda-state-tests.js --- a/test/casperjs/hda-state-tests.js +++ b/test/casperjs/hda-state-tests.js @@ -40,22 +40,22 @@ spaceghost.info( 'Will use fixtureData.testUser: ' + email ); } -var newHistoryName = "Test History", +var tooltipSelector = '.bs-tooltip'; + +var utils = require( 'utils' ), historyFrameInfo = {}, filepathToUpload = '../../test-data/1.txt', - possibleHDAStates = [], - testUploadInfo = {}; + testUploadInfo = {}, + //TODO: get from the api module - that doesn't exist yet + summaryShouldBeArray = [ '10 lines', 'format: txt' ], + infoShouldBe = 'uploaded txt file', + peekShouldBeArray = []; // ------------------------------------------------------------------- set up // start a new user spaceghost.user.loginOrRegisterUser( email, password ); -// ??: why is a reload needed here? If we don't, loggedInAs === '' ... -spaceghost.thenOpen( spaceghost.baseUrl, function(){ - var loggedInAs = spaceghost.user.loggedInAs(); - this.test.assert( loggedInAs === email, 'loggedInAs() matches email: "' + loggedInAs + '"' ); -}); -// grab the history frame bounds for mouse later tests +// grab the history frame bounds for later mouse tests spaceghost.then( function(){ historyFrameInfo = this.getElementInfo( 'iframe[name="galaxy_history"]' ); //this.debug( 'historyFrameInfo:' + this.jsonStr( historyFrameInfo ) ); @@ -69,15 +69,211 @@ }); }); -spaceghost.then( function getHDAStates(){ + +// =================================================================== TESTS +// ------------------------------------------------------------------- helpers +//NOTE: to be called with fn.call( spaceghost, ... ) + +function testTitle( hdaSelector, hid, name ){ + var titleSelector = hdaSelector + ' ' + this.historypanel.data.selectors.hda.title, + titleShouldBe = hid + ': ' + name; + this.test.assertVisible( titleSelector, + 'HDA title is visible' ); + this.test.assertSelectorHasText( titleSelector, titleShouldBe, + 'HDA has proper hid and title' ); +} + +function testTitleButtonStructure( hdaSelector, shouldHaveTheseButtons ){ + // defaults to the current buttons most states should have + shouldHaveTheseButtons = shouldHaveTheseButtons || [ 'display', 'edit', 'delete' ]; + + var hdaDbId = this.getElementAttribute( hdaSelector, 'id' ).split( '-' )[1], + buttonsArea = hdaSelector + ' ' + this.historypanel.data.selectors.hda.titleButtons, + buttons = { + // this seems backwards -> TODO: move buttonsArea concat into loop below, move this data to historypanel.data + display : { + nodeName : this.historypanel.data.text.hda.ok.nodeNames.displayButton, + selector : buttonsArea + ' ' + this.historypanel.data.selectors.hda.displayButton, + tooltip : this.historypanel.data.text.hda.ok.tooltips.displayButton, + hrefTpl : this.historypanel.data.text.hda.ok.hrefs.displayButton + }, + edit : { + nodeName : this.historypanel.data.text.hda.ok.nodeNames.editAttrButton, + selector : buttonsArea + ' ' + this.historypanel.data.selectors.hda.editAttrButton, + tooltip : this.historypanel.data.text.hda.ok.tooltips.editAttrButton, + hrefTpl : this.historypanel.data.text.hda.ok.hrefs.editAttrButton + }, + 'delete' : { + nodeName : this.historypanel.data.text.hda.ok.nodeNames.deleteButton, + selector : buttonsArea + ' ' + this.historypanel.data.selectors.hda.deleteButton, + tooltip : this.historypanel.data.text.hda.ok.tooltips.deleteButton, + hrefTpl : this.historypanel.data.text.hda.ok.hrefs.deleteButton + } + }; + this.test.assertVisible( buttonsArea, 'Button area is visible' ); + + for( var i=0; i<shouldHaveTheseButtons.length; i++ ){ + // don't use button names we don't have data for + var buttonName = shouldHaveTheseButtons[ i ]; + if( !buttons.hasOwnProperty( buttonName ) ){ continue; } + + this.test.comment( buttonName + ' should exist, be visible, and well formed' ); + var button = buttons[ buttonName ]; + this.debug( 'checking button "' + buttonName + '" on hda "' + hdaDbId + '":\n' + this.jsonStr( button ) ); + this.test.assertExists( button.selector, buttonName + ' button exists' ); + this.test.assertVisible( button.selector, buttonName + ' button is visible' ); + + var buttonElement = this.getElementInfo( button.selector ); + this.debug( 'buttonElement:' + this.jsonStr( buttonElement ) ); + + // should be an anchor + this.test.assert( buttonElement.nodeName === button.nodeName, + buttonName + ' is proper node type (' + button.nodeName + '): ' + buttonElement.nodeName ); + + // should have a proper href + var href = buttonElement.attributes.href, + hrefShouldBe = utils.format( button.hrefTpl, hdaDbId ); + this.assertTextContains( href, hrefShouldBe, + buttonName + ' has proper href (' + hrefShouldBe + '): ' + href ); + + this.historypanel.hoverOver( button.selector, function testingHover(){ + var tooltipText = button.tooltip; + this.test.assertVisible( tooltipSelector, buttonName + ' button tooltip is visible when hovering' ); + this.test.assertSelectorHasText( tooltipSelector, tooltipText, + buttonName + ' button has tooltip text: "' + tooltipText + '"' ); + }, historyFrameInfo ); + } +} + +function testDbkey( hdaSelector, dbkeySetTo ){ + var dbkeySelector = hdaSelector + ' ' + this.historypanel.data.selectors.hda.body + + ' ' + this.historypanel.data.selectors.hda.dbkey, + unspecifiedDbkeyText = '?', + unspecifiedDbkeyNodeName = 'a', + specifiedDbkeyNodeName = 'span', + editAttrHrefRegex = /\/datasets\/\w+\/edit/; + + this.test.assertExists( dbkeySelector, 'dbkey exists' ); + this.test.assertVisible( dbkeySelector, 'dbkey is visible' ); + var dbkey = this.elementInfoOrNull( dbkeySelector ); + if( !dbkey ){ return; } + + // dbkey is set, check text + if( dbkeySetTo ){ + this.test.comment( '(specified) dbkey should be displayed correctly' ); + this.test.assertSelectorHasText( dbkeySelector, dbkeySetTo, + 'dbkey is specified: ' + dbkey.text ); + this.test.assert( dbkey.nodeName === specifiedDbkeyNodeName, + 'dbkey has proper nodeName (' + specifiedDbkeyNodeName + '):' + dbkey.nodeName ); + + // dbkey expected to be not set + } else { + this.test.comment( '(unspecified) dbkey should be displayed correctly' ); + this.test.assertSelectorHasText( dbkeySelector, unspecifiedDbkeyText, + 'dbkey is not specified: ' + dbkey.text ); + this.test.assert( dbkey.nodeName === unspecifiedDbkeyNodeName, + 'dbkey has proper nodeName (' + unspecifiedDbkeyNodeName + '):' + dbkey.nodeName ); + + this.test.comment( '(unspecified) dbkey href should point to edit attributes' ); + this.test.assertMatch( dbkey.attributes.href, editAttrHrefRegex, + 'dbkey has a proper href: ' + dbkey.attributes.href ); + } +} + +function testPrimaryActionButtons( hdaSelector ){ + var buttonsSelector = hdaSelector + ' ' + this.historypanel.data.selectors.hda.body + + ' ' + this.historypanel.data.selectors.hda.primaryActionButtons; + this.test.comment( 'Primary action buttons div should exist and be visible' ); + this.test.assertExists( buttonsSelector, 'Primary action buttons div exists' ); + this.test.assertVisible( buttonsSelector, 'Primary action buttons div is visible' ); +} + +function testSecondaryActionButtons( hdaSelector ){ + var buttonsSelector = hdaSelector + ' ' + this.historypanel.data.selectors.hda.body + + ' ' + this.historypanel.data.selectors.hda.secondaryActionButtons; + this.test.comment( 'Secondary action buttons div should exist and be visible' ); + this.test.assertExists( buttonsSelector, 'Secondary action buttons div exists' ); + this.test.assertVisible( buttonsSelector, 'Secondary action buttons div is visible' ); +} + +function testPeek( hdaSelector, expectedPeekArray ){ + var peekSelector = hdaSelector + ' ' + this.historypanel.data.selectors.hda.body + + ' ' + this.historypanel.data.selectors.hda.peek; + this.test.comment( 'Peek div should exist and be visible' ); + this.test.assertExists( peekSelector, 'peek exists' ); + this.test.assertVisible( peekSelector, 'peek is visible' ); + expectedPeekArray.forEach( function( string, i ){ + spaceghost.test.assertSelectorHasText( peekSelector, string, 'peek has proper text (' + string + ')' ); + }); +} + +function testExpandedBody( hdaSelector, expectedSummaryTextArray, expectedInfoText, dbkeySetTo ){ + var body = hdaSelector + ' ' + this.historypanel.data.selectors.hda.body; + this.test.assertExists( body, 'body exists' ); + this.test.assertVisible( body, 'body is visible' ); + + //TODO: create api module, match with api history_contents + + this.test.comment( 'Summary should be displayed correctly' ); + var summary = body + ' ' + this.historypanel.data.selectors.hda.summary; + this.test.assertExists( summary, 'summary exists' ); + this.test.assertVisible( summary, 'summary is visible' ); + // summary text is broken up by whitespace making it inconv. to test in one go + expectedSummaryTextArray.forEach( function( string, i ){ + spaceghost.test.assertSelectorHasText( summary, string, 'summary has proper text (' + string + ')' ); + }); + this.debug( 'summary text: ' + this.fetchText( summary ) ); + + testDbkey.call( this, hdaSelector, dbkeySetTo ); + + this.test.comment( 'Info should be displayed correctly' ); + var info = body + ' ' + this.historypanel.data.selectors.hda.info; + this.test.assertExists( info, 'info exists' ); + this.test.assertVisible( info, 'info is visible' ); + this.test.assertSelectorHasText( info, expectedInfoText, + 'info has proper text (' + expectedInfoText + '): ' + this.fetchText( info ) ); + + testPrimaryActionButtons.call( this, hdaSelector ); + testSecondaryActionButtons.call( this, hdaSelector ); //TODO: isAnonymous + testPeek.call( this, hdaSelector, peekShouldBeArray ); +} + +// ------------------------------------------------------------------- ok state +spaceghost.then( function checkOkState(){ + this.test.comment( 'HDAs in the "ok" state should be well formed' ); + this.withFrame( this.selectors.frames.history, function(){ - var model = this.evaluate( function(){ - return Galaxy.currHistoryPanel.model.hdas.at( 0 ).attributes; + var uploadSelector = '#' + testUploadInfo.hdaElement.attributes.id; + this.test.assertVisible( uploadSelector, 'HDA is visible' ); + + this.test.comment( 'should have the proper state class' ); + var okStateClass = this.historypanel.data.selectors.hda.wrapper.stateClasses.ok, + uploadElement = this.getElementInfo( uploadSelector ); + this.test.assert( uploadElement.attributes['class'].indexOf( okStateClass ) !== -1, + 'HDA has "ok" state class' ); + + // since we're using css there's no great way to test state icon (.state-icon is empty) + + this.test.comment( 'should have proper title and hid' ); + testTitle.call( spaceghost, uploadSelector, testUploadInfo.hid, testUploadInfo.name ); + + this.test.comment( 'should have all of the three, main buttons' ); + testTitleButtonStructure.call( spaceghost, uploadSelector ); + + this.test.comment( 'body is not visible before clicking the hda title' ); + var body = uploadSelector + ' ' + this.historypanel.data.selectors.hda.body; + this.test.assertNotVisible( body, 'body is not visible' ); + + this.test.comment( 'clicking the hda title should expand its body' ); + var hdaTitle = uploadSelector + ' ' + this.historypanel.data.selectors.hda.title; + this.click( hdaTitle ); + this.wait( 500, function(){ + testExpandedBody.call( spaceghost, uploadSelector, summaryShouldBeArray, infoShouldBe, false ); }); - this.info( 'model:' + this.jsonStr( model ) ); }); }); +/* spaceghost.then( function checkNewState(){ this.test.comment( 'HDAs in the "new" state should be well formed' ); @@ -93,15 +289,10 @@ this.test.assertVisible( uploadSelector, 'HDA is visible' ); // should have proper title and hid - var titleSelector = uploadSelector + ' .historyItemTitle'; - this.test.assertVisible( titleSelector, 'HDA title is visible' ); - this.test.assertSelectorHasText( titleSelector, testUploadInfo.name, - 'HDA has proper title' ); - this.test.assertSelectorHasText( titleSelector, testUploadInfo.hid, - 'HDA has proper hid' ); + testTitle.call( spaceghost, uploadSelector, testUploadInfo.hid, testUploadInfo.name ); // should have the new state class - var newStateClass = 'historyItem-new', + var newStateClass = this.historypanel.data.selectors.hda.wrapper.stateClasses['new']; uploadElement = this.getElementInfo( uploadSelector ); this.test.assert( uploadElement.attributes['class'].indexOf( newStateClass ) !== -1, 'HDA has new state class' ); @@ -111,13 +302,13 @@ //this.test.assertVisible( stateIconSelector, 'HDA has proper hid' ); // should NOT have any of the three, main buttons - var buttonSelector = uploadSelector + ' .historyItemButtons a'; + var buttonSelector = uploadSelector + ' ' + this.historypanel.data.selectors.hda.titleButtons + ' a'; this.test.assertDoesntExist( buttonSelector, 'No display, edit, or delete buttons' ); // expand and check the body this.click( titleSelector ); this.wait( 500, function(){ - var bodySelector = uploadSelector + ' .historyItemBody'; + var bodySelector = uploadSelector + ' ' + this.historypanel.data.selectors.hda.body; this.test.assertVisible( bodySelector, 'HDA body is visible (after expanding)' ); var expectedBodyText = 'This is a new dataset'; @@ -130,8 +321,7 @@ }); }); }); - -// =================================================================== TESTS +*/ // =================================================================== diff -r 4827bed265e7563c58346654f5a668f5de6d9d68 -r c2f310489973fc2ba42e5289ccec3ed88a42aabc test/casperjs/history-panel-tests.js --- a/test/casperjs/history-panel-tests.js +++ b/test/casperjs/history-panel-tests.js @@ -79,7 +79,7 @@ this.test.assert( loggedInAs === email, 'loggedInAs() matches email: "' + loggedInAs + '"' ); }); -// grab the history frame bounds for mouse later tests +// grab the history frame bounds for later mouse tests spaceghost.then( function(){ historyFrameInfo = this.getElementInfo( 'iframe[name="galaxy_history"]' ); //this.debug( 'historyFrameInfo:' + this.jsonStr( historyFrameInfo ) ); diff -r 4827bed265e7563c58346654f5a668f5de6d9d68 -r c2f310489973fc2ba42e5289ccec3ed88a42aabc test/casperjs/modules/historypanel.js --- a/test/casperjs/modules/historypanel.js +++ b/test/casperjs/modules/historypanel.js @@ -136,12 +136,12 @@ }; //TODO!: this will break if the hda name has single or double quotes (which are permitted in names) -/** Find the id of the hda wrapper given the hda title and hid. +/** Find the DOM id of the hda wrapper given the hda title and hid. * @param {String} title the title of the hda * @param {Int} hid (optional) the hid of the hda to look for * @returns {String|null} DOM id of the historyItemWrapper found, null if not found */ -HistoryPanel.prototype.hdaIdByTitle = function hdaIdByTitle( title, hid ){ +HistoryPanel.prototype.hdaElementIdByTitle = function hdaElementIdByTitle( title, hid ){ var elementInfo = this.hdaElementInfoByTitle( title, hid ); return (( elementInfo && elementInfo.attributes && elementInfo.attributes.id )? ( elementInfo.attributes.id ):( null )); @@ -200,6 +200,49 @@ return spaceghost; }; +/** Hover over an element in the history panel. + * This is re-implemented here because element bounds in iframes are calc'd + * relative to the iframe - but mouse coords are not. Capture the iframe + * bounds first to re-calc for mouse coords + * @param {String} selector a css or xpath selector for an historyItemWrapper + * @param {Function} whenHovering a function to call after the hover (will be scoped to spaceghost) + * @param {ElementInfo} historyFrameInfo casper ElementInfo for the history iframe (optional) + * If undefined, hoverOver will use withFrame first and gather the information itself. + * Send in history iframe info if you're already in the frame when calling this. bleh. + */ +HistoryPanel.prototype.hoverOver = function hoverOver( selector, whenHovering, historyFrameInfo ){ + var spaceghost = this.spaceghost; + + // helper function + function hoverAndCallback( historyFrameInfo, selector, whenHovering ){ + // ...this suddenly started working when I upped the viewport size + //this.debug( 'historyFrameInfo:' + this.jsonStr( historyFrameInfo ) ); + var elementInfo = this.getElementInfo( selector ), + newCoords = { x: ( historyFrameInfo.x + elementInfo.x ), + y: ( historyFrameInfo.y + elementInfo.y ) }; + //this.debug( 'elementInfo:' + this.jsonStr( elementInfo ) ); + //this.debug( 'newCoords:' + this.jsonStr( newCoords ) ); + this.page.sendEvent( 'mousemove', newCoords.x + 1, newCoords.y + 1 ); + if( whenHovering ){ + whenHovering.call( this ); + } + } + + // complicated by iframes + // if no history frame info was passed - assume not in history frame already and move into using withFrame + if( !historyFrameInfo ){ + //TODO: move selector to data (use selectors.frames? ) + historyFrameInfo = spaceghost.getElementInfo( 'iframe[name="galaxy_history"]' ); + spaceghost.withFrame( spaceghost.selectors.frames.history, function inHistoryPanel(){ + hoverAndCallback.call( spaceghost, historyFrameInfo, selector, whenHovering ); + }); + + // otherwise, assume we're already 'in' the history frame and use the passed info + } else { + hoverAndCallback.call( spaceghost, historyFrameInfo, selector, whenHovering ); + } + //return spaceghost; +}; // =================================================================== SELECTORS //TODO: data is not a very good name @@ -219,9 +262,22 @@ wrapper : { stateClasses : { prefix : 'historyItem-', - ok : 'historyItem-ok' + ok : 'historyItem-ok', + 'new' : 'historyItem-new' } - } + }, + title : '.historyItemTitle', + titleButtons : '.historyItemButtons', + displayButton : '.icon-button.display', + editAttrButton : '.icon-button.edit', + deleteButton : '.icon-button.delete', + body : '.historyItemBody', + summary : '.hda-summary', + dbkey : '.metadata-dbkey', + info : '.hda-info', + primaryActionButtons : 'div[id^="primary-actions"]', + secondaryActionButtons : 'div[id^="secondary-actions"]', + peek : 'pre.peek' } }, labels : { @@ -247,6 +303,23 @@ emptyMsg : "Your history is empty. Click 'Get Data' on the left pane to start" }, hda : { + ok: { + tooltips : { + displayButton : 'Display data in browser', + editAttrButton : 'Edit Attributes', + deleteButton : 'Delete' + }, + hrefs : { + displayButton : '/datasets/%s/display', + editAttrButton : '/datasets/%s/edit', + deleteButton : '/datasets/%s/delete_async' + }, + nodeNames : { + displayButton : 'a', + editAttrButton : 'a', + deleteButton : 'a' + } + } } } }; diff -r 4827bed265e7563c58346654f5a668f5de6d9d68 -r c2f310489973fc2ba42e5289ccec3ed88a42aabc test/casperjs/spaceghost.js --- a/test/casperjs/spaceghost.js +++ b/test/casperjs/spaceghost.js @@ -136,7 +136,6 @@ this.debug( 'clientScripts:\n' + this.jsonStr( this.options.clientScripts ) ); this._loadModules(); - }; /** Allow CLI arguments to set options if the proper option name is used. @@ -415,6 +414,15 @@ }; // =================================================================== PAGE CONTROL +/** An override of casper.start for additional set up. + * (Currently only used to change viewport) + */ +SpaceGhost.prototype.start = function start(){ + var returned = Casper.prototype.start.apply( this, arguments ); + this.viewport( 1024, 728 ); + return returned; +}; + /** An override of casper.open specifically for Galaxy. * (Currently only used to change language headers) */ @@ -492,18 +500,20 @@ }); }; -/** Override capture to save to environ: GALAXY_TEST_SAVE (or passed in from CLI) - * @param {String} filename the image filename +/** Hover over an element. + * NOTE: not for use with iframes (main, tool, history) - they need to re-calc + * for the iframe bounds and should be implemented in their own modules + * @param {String} selector a css or xpath selector for an historyItemWrapper + * @param {Function} whenHovering a function to call after the hover (will be scoped to spaceghost) */ -SpaceGhost.prototype.capture = function capture( filename, clipRect_or_selector ){ - //TODO: override with saved output dir - if( clipRect_or_selector && ( !utils.isClipRect( clipRect_or_selector ) ) ){ - this.debug( "USING CAPTURE SELECTOR" ); - return this.captureSelector( filename, clipRect_or_selector ); - } - return Casper.prototype.capture.apply( this, arguments ); +SpaceGhost.prototype.hoverOver = function hoverOver( selector, whenHovering ){ + var elementInfo = this.getElementInfo( selector ); + this.page.sendEvent( 'mousemove', elementInfo.x + 1, elementInfo.y + 1 ); + whenHovering.call( this ); + return this; }; + // =================================================================== TESTING //TODO: form fill doesn't work as casperjs would want it - often a button -> controller url //TODO: saveScreenshot (to GALAXY_TEST_SAVE) @@ -591,6 +601,15 @@ }); }; +/** Assert that a given string (toSearch) contains some given string (searchFor). + * @param {String} toSearch the string to search + * @param {String} searchFor the string to search for + * @param {String} msg assertion msg to display + */ +SpaceGhost.prototype.assertTextContains = function assertTextContains( toSearch, searchFor, msg ){ + this.test.assert( toSearch.indexOf( searchFor ) !== -1, msg ); +}; + // =================================================================== CONVENIENCE /** Wraps casper.getElementInfo in try, returning null if element not found instead of erroring. * @param {String} selector css or xpath selector for the element to find @@ -619,6 +638,17 @@ // =================================================================== GALAXY CONVENIENCE // =================================================================== MISCELAIN +/** Override capture to save to environ: GALAXY_TEST_SAVE (or passed in from CLI) + * @param {String} filename the image filename + */ +SpaceGhost.prototype.capture = function capture( filename, clipRect_or_selector ){ + //TODO: override with saved output dir + if( clipRect_or_selector && ( !utils.isClipRect( clipRect_or_selector ) ) ){ + return this.captureSelector( filename, clipRect_or_selector ); + } + return Casper.prototype.capture.apply( this, arguments ); +}; + /** Pop all handlers for eventName from casper and return them in order. * @param {String} eventName the name of the event from which to remove handlers */ @@ -736,7 +766,7 @@ // =================================================================== TEST DATA // maintain selectors, labels, text here in one central location -//TODO: to separate file? +//TODO: to data SpaceGhost.prototype.selectors = { masthead : { userMenu : { Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a new ~/tool_shed/util/metadata_util.py component to contain all tool shed related repository metadata utility methods.
by commits-noreply＠bitbucket.org 13 Mar '13

13 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4827bed265e7/ changeset: 4827bed265e7 user: greg date: 2013-03-13 20:37:29 summary: Add a new ~/tool_shed/util/metadata_util.py component to contain all tool shed related repository metadata utility methods. affected #: 11 files diff -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 -r 4827bed265e7563c58346654f5a668f5de6d9d68 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -7,6 +7,7 @@ from galaxy.model.orm import or_ import tool_shed.util.shed_util as shed_util import tool_shed.util.shed_util_common as suc +import tool_shed.util.metadata_util as metadata_util from tool_shed.util import encoding_util from galaxy.webapps.tool_shed.util import workflow_util import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids @@ -380,16 +381,16 @@ when an admin is installing a new repository or reinstalling an uninstalled repository. """ shed_config_dict = trans.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) - metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=trans.app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict=shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) + metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app, + repository=tool_shed_repository, + changeset_revision=tool_shed_repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict=shed_config_dict, + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=True ) tool_shed_repository.metadata = metadata_dict trans.sa_session.add( tool_shed_repository ) trans.sa_session.flush() @@ -1462,7 +1463,7 @@ @web.require_admin def reset_metadata_on_selected_installed_repositories( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - message, status = suc.reset_metadata_on_selected_repositories( trans, **kwd ) + message, status = metadata_util.reset_metadata_on_selected_repositories( trans, **kwd ) else: message = util.restore_text( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) @@ -1482,16 +1483,16 @@ tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: original_metadata_dict = repository.metadata - metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - changeset_revision=repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict = repository.get_shed_config_dict( trans.app ), - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=False ) + metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app, + repository=repository, + changeset_revision=repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict = repository.get_shed_config_dict( trans.app ), + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=False ) repository.metadata = metadata_dict if metadata_dict != original_metadata_dict: suc.update_in_shed_tool_config( trans.app, repository ) @@ -1687,16 +1688,16 @@ if repository.includes_data_managers: shed_util.remove_from_data_manager( trans.app, repository ) # Update the repository metadata. - metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - changeset_revision=latest_changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict = repository.get_shed_config_dict( trans.app ), - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=True, - persist=True ) + metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app, + repository=repository, + changeset_revision=latest_changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict = repository.get_shed_config_dict( trans.app ), + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=True, + persist=True ) repository.metadata = metadata_dict # Update the repository changeset_revision in the database. repository.changeset_revision = latest_changeset_revision diff -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 -r 4827bed265e7563c58346654f5a668f5de6d9d68 lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -3,6 +3,7 @@ from galaxy.web.base.controllers.admin import Admin from galaxy.util import inflector import tool_shed.util.shed_util_common as suc +import tool_shed.util.metadata_util as metadata_util import tool_shed.grids.admin_grids as admin_grids from galaxy import eggs @@ -301,7 +302,7 @@ @web.require_admin def reset_metadata_on_selected_repositories_in_tool_shed( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - message, status = suc.reset_metadata_on_selected_repositories( trans, **kwd ) + message, status = metadata_util.reset_metadata_on_selected_repositories( trans, **kwd ) else: message = util.restore_text( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) diff -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 -r 4827bed265e7563c58346654f5a668f5de6d9d68 lib/galaxy/webapps/tool_shed/controllers/hg.py --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -1,7 +1,8 @@ import os, logging from galaxy import web from galaxy.web.base.controller import BaseUIController -from tool_shed.util.shed_util_common import get_repository_by_name_and_owner, set_repository_metadata +from tool_shed.util.shed_util_common import get_repository_by_name_and_owner +from tool_shed.util.metadata_util import set_repository_metadata from galaxy import eggs eggs.require('mercurial') diff -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 -r 4827bed265e7563c58346654f5a668f5de6d9d68 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -11,6 +11,7 @@ from galaxy.util import json from galaxy.model.orm import and_, or_ import tool_shed.util.shed_util_common as suc +import tool_shed.util.metadata_util as metadata_util from tool_shed.util import encoding_util from galaxy.webapps.tool_shed.util import workflow_util from galaxy.webapps.tool_shed.util import common_util @@ -365,7 +366,7 @@ operation = kwd[ 'operation' ].lower() if operation == "preview_tools_in_changeset": repository = suc.get_repository_in_tool_shed( trans, repository_id ) - repository_metadata = suc.get_latest_repository_metadata( trans, repository.id ) + repository_metadata = metadata_util.get_latest_repository_metadata( trans, repository.id ) latest_installable_changeset_revision = repository_metadata.changeset_revision return trans.response.send_redirect( web.url_for( controller='repository', action='preview_tools_in_changeset', @@ -1914,7 +1915,7 @@ def reset_all_metadata( self, trans, id, **kwd ): # This method is called only from the ~/templates/webapps/tool_shed/repository/manage_repository.mako template. # It resets all metadata on the complete changelog for a single repository in the tool shed. - invalid_file_tups, metadata_dict = suc.reset_all_metadata_on_repository_in_tool_shed( trans, id, **kwd ) + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, id, **kwd ) if invalid_file_tups: repository = suc.get_repository_in_tool_shed( trans, id ) message = suc.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict ) @@ -2082,7 +2083,7 @@ else: message += 'The selected files were deleted from the repository. ' kwd[ 'message' ] = message - suc.set_repository_metadata_due_to_new_tip( trans, repository, **kwd ) + metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, **kwd ) else: message = "Select at least 1 file to delete from the repository before clicking Delete selected files." status = "error" diff -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 -r 4827bed265e7563c58346654f5a668f5de6d9d68 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -3,6 +3,7 @@ from galaxy import web, util from galaxy.datatypes import checkers import tool_shed.util.shed_util_common as suc +import tool_shed.util.metadata_util as metadata_util from galaxy import eggs eggs.require('mercurial') @@ -15,6 +16,26 @@ CHUNK_SIZE = 2**20 # 1Mb class UploadController( BaseUIController ): + def check_file_contents_for_email_alerts( self, trans ): + """ + See if any admin users have chosen to receive email alerts when a repository is updated. If so, the file contents of the update must be + checked for inappropriate content. + """ + admin_users = trans.app.config.get( "admin_users", "" ).split( "," ) + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.email_alerts != None ): + email_alerts = json.from_json_string( repository.email_alerts ) + for user_email in email_alerts: + if user_email in admin_users: + return True + return False + def check_file_content_for_html_and_images( self, file_path ): + message = '' + if checkers.check_html( file_path ): + message = 'The file "%s" contains HTML content.\n' % str( file_path ) + elif checkers.check_image( file_path ): + message = 'The file "%s" contains image content.\n' % str( file_path ) + return message @web.expose @web.require_login( 'upload', use_panels=True ) def upload( self, trans, **kwd ): @@ -120,9 +141,9 @@ shutil.move( uploaded_file_name, full_path ) # See if any admin users have chosen to receive email alerts when a repository is # updated. If so, check every uploaded file to ensure content is appropriate. - check_contents = suc.check_file_contents( trans ) + check_contents = self.check_file_contents_for_email_alerts( trans ) if check_contents and os.path.isfile( full_path ): - content_alert_str = self.__check_file_content( full_path ) + content_alert_str = self.check_file_content_for_html_and_images( full_path ) else: content_alert_str = '' commands.add( repo.ui, repo, full_path ) @@ -166,7 +187,7 @@ else: message += " %d files were removed from the repository root. " % len( files_to_remove ) kwd[ 'message' ] = message - suc.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str, **kwd ) + metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str, **kwd ) if repository.metadata_revisions: # A repository's metadata revisions are order descending by update_time, so the zeroth revision will be the tip just after an upload. metadata_dict = repository.metadata_revisions[0].metadata @@ -330,11 +351,11 @@ pass # See if any admin users have chosen to receive email alerts when a repository is # updated. If so, check every uploaded file to ensure content is appropriate. - check_contents = suc.check_file_contents( trans ) + check_contents = self.check_file_contents_for_email_alerts( trans ) for filename_in_archive in filenames_in_archive: # Check file content to ensure it is appropriate. if check_contents and os.path.isfile( filename_in_archive ): - content_alert_str += self.__check_file_content( filename_in_archive ) + content_alert_str += self.check_file_content_for_html_and_images( filename_in_archive ) commands.add( repo.ui, repo, filename_in_archive ) if filename_in_archive.endswith( 'tool_data_table_conf.xml.sample' ): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries @@ -421,10 +442,3 @@ message = "Uploaded archives cannot contain hgrc files." return False, message return True, '' - def __check_file_content( self, file_path ): - message = '' - if checkers.check_html( file_path ): - message = 'The file "%s" contains HTML content.\n' % str( file_path ) - elif checkers.check_image( file_path ): - message = 'The file "%s" contains image content.\n' % str( file_path ) - return message diff -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 -r 4827bed265e7563c58346654f5a668f5de6d9d68 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -8,6 +8,7 @@ from galaxy.util.json import from_json_string, to_json_string import tool_shed.util.shed_util as shed_util import tool_shed.util.shed_util_common as suc +import tool_shed.util.metadata_util as metadata_util from galaxy.util.odict import odict from tool_shed.util import common_util @@ -179,16 +180,16 @@ else: print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \ % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) - metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=self.app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict = self.shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) + metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=self.app, + repository=tool_shed_repository, + changeset_revision=tool_shed_repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict = self.shed_config_dict, + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=True ) tool_shed_repository.metadata = metadata_dict self.app.sa_session.add( tool_shed_repository ) self.app.sa_session.flush() diff -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 -r 4827bed265e7563c58346654f5a668f5de6d9d68 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -1,5 +1,6 @@ import tool_shed.util.shed_util as shed_util import tool_shed.util.shed_util_common as suc +import tool_shed.util.metadata_util as metadata_util def handle_repository_contents( app, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ): @@ -9,16 +10,16 @@ """ sa_session = app.model.context.current shed_config_dict = app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) - metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict=shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) + metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=app, + repository=tool_shed_repository, + changeset_revision=tool_shed_repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict=shed_config_dict, + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=True ) tool_shed_repository.metadata = metadata_dict sa_session.add( tool_shed_repository ) sa_session.flush() diff -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 -r 4827bed265e7563c58346654f5a668f5de6d9d68 lib/tool_shed/grids/repository_review_grids.py --- a/lib/tool_shed/grids/repository_review_grids.py +++ b/lib/tool_shed/grids/repository_review_grids.py @@ -4,6 +4,7 @@ from galaxy.model.orm import and_, or_ from tool_shed.grids.repository_grids import RepositoryGrid import tool_shed.util.shed_util_common as suc +import tool_shed.util.metadata_util as metadata_util from galaxy import eggs eggs.require('mercurial') @@ -76,9 +77,9 @@ if repository_metadata_revisions: rval = '' for repository_metadata in repository_metadata_revisions: - rev, label, changeset_revision = suc.get_rev_label_changeset_revision_from_repository_metadata( trans, - repository_metadata, - repository=repository ) + rev, label, changeset_revision = metadata_util.get_rev_label_changeset_revision_from_repository_metadata( trans, + repository_metadata, + repository=repository ) rval += '<a href="manage_repository_reviews_of_revision?id=%s&changeset_revision=%s">%s</a> ' % \ ( trans.security.encode_id( repository.id ), changeset_revision, label ) return rval diff -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 -r 4827bed265e7563c58346654f5a668f5de6d9d68 lib/tool_shed/grids/util.py --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -1,5 +1,6 @@ import os, logging import tool_shed.util.shed_util_common as suc +import tool_shed.util.metadata_util as metadata_util from galaxy.web.form_builder import SelectField def build_approved_select_field( trans, name, selected_value=None, for_component=True ): @@ -43,7 +44,7 @@ # Restrict the options to all revisions that have associated metadata. repository_metadata_revisions = repository.metadata_revisions for repository_metadata in repository_metadata_revisions: - rev, label, changeset_revision = suc.get_rev_label_changeset_revision_from_repository_metadata( trans, repository_metadata, repository=repository ) + rev, label, changeset_revision = metadata_util.get_rev_label_changeset_revision_from_repository_metadata( trans, repository_metadata, repository=repository ) changeset_tups.append( ( rev, label, changeset_revision ) ) refresh_on_change_values.append( changeset_revision ) # Sort options by the revision label. Even though the downloadable_revisions query sorts by update_time, This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Comment out description for vcf to summary tree converter tool.
by commits-noreply＠bitbucket.org 13 Mar '13

13 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/185a0ed00d8f/ changeset: 185a0ed00d8f user: dan date: 2013-03-13 20:31:06 summary: Comment out description for vcf to summary tree converter tool. affected #: 1 file diff -r bdd46627880ce13479c49c065ee118952ba8e2f2 -r 185a0ed00d8f245aa4ec161ba134df38edf19f70 lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.xml +++ b/lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.xml @@ -1,5 +1,5 @@ <tool id="CONVERTER_vcf_to_summary_tree_0" name="Convert VCF to Summary Tree" version="1.0.0" hidden="true"> - <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> + <command interpreter="python">vcf_to_summary_tree_converter.py $input1 $output1</command><inputs><page> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Handle Updating Data Managers when installed from Tool Shed.
by commits-noreply＠bitbucket.org 13 Mar '13

13 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/bdd46627880c/ changeset: bdd46627880c user: dan date: 2013-03-13 20:12:55 summary: Handle Updating Data Managers when installed from Tool Shed. affected #: 1 file diff -r 188400f16e5f997f16c2a31db99da55c37909d00 -r bdd46627880ce13479c49c065ee118952ba8e2f2 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1683,6 +1683,9 @@ shed_util.pull_repository( repo, repository_clone_url, latest_ctx_rev ) suc.update_repository( repo, latest_ctx_rev ) tool_shed = suc.clean_tool_shed_url( tool_shed_url ) + # Remove old Data Manager entries + if repository.includes_data_managers: + shed_util.remove_from_data_manager( trans.app, repository ) # Update the repository metadata. metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=trans.app, repository=repository, @@ -1715,6 +1718,11 @@ shed_tool_conf=shed_tool_conf, tool_panel_dict=tool_panel_dict, new_install=False ) + # Add new Data Manager entries + if 'data_manager' in metadata_dict: + new_data_managers = shed_util.install_data_managers( trans.app, trans.app.config.shed_data_manager_config_file, metadata_dict, + repository.get_shed_config_dict( trans.app ), os.path.join( relative_install_dir, name ), + repository, repository_tools_tups ) # Create tool_dependency records if necessary. if 'tool_dependencies' in metadata_dict: tool_dependencies = shed_util.create_tool_dependency_objects( trans.app, repository, relative_install_dir, set_status=False ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix the json import in the new Galaxy bioatar controller.
by commits-noreply＠bitbucket.org 13 Mar '13

13 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/188400f16e5f/ changeset: 188400f16e5f user: greg date: 2013-03-13 19:52:42 summary: Fix the json import in the new Galaxy bioatar controller. affected #: 1 file diff -r 5a17ab5f5466eaf61e80d6efda90c394c601f4ad -r 188400f16e5f997f16c2a31db99da55c37909d00 lib/galaxy/webapps/galaxy/controllers/biostar.py --- a/lib/galaxy/webapps/galaxy/controllers/biostar.py +++ b/lib/galaxy/webapps/galaxy/controllers/biostar.py @@ -5,7 +5,7 @@ from galaxy.web.base.controller import BaseUIController, url_for, error, web import base64 -import json +from galaxy.util import json import hmac # Biostar requires all keys to be present, so we start with a template Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Move build_tool_version_select_field() to ToolBox, to follow move of get_tool_components() in a8caad109542661a7222c504a529ab5afbe303fb.
by commits-noreply＠bitbucket.org 13 Mar '13

13 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5a17ab5f5466/ changeset: 5a17ab5f5466 user: dan date: 2013-03-13 18:50:11 summary: Move build_tool_version_select_field() to ToolBox, to follow move of get_tool_components() in a8caad109542661a7222c504a529ab5afbe303fb. affected #: 2 files diff -r 8a3d271d79bb6736b88b76918e4820d771c12250 -r 5a17ab5f5466eaf61e80d6efda90c394c601f4ad lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -56,6 +56,7 @@ from galaxy.util.template import fill_template from galaxy.visualization.genome.visual_analytics import TracksterConfig from galaxy.web import url_for +from galaxy.web.form_builder import SelectField from tool_shed.util import shed_util_common log = logging.getLogger( __name__ ) @@ -425,7 +426,23 @@ tool_version_select_field = self.build_tool_version_select_field( tools, tool.id, set_selected ) break return tool_version_select_field, tools, tool - + + def build_tool_version_select_field( self, tools, tool_id, set_selected ): + """Build a SelectField whose options are the ids for the received list of tools.""" + options = [] + refresh_on_change_values = [] + for tool in tools: + options.insert( 0, ( tool.version, tool.id ) ) + refresh_on_change_values.append( tool.id ) + select_field = SelectField( name='tool_id', refresh_on_change=True, refresh_on_change_values=refresh_on_change_values ) + for option_tup in options: + selected = set_selected and option_tup[1] == tool_id + if selected: + select_field.add_option( 'version %s' % option_tup[0], option_tup[1], selected=True ) + else: + select_field.add_option( 'version %s' % option_tup[0], option_tup[1] ) + return select_field + def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: path = elem.get( "file" ) diff -r 8a3d271d79bb6736b88b76918e4820d771c12250 -r 5a17ab5f5466eaf61e80d6efda90c394c601f4ad lib/galaxy/webapps/galaxy/controllers/tool_runner.py --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -14,7 +14,6 @@ from galaxy.util.hash_util import is_hashable from galaxy.web import error, url_for from galaxy.web.base.controller import BaseUIController -from galaxy.web.form_builder import SelectField log = logging.getLogger( __name__ ) @@ -219,21 +218,7 @@ add_frame=add_frame, tool_id_version_message=tool_id_version_message, **vars ) - def build_tool_version_select_field( self, tools, tool_id, set_selected ): - """Build a SelectField whose options are the ids for the received list of tools.""" - options = [] - refresh_on_change_values = [] - for tool in tools: - options.insert( 0, ( tool.version, tool.id ) ) - refresh_on_change_values.append( tool.id ) - select_field = SelectField( name='tool_id', refresh_on_change=True, refresh_on_change_values=refresh_on_change_values ) - for option_tup in options: - selected = set_selected and option_tup[1] == tool_id - if selected: - select_field.add_option( 'version %s' % option_tup[0], option_tup[1], selected=True ) - else: - select_field.add_option( 'version %s' % option_tup[0], option_tup[1] ) - return select_field + @web.expose def redirect( self, trans, redirect_url=None, **kwd ): if not redirect_url: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0