# HG changeset patch -- Bitbucket.org
# Project galaxy-dist
# URL http://bitbucket.org/galaxy/galaxy-dist/overview
# User Greg Von Kuster <greg(a)bx.psu.edu>
# Date 1279831412 14400
# Node ID cecc290755014c02aa21906d433a3d39f2d657d5
# Parent db1ae0bc8995cb30deef73ce9d14ef175c512f90
Galaxy Tool Shed enhancements
1. Rename Galaxy Community Space to Galaxy Tool Shed
2. Add the ability to upload an archive consisting of a suite of tools in addition to a single tool archive.
3. Miscellaneous code cleanup including elimination of all sniffer related code.
--- a/templates/webapps/community/tool/edit_tool.mako
+++ b/templates/webapps/community/tool/edit_tool.mako
@@ -93,7 +93,7 @@
%if can_edit:
<form id="edit_tool" name="edit_tool" action="${h.url_for( controller='common', action='edit_tool' )}" method="post">
- %if tool.is_rejected():
+ %if tool.is_rejected:
<div class="toolForm"><div class="toolFormTitle">Reason for rejection</div><div class="toolFormBody">
@@ -169,7 +169,7 @@
</div></div><p/>
- %if tool.is_new() or tool.is_rejected():
+ %if tool.is_new or tool.is_rejected:
<div class="toolForm"><div class="toolFormTitle">Get approval for publishing</div><div class="toolFormBody">
--- a/lib/galaxy/webapps/community/security/__init__.py
+++ b/lib/galaxy/webapps/community/security/__init__.py
@@ -1,5 +1,5 @@
"""
-Galaxy Community Space Security
+Galaxy Tool Shed Security
"""
import logging, socket, operator
from datetime import datetime, timedelta
@@ -167,14 +167,14 @@ class CommunityRBACAgent( RBACAgent ):
def can_approve_or_reject( self, user, user_is_admin, cntrller, item ):
# The current user can approve or reject the item if the user
# is an admin, and the item's state is WAITING.
- return user and user_is_admin and cntrller=='admin' and item.is_waiting()
+ return user and user_is_admin and cntrller=='admin' and item.is_waiting
def can_delete( self, user, user_is_admin, cntrller, item ):
# The current user can delete the item if they are an admin or if they uploaded the
# item and in either case the item's state is not DELETED.
if user and user_is_admin and cntrller == 'admin':
- can_delete = not item.is_deleted()
+ can_delete = not item.is_deleted
elif cntrller in [ 'tool' ]:
- can_delete = user==item.user and not item.is_deleted()
+ can_delete = user==item.user and not item.is_deleted
else:
can_delete = False
return can_delete
@@ -184,7 +184,7 @@ class CommunityRBACAgent( RBACAgent ):
if user and user_is_admin and cntrller == 'admin':
return True
elif cntrller in [ 'tool' ]:
- can_download = not( item.is_new() or item.is_waiting() )
+ can_download = not( item.is_new or item.is_waiting )
else:
can_download = False
return can_download
@@ -194,7 +194,7 @@ class CommunityRBACAgent( RBACAgent ):
if user and user_is_admin and cntrller == 'admin':
return True
if cntrller in [ 'tool' ]:
- return user and user==item.user and ( item.is_new() or item.is_rejected() )
+ return user and user==item.user and ( item.is_new or item.is_rejected )
return False
def can_purge( self, user, user_is_admin, cntrller ):
# The current user can purge the item if they are an admin.
@@ -206,7 +206,7 @@ class CommunityRBACAgent( RBACAgent ):
versions = get_versions( item )
state_ok = True
for version in versions:
- if version.is_new() or version.is_waiting():
+ if version.is_new or version.is_waiting:
state_ok = False
break
return state_ok
@@ -215,7 +215,7 @@ class CommunityRBACAgent( RBACAgent ):
# or if the item's state is APPROVED.
if user and user_is_admin and cntrller == 'admin':
return True
- if cntrller in [ 'tool' ] and item.is_approved():
+ if cntrller in [ 'tool' ] and item.is_approved:
return True
return user and user==item.user
def get_all_action_permissions( self, user, user_is_admin, cntrller, item ):
@@ -236,7 +236,7 @@ class CommunityRBACAgent( RBACAgent ):
elif cntrller in [ 'tool' ]:
visible_versions = []
for version in get_versions( item ):
- if version.is_approved() or version.is_archived() or version.user == user:
+ if version.is_approved or version.is_archived or version.user == user:
visible_versions.append( version )
else:
visible_versions = []
--- a/templates/webapps/community/tool/view_tool.mako
+++ b/templates/webapps/community/tool/view_tool.mako
@@ -85,7 +85,7 @@
%endif
%if can_view:
- %if tool.is_rejected():
+ %if tool.is_rejected:
<div class="toolForm"><div class="toolFormTitle">Reason for rejection</div><div class="toolFormBody">
@@ -168,7 +168,7 @@
<div class="toolFormBody"><div class="form-row"><ul class="toolFile">
- <li><a href="${h.url_for( controller='tool', action='download_tool', id=trans.app.security.encode_id( tool.id ) )}">${tool.download_file_name}</a></li>
+ <li><a href="${h.url_for( controller='common', action='download_tool', id=trans.app.security.encode_id( tool.id ), cntrller=cntrller )}">${tool.download_file_name}</a></li><ul class="fileBrowser">
%for name in tool_file_contents:
<li><a href="${h.url_for( controller='tool', action='view_tool_file', id=trans.app.security.encode_id( tool.id ), file_name=quote_plus( name ) )}">${name}</a></li>
--- a/lib/galaxy/webapps/community/controllers/common.py
+++ b/lib/galaxy/webapps/community/controllers/common.py
@@ -14,6 +14,11 @@ class ToolListGrid( grids.Grid ):
class NameColumn( grids.TextColumn ):
def get_value( self, trans, grid, tool ):
return tool.name
+ class TypeColumn( grids.GridColumn ):
+ def get_value( self, trans, grid, tool ):
+ if tool.is_suite:
+ return 'Suite'
+ return 'Tool'
class VersionColumn( grids.TextColumn ):
def get_value( self, trans, grid, tool ):
return tool.version
@@ -60,6 +65,10 @@ class ToolListGrid( grids.Grid ):
model_class=model.Tool,
attach_popup=False
),
+ TypeColumn( "Type",
+ key="suite",
+ model_class=model.Tool,
+ attach_popup=False ),
VersionColumn( "Version",
key="version",
model_class=model.Tool,
@@ -229,7 +238,7 @@ class CommonController( BaseController )
in_categories.append( ( category.id, category.name ) )
else:
out_categories.append( ( category.id, category.name ) )
- if tool.is_rejected():
+ if tool.is_rejected:
# Include the comments regarding the reason for rejection
reason_for_rejection = get_most_recent_event( tool ).comment
else:
@@ -283,7 +292,7 @@ class CommonController( BaseController )
visible_versions = trans.app.security_agent.get_visible_versions( trans.user, trans.user_is_admin(), cntrller, tool )
categories = [ tca.category for tca in tool.categories ]
tool_file_contents = tarfile.open( tool.file_name, 'r' ).getnames()
- if tool.is_rejected():
+ if tool.is_rejected:
# Include the comments regarding the reason for rejection
reason_for_rejection = get_most_recent_event( tool ).comment
else:
--- /dev/null
+++ b/lib/galaxy/webapps/community/model/migrate/versions/0002_add_tool_suite_column.py
@@ -0,0 +1,48 @@
+"""
+Migration script to add the suite column to the tool table.
+"""
+
+from sqlalchemy import *
+from sqlalchemy.orm import *
+from migrate import *
+from migrate.changeset import *
+
+import logging
+log = logging.getLogger( __name__ )
+
+metadata = MetaData( migrate_engine )
+db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
+
+def upgrade():
+ print __doc__
+ metadata.reflect()
+
+ # Create and initialize imported column in job table.
+ Tool_table = Table( "tool", metadata, autoload=True )
+ c = Column( "suite", Boolean, default=False, index=True )
+ try:
+ # Create
+ c.create( Tool_table )
+ assert c is Tool_table.c.suite
+
+ # Initialize.
+ if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
+ default_false = "0"
+ elif migrate_engine.name == 'postgres':
+ default_false = "false"
+ db_session.execute( "UPDATE tool SET suite=%s" % default_false )
+
+ except Exception, e:
+ print "Adding suite column to the tool table failed: %s" % str( e )
+ log.debug( "Adding suite column to the tool table failed: %s" % str( e ) )
+
+def downgrade():
+ metadata.reflect()
+
+ # Drop imported column from job table.
+ Tool_table = Table( "tool", metadata, autoload=True )
+ try:
+ Tool_table.c.suite.drop()
+ except Exception, e:
+ print "Dropping column suite from the tool table failed: %s" % str( e )
+ log.debug( "Dropping column suite from the tool table failed: %s" % str( e ) )
--- a/lib/galaxy/webapps/community/controllers/upload.py
+++ b/lib/galaxy/webapps/community/controllers/upload.py
@@ -2,6 +2,7 @@ import sys, os, shutil, logging, urllib2
from galaxy.web.base.controller import *
from galaxy.web.framework.helpers import time_ago, iff, grids
from galaxy.model.orm import *
+from galaxy.web.form_builder import SelectField
from galaxy.webapps.community import datatypes
from common import get_categories, get_category, get_versions
@@ -25,12 +26,13 @@ class UploadController( BaseController )
replace_id = params.get( 'replace_id', None )
replace_version = None
uploaded_file = None
+ upload_type = params.get( 'upload_type', 'tool' )
categories = get_categories( trans )
if not categories:
return trans.response.send_redirect( web.url_for( controller='tool',
action='browse_tools',
cntrller='tool',
- message='No categories have been configured in this instance of the Galaxy Community. An administrator needs to create some via the Administrator control panel before anything can be uploaded',
+ message='No categories have been configured in this instance of the Galaxy Tool Shed. An administrator needs to create some via the Administrator control panel before anything can be uploaded',
status='error' ) )
if params.get( 'upload_button', False ):
url_paste = params.get( 'url', '' ).strip()
@@ -50,8 +52,6 @@ class UploadController( BaseController )
elif file_data not in ( '', None ):
uploaded_file = file_data.file
if uploaded_file:
- # TODO: tool should no longer be the default when we upload histories and workflows
- upload_type = params.get( 'upload_type', 'tool' )
datatype = trans.app.datatypes_registry.get_datatype_by_extension( upload_type )
if datatype is None:
message = 'An unknown file type was selected. This should not be possible, please report the error.'
@@ -62,6 +62,7 @@ class UploadController( BaseController )
meta = datatype.verify( uploaded_file )
meta.user = trans.user
meta.guid = trans.app.security.get_new_guid()
+ meta.suite = upload_type == 'toolsuite'
obj = datatype.create_model_object( meta )
trans.sa_session.add( obj )
if isinstance( obj, trans.app.model.Tool ):
@@ -71,25 +72,32 @@ class UploadController( BaseController )
if replace_id:
replace_version = trans.sa_session.query( trans.app.model.Tool ).get( trans.security.decode_id( replace_id ) )
if existing and not replace_id:
- raise UploadError( 'A tool with the same ID already exists. If you are trying to update this tool to a new version, please use the upload form on the "Edit Tool" page. Otherwise, please choose a new ID.' )
+ raise UploadError( 'A %s with the same Id already exists. If you are trying to update this %s to a new version, use the upload form on the "Edit Tool" page. Otherwise, change the Id in the %s config.' % \
+ ( obj.label, obj.label, obj.label ) )
elif replace_id and not existing:
- raise UploadError( 'Tool ids must match when uploading a new version of a tool. The new tool id does not match the old tool id (%s). Check the tool XML files.' % str( replace_version.tool_id ) )
+ raise UploadError( 'The new %s id (%s) does not match the old %s id (%s). Check the %s config files.' % \
+ ( obj.label, str( meta.id ), obj.label, str( replace_version.tool_id ), obj.label ) )
elif existing and replace_id:
if replace_version.newer_version:
# If the user has picked an old version, switch to the newest version
replace_version = get_versions( replace_version )[0]
if replace_version.tool_id != meta.id:
- raise UploadError( 'Tool ids must match when uploading a new version of a tool. The new tool id (%s) does not match the old tool id (%s). Check the tool XML files.' % ( str( meta.id ), str( replace_version.tool_id ) ) )
+ raise UploadError( 'The new %s id (%s) does not match the old %s id (%s). Check the %s config files.' % \
+ ( obj.label, str( meta.id ), obj.label, str( replace_version.tool_id ), obj.label ) )
for old_version in get_versions( replace_version ):
if old_version.version == meta.version:
- raise UploadError( 'The new version (%s) matches an old version. Check your version in the tool XML file.' % str( meta.version ) )
- if old_version.is_new():
- raise UploadError( 'There is an existing version of this tool which has not yet been submitted for approval, so either <a href="%s">submit or delete it</a> before uploading a new version.' % url_for( controller='common',
- action='view_tool',
- cntrller='tool',
- id=trans.security.encode_id( old_version.id ) ) )
- if old_version.is_waiting():
- raise UploadError( 'There is an existing version of this tool which is waiting for administrative approval, so contact an administrator for help.' )
+ raise UploadError( 'The new version (%s) matches an old version. Check your version in the %s config file.' % \
+ ( str( meta.version ), obj.label ) )
+ if old_version.is_new:
+ raise UploadError( 'There is an existing version of this %s which has not yet been submitted for approval, so either <a href="%s">submit it or delete it</a> before uploading a new version.' % \
+ ( obj.label,
+ url_for( controller='common',
+ action='view_tool',
+ cntrller='tool',
+ id=trans.security.encode_id( old_version.id ) ) ) )
+ if old_version.is_waiting:
+ raise UploadError( 'There is an existing version of this %s which is waiting for administrative approval, so contact an administrator for help.' % \
+ obj.label )
# Defer setting the id since the newer version id doesn't exist until the new Tool object is flushed
if category_ids:
for category_id in category_ids:
@@ -128,10 +136,10 @@ class UploadController( BaseController )
replace_version = trans.sa_session.query( trans.app.model.Tool ).get( int( trans.app.security.decode_id( replace_id ) ) )
old_version = None
for old_version in get_versions( replace_version ):
- if old_version.is_new():
+ if old_version.is_new:
message = 'There is an existing version of this tool which has not been submitted for approval, so either submit or delete it before uploading a new version.'
break
- if old_version.is_waiting():
+ if old_version.is_waiting:
message = 'There is an existing version of this tool which is waiting for administrative approval, so contact an administrator for help.'
break
else:
@@ -143,13 +151,23 @@ class UploadController( BaseController )
id=trans.app.security.encode_id( old_version.id ),
message=message,
status='error' ) )
- selected_upload_type = params.get( 'type', 'tool' )
selected_categories = [ trans.security.decode_id( id ) for id in category_ids ]
+ datatype_labels=trans.app.datatypes_registry.get_datatype_labels()
+ type_ids = [ tup[0] for tup in datatype_labels ]
+ upload_type_select_list = SelectField( 'upload_type',
+ refresh_on_change=True,
+ refresh_on_change_values=type_ids )
+ for type_id, type_label in datatype_labels:
+ if type_id == upload_type:
+ upload_type_select_list.add_option( type_label, type_id, selected = True )
+ else:
+ upload_type_select_list.add_option( type_label, type_id, selected = False )
return trans.fill_template( '/webapps/community/upload/upload.mako',
message=message,
status=status,
- selected_upload_type=selected_upload_type,
- upload_types=trans.app.datatypes_registry.get_datatypes_for_select_list(),
+ selected_upload_type=upload_type,
+ upload_type_select_list=upload_type_select_list,
+ datatype_labels=trans.app.datatypes_registry.get_datatype_labels(),
replace_id=replace_id,
selected_categories=selected_categories,
categories=get_categories( trans ) )
--- a/community_datatypes_conf.xml.sample
+++ b/community_datatypes_conf.xml.sample
@@ -2,8 +2,6 @@
<datatypes><registration><datatype extension="tool" type="galaxy.webapps.community.datatypes:Tool" model="galaxy.webapps.community.model:Tool"/>
+ <datatype extension="toolsuite" type="galaxy.webapps.community.datatypes:ToolSuite" model="galaxy.webapps.community.model:Tool"/></registration>
- <sniffers>
- <sniffer type="galaxy.webapps.community.datatypes:Tool"/>
- </sniffers></datatypes>
--- a/templates/webapps/community/upload/upload.mako
+++ b/templates/webapps/community/upload/upload.mako
@@ -9,16 +9,43 @@
%><%inherit file="${inherit(context)}"/>
-<%def name="title()">Upload</%def>
+<%def name="javascripts()">
+ ${parent.javascripts()}
+ <script type="text/javascript">
+ $( function() {
+ $( "select[refresh_on_change='true']").change( function() {
+ $( "#upload_form" ).submit();
+ });
+ });
+ </script>
+</%def>
-<h2>Upload</h2>
+<%def name="title()">
+ %if selected_upload_type == 'tool':
+ Upload a tool archive
+ %elif selected_upload_type == 'toolsuite':
+ Upload a tool suite archive
+ %endif
+</%def>
+
+<h2>
+ %if selected_upload_type == 'tool':
+ Upload a tool archive
+ %elif selected_upload_type == 'toolsuite':
+ Upload a tool suite archive
+ %endif
+</h2>
%if message:
${render_msg( message, status )}
%endif
<div class="toolForm">
- <div class="toolFormTitle">Upload</div>
+ %if selected_upload_type == 'tool':
+ <div class="toolFormTitle">Upload a single tool archive</div>
+ %else:
+ <div class="toolFormTitle">Upload a tool suite archive</div>
+ %endif
<div class="toolFormBody">
## TODO: nginx
<form id="upload_form" name="upload_form" action="${h.url_for( controller='upload', action='upload' )}" enctype="multipart/form-data" method="post">
@@ -28,18 +55,14 @@
<div class="form-row"><label>Upload Type</label><div class="form-row-input">
- <select name="upload_type">
- %for type_id, type_name in upload_types:
- %if type_id == selected_upload_type:
- <option value="${type_id}" selected>${type_name}</option>
- %else:
- <option value="${type_id}">${type_name}</option>
- %endif
- %endfor
- </select>
+ ${upload_type_select_list.get_html()}
</div><div class="toolParamHelp" style="clear: both;">
- Need help creating a tool file? See the <a href="${h.url_for( controller='tool', action='help' )}">Tool Help</a> page.
+ %if selected_upload_type == 'tool':
+ Need help creating a single tool archive? See details below.
+ %elif selected_upload_type == 'toolsuite':
+ Need help creating a tool suite archive? See details below.
+ %endif
</div><div style="clear: both"></div></div>
@@ -77,3 +100,151 @@
</form></div></div>
+<p/>
+<div class="toolFormTitle">Creating an archive containing a tool or a suite of tools</div>
+<p>
+ A tool or tool suite archive is a tar-format file (bzipped or gzipped tar are valid)
+ containing all the files necessary to load the tool(s) into a Galaxy instance.
+</p>
+%if selected_upload_type == 'toolsuite':
+ <h3>Tool Suite Archive</h3>
+ <p>
+ A tools suite must include a file named <code>suite_config.xml</code> which provides information about the id, name,
+ version and description of the tool suite, as well as the id, name, version and description of each tool
+ in the suite. Here is an example <code>suite_config.xml</code> file.
+ </p>
+ <p>
+<pre>
+ <suite id="lastz_toolsuite" name="Suite of Lastz tools" version="1.0.0">
+ <description>This suite contains all of my Lastz tools for Galaxy</description>
+ <tool id="lastz_wrapper_2" name="Lastz" version="1.1.0">
+ <description> map short reads against reference sequence</description>
+ </tool>
+ <tool id="lastz_paired_reads_wrapper" name="Lastz paired reads" version="1.0.0">
+ <description> map short paired reads against reference sequence</description>
+ </tool>
+ </suite>
+</pre>
+ </p>
+ </p>
+ <p>
+ New versions of the suite can be uploaded, replacing an older version of the suite, but the version attribute
+ of the <suite> tag must be altered the same way that the version attribute of a single tool config must be altered
+ if uploading a new version of a tool.
+ </p>
+ <p>
+ The id, name and version attributes of each <tool> tag in the <code>suite_config.xml</code> file must exactly match the same
+ attributes in each associated tool config in the archive or you will not be allowed to upload the archive.
+ </p>
+ <p>
+ In addition to the <code>suite_config.xml</code> file, the archive must include all
+ <a href="http://bitbucket.org/galaxy/galaxy-central/wiki/ToolConfigSyntax" target="_blank">tool config files</a>,
+ executables, functional test data (if your tool config includes functional tests) and other files needed for each
+ of the tools in your suite to function within Galaxy. See the information about single tool archives below for
+ additional hints to enable ease-of-use when others download your suite of tools.
+ </p>
+ <p>
+ For example, to package the above Lastz suite of tools:
+<pre>
+ user@host:~% tar jcvf ~/Desktop/galaxy_lastz_toolsuite.tar.bz2 lastzsuite
+ lastzsuite/
+ lastzsuite/README
+ lastzsuite/suite_config.xml
+ lastzsuite/lastz_paired_reads_wrapper.py
+ lastzsuite/lastz_paired_reads_wrapper.xml
+ lastzsuite/lastz_wrapper.py
+ lastzsuite/lastz_wrapper.xml
+ lastzsuite/lastz-distrib-1.02.00/
+ lastzsuite/lastz-distrib-1.02.00/src/
+ lastzsuite/lastz-distrib-1.02.00/src/Makefile
+ lastzsuite/lastz-distrib-1.02.00/src/version.mak
+ lastzsuite/lastz-distrib-1.02.00/src/lastz.c
+ lastzsuite/lastz-distrib-1.02.00/src/lastz.h
+ ...
+</pre>
+ ~/Desktop/galaxy_lastz_tool.tar.bz2 is now ready to be uploaded.
+ </p>
+%endif
+<h3>Single Tool Archive</h3>
+<p>
+ A single tool archive must include a
+ <a href="http://bitbucket.org/galaxy/galaxy-central/wiki/ToolConfigSyntax" target="_blank">tool config file</a>
+ and will probably also include a tool script. If any steps are necessary to install your tool beyond the basic
+ instructions below, include a README file to provide details. If the tool (or parts of it) are written in C,
+ the source code can be included (or put links to the source in the README). Do not include pre-compiled binaries
+ without source since Galaxy is run on a wide variety of platforms. Also, if you are only wrapping or providing a
+ Galaxy config for a tool that is not your own, be sure the license allows for redistribution before including any
+ part of that tool in the archive.
+</p>
+<p>
+ There are no requirements about the directory structure inside the archive, but for ease of use it's generally
+ a good idea to put everything inside a sub-directory, instead of directly at the top level.
+</p>
+<p>
+ For example, to package the Lastz tool's config file, Galaxy wrapper, and the C source:
+<pre>
+ user@host:~% tar jcvf ~/Desktop/galaxy_lastz_tool.tar.bz2 lastz
+ lastz/
+ lastz/README
+ lastz/lastz_wrapper.py
+ lastz/lastz_wrapper.xml
+ lastz/lastz-distrib-1.02.00/
+ lastz/lastz-distrib-1.02.00/src/
+ lastz/lastz-distrib-1.02.00/src/Makefile
+ lastz/lastz-distrib-1.02.00/src/version.mak
+ lastz/lastz-distrib-1.02.00/src/lastz.c
+ lastz/lastz-distrib-1.02.00/src/lastz.h
+ ...
+</pre>
+ ~/Desktop/galaxy_lastz_tool.tar.bz2 is now ready to be uploaded.
+</p>
+<h3>Editing Information, Categories, and Submitting For Approval</h3>
+<p>
+ Simply uploading a tool to the Galaxy too shed will not allow other users to find and download your tool. It will
+ need to be approved by an administrator before it appears in the tool list.
+</p>
+<p>
+ After your archive has successfully uploaded, you will be redirected to the Edit Tool page. Provide a detailed
+ description of what the tool does - this will be used by administrators to understand the tool before approving it
+ for display on the site. Once approved, this information will be displayed to users who view your tool. In addition,
+ the site administrators will have configured a number of categories with which you can associate your tool to make it
+ easy to find by users looking to solve specific problems. Associate as many categories as are relevant to your tool.
+ You may change the description and associated categories as often as you'd like until you click the "<strong>Submit for
+ approval</strong>" button. Once submitted, the tool will be approved or rejected by an administrator. If the tool is
+ rejected, you will see information about why it was rejected, and you can make appropriate changes to the archive and
+ re-submit it for approval. When it is approved, your archive will be visible to everyone. At that point, the description
+ and associated categories can only be changed by an administrator.
+</p>
+<p>
+ When the tool has been approved or rejected, you may upload a new version by browsing to the tool's "View Tool" page,
+ clicking the "Tool actions" menu in the upper right corner of the page, and selecting "Upload a new version" from the
+ menu.
+</p>
+<hr/>
+<h3>Downloading and Installing Tools</h3>
+<p>
+ A tool's download link will send you the tool archive. Once downloaded, unpack the tool on your local Galaxy instance's server:
+<pre>
+ user@host:~% tar xvf galaxy_lastz_tool.tar
+ ...
+ user@host:~% tar zxvf galaxy_lastz_tool.tar.gz
+ ...
+ user@host:~% tar jxvf galaxy_lastz_tool.tar.bz2
+ ...
+</pre>
+ If the archive includes a README file, consult it for installation instructions. If not, follow these basic steps:
+ <ol>
+ <li>Create a directory under <code>galaxy_dist/tools/</code> to house downloaded tool(s).</li>
+ <li>In the new directory, place the XML and any script file(s) which were contained in the archive.</li>
+ <li>
+ If the tool includes binaries, you'll need to copy them to a directory on your <code>$PATH</code>. If the tool depends on
+ C binaries but does not come with them (only source), you'll need to compile the source first.
+ </li>
+ <li>Add the tool to <code>galaxy_dist/tool_conf.xml</code>.</li>
+ <li>Restart your Galaxy server process.</li>
+ </ol>
+</p>
+<p>
+ In the near future, we plan to implement a more direct method to install tools via the Galaxy administrator user interface instead
+ of placing files on the filesystem and manually managing the <code>tool_conf.xml</code> file.
+</p>
--- a/templates/webapps/community/admin/index.mako
+++ b/templates/webapps/community/admin/index.mako
@@ -1,7 +1,7 @@
<%inherit file="/webapps/community/base_panels.mako"/>
## Default title
-<%def name="title()">Galaxy Community Space Administration</%def>
+<%def name="title()">Galaxy Tool Shed Administration</%def><%def name="stylesheets()">
${parent.stylesheets()}
--- a/templates/webapps/community/tool/help.mako
+++ b/templates/webapps/community/tool/help.mako
@@ -17,35 +17,87 @@
${render_msg( message, status )}
%endif
-<h3>Uploading Tools</h3>
-
-<p><strong>Tool File Format</strong></p>
+<p/>
+<div class="toolFormTitle">Creating an archive containing a tool or a suite of tools</div><p>
-
- A tool file is a tar-format (bzipped or gzipped tar are valid) archive
- containing all the files necessary to load the tool in Galaxy. At the very
- least, it must contain a
- <a href="http://bitbucket.org/galaxy/galaxy-central/wiki/ToolConfigSyntax" target="_blank">Tool XML File</a>,
- and will probably also include a tool script. If any steps are necessary
- to install your tool beyond the basic instructions below, please include a
- README file which details these steps. If the tool (or parts of it) are
- written in C, the source code can be included (or put links to the source
- in the README). Please do not include precompiled binaries without source,
- since Galaxy is run on a wide variety of platforms. Also, if you are only
- wrapping or providing a Galaxy config for a tool that is not your own,
- please be sure the license allows for redistribution before including any
- part of that tool in the tar archive!
+ A tool or tool suite archive is a tar-format file (bzipped or gzipped tar are valid)
+ containing all the files necessary to load the tool(s) into a Galaxy instance.
+</p>
+<h3>Tool Suite Archive</h3>
+<p>
+ A tools suite must include a file named <code>suite_config.xml</code> which provides information about the id, name,
+ version and description of the tool suite, as well as the id, name, version and description of each tool
+ in the suite. Here is an example <code>suite_config.xml</code> file.
</p><p>
- There are no requirements about the directory structure inside the tar
- archive, but for ease of use, it's generally a good idea to put everything
- inside of a subdirectory, instead of directly at the top level.
+<pre>
+ <suite id="lastz_toolsuite" name="Suite of Lastz tools" version="1.0.0">
+ <description>This suite contains all of my Lastz tools for Galaxy</description>
+ <tool id="lastz_wrapper_2" name="Lastz" version="1.1.0">
+ <description> map short reads against reference sequence</description>
+ </tool>
+ <tool id="lastz_paired_reads_wrapper" name="Lastz paired reads" version="1.0.0">
+ <description> map short paired reads against reference sequence</description>
+ </tool>
+ </suite>
+</pre></p>
-
-<p><strong>Tool File Example</strong></p>
+</p><p>
- To package up the LASTZ tool's config file, Galaxy wrapper, and the C source:
- <pre>
+ New versions of the suite can be uploaded, replacing an older version of the suite, but the version attribute
+ of the <suite> tag must be altered the same way that the version attribute of a single tool config must be altered
+ if uploading a new version of a tool.
+</p>
+<p>
+ The id, name and version attributes of each <tool> tag in the <code>suite_config.xml</code> file must exactly match the same
+ attributes in each associated tool config in the archive or you will not be allowed to upload the archive.
+</p>
+<p>
+ In addition to the <code>suite_config.xml</code> file, the archive must include all
+ <a href="http://bitbucket.org/galaxy/galaxy-central/wiki/ToolConfigSyntax" target="_blank">tool config files</a>,
+ executables, functional test data (if your tool config includes functional tests) and other files needed for each
+ of the tools in your suite to function within Galaxy. See the information about single tool archives below for
+ additional hints to enable ease-of-use when others download your suite of tools.
+</p>
+<p>
+ For example, to package the above Lastz suite of tools:
+<pre>
+ user@host:~% tar jcvf ~/Desktop/galaxy_lastz_toolsuite.tar.bz2 lastzsuite
+ lastzsuite/
+ lastzsuite/README
+ lastzsuite/suite_config.xml
+ lastzsuite/lastz_paired_reads_wrapper.py
+ lastzsuite/lastz_paired_reads_wrapper.xml
+ lastzsuite/lastz_wrapper.py
+ lastzsuite/lastz_wrapper.xml
+ lastzsuite/lastz-distrib-1.02.00/
+ lastzsuite/lastz-distrib-1.02.00/src/
+ lastzsuite/lastz-distrib-1.02.00/src/Makefile
+ lastzsuite/lastz-distrib-1.02.00/src/version.mak
+ lastzsuite/lastz-distrib-1.02.00/src/lastz.c
+ lastzsuite/lastz-distrib-1.02.00/src/lastz.h
+ ...
+</pre>
+ ~/Desktop/galaxy_lastz_tool.tar.bz2 is now ready to be uploaded.
+</p>
+<h3>Single Tool Archive</h3>
+<p>
+ A single tool archive must include a
+ <a href="http://bitbucket.org/galaxy/galaxy-central/wiki/ToolConfigSyntax" target="_blank">tool config file</a>
+ and will probably also include a tool script. If any steps are necessary to install your tool beyond the basic
+ instructions below, include a README file to provide details. If the tool (or parts of it) are written in C,
+ the source code can be included (or put links to the source in the README). Do not include pre-compiled binaries
+ without source since Galaxy is run on a wide variety of platforms. Also, if you are only wrapping or providing a
+ Galaxy config for a tool that is not your own, be sure the license allows for redistribution before including any
+ part of that tool in the archive.
+</p>
+<p>
+ There are no requirements about the directory structure inside the archive, but for ease of use it's generally
+ a good idea to put everything inside a sub-directory, instead of directly at the top level.
+</p>
+<p>
+ For example, to package the Lastz tool's config file, Galaxy wrapper, and the C source:
+<pre>
user@host:~% tar jcvf ~/Desktop/galaxy_lastz_tool.tar.bz2 lastz
lastz/
lastz/README
@@ -58,68 +110,57 @@
lastz/lastz-distrib-1.02.00/src/lastz.c
lastz/lastz-distrib-1.02.00/src/lastz.h
...
- </pre>
- <code>~/Desktop/galaxy_lastz_tool.tar.bz2</code> is now ready to be uploaded.
+</pre>
+ ~/Desktop/galaxy_lastz_tool.tar.bz2 is now ready to be uploaded.
+</p>
+<h3>Editing Information, Categories, and Submitting For Approval</h3>
+<p>
+ Simply uploading a tool to the Galaxy too shed will not allow other users to find and download your tool. It will
+ need to be approved by an administrator before it appears in the tool list.
+</p>
+<p>
+ After your archive has successfully uploaded, you will be redirected to the Edit Tool page. Provide a detailed
+ description of what the tool does - this will be used by administrators to understand the tool before approving it
+ for display on the site. Once approved, this information will be displayed to users who view your tool. In addition,
+ the site administrators will have configured a number of categories with which you can associate your tool to make it
+ easy to find by users looking to solve specific problems. Associate as many categories as are relevant to your tool.
+ You may change the description and associated categories as often as you'd like until you click the "<strong>Submit for
+ approval</strong>" button. Once submitted, the tool will be approved or rejected by an administrator. If the tool is
+ rejected, you will see information about why it was rejected, and you can make appropriate changes to the archive and
+ re-submit it for approval. When it is approved, your archive will be visible to everyone. At that point, the description
+ and associated categories can only be changed by an administrator.
+</p>
+<p>
+ When the tool has been approved or rejected, you may upload a new version by browsing to the tool's "View Tool" page,
+ clicking the "Tool actions" menu in the upper right corner of the page, and selecting "Upload a new version" from the
+ menu.
+</p>
+<hr/>
+<h3>Downloading and Installing Tools</h3>
+<p>
+ A tool's download link will send you the tool archive. Once downloaded, unpack the tool on your local Galaxy instance's server:
+<pre>
+ user@host:~% tar xvf galaxy_lastz_tool.tar
+ ...
+ user@host:~% tar zxvf galaxy_lastz_tool.tar.gz
+ ...
+ user@host:~% tar jxvf galaxy_lastz_tool.tar.bz2
+ ...
+</pre>
+ If the archive includes a README file, consult it for installation instructions. If not, follow these basic steps:
+ <ol>
+ <li>Create a directory under <code>galaxy_dist/tools/</code> to house downloaded tool(s).</li>
+ <li>In the new directory, place the XML and any script file(s) which were contained in the archive.</li>
+ <li>
+ If the tool includes binaries, you'll need to copy them to a directory on your <code>$PATH</code>. If the tool depends on
+ C binaries but does not come with them (only source), you'll need to compile the source first.
+ </li>
+ <li>Add the tool to <code>galaxy_dist/tool_conf.xml</code>.</li>
+ <li>Restart your Galaxy server process.</li>
+ </ol>
+</p>
+<p>
+ In the near future, we plan to implement a more direct method to install tools via the Galaxy administrator user interface instead
+ of placing files on the filesystem and manually managing the <code>tool_conf.xml</code> file.
</p>
-<p><strong>Editing Information, Categories, and Submitting For Approval</strong></p>
-
-<p>
- Simply uploading a tool to the Community will not allow other users to find
- and download your tool. It will need to be approved by an administrator
- before it appears in the tool list.
-</p>
-<p>
- After the tool has successfully uploaded, you will be redirected to the
- Edit Tool page. Please provide a detailed description of what the tool
- does - this will be used by administrators to understand the tool before
- approving it for display on the site. Once approved, this information will
- be displayed to users who view your tool. In addition, the site
- administrators will have configured a number of categories with which you
- can associate your tool to make it easily findable by users looking to
- solve specific problems. Please associate as many categories as are
- relevant to your tool. You may change the description and associated
- categories as often as you'd like until you click the "<strong>Submit for
- approval</strong>" button. Once submitted, the tool will be approved or
- rejected by an administrator. Once approved, it will be visible to
- everyone. At that point, the description and associated categories can
- only be changed by an administrator.
-</p>
-<p>
- Once the tool has been approved or rejected, you may upload a new version
- by browsing to the tool's "View Tool" page, clicking the context menu to
- the right of the tool's name, and selecting "Upload a new version."
-</p>
-
-<hr/>
-
-<h3>Downloading and Installing Tools</h3>
-
-<p>
- A tool's download link will send you the tool tar archive. Once
- downloaded, unpack the tool on your local Galaxy instance's server:
- <pre>
- user@host:~% tar xvf galaxy_tool.tar
- ...
- user@host:~% tar zxvf galaxy_tool.tar.gz
- ...
- user@host:~% tar jxvf galaxy_tool.tar.bz2
- ...
- </pre>
- If the tar archive includes a README file, consult it for installation
- instructions. If not, follow these basic steps:
- <ol>
- <li>Create a directory under <code>galaxy_dist/tools/</code> to house downloaded tool(s).</li>
- <li>In the new directory, place the XML and any script file(s) which were contained in the tar archive.</li>
- <li>If the tool includes binaries, you'll need to copy them to a directory on your <code>$PATH</code>. If the tool depends on C binaries but does not come with them (only source), you'll need to compile the source first.</li>
- <li>Add the tool to <code>galaxy_dist/tool_conf.xml</code>.</li>
- <li>Restart the Galaxy server process.</li>
- </ol>
-</p>
-
-<p>
- We plan to implement a more direct method to install tools via the Galaxy
- administrator user interface instead of placing files on the filesystem and
- managing the <code>tool_conf.xml</code> file by hand. In the meantime,
- this is the process.
-</p>
--- a/lib/galaxy/webapps/community/model/__init__.py
+++ b/lib/galaxy/webapps/community/model/__init__.py
@@ -1,5 +1,5 @@
"""
-Galaxy Community Space data model classes
+Galaxy Tool Shed data model classes
Naming: try to use class names that have a distinct plural form so that
the relationship cardinalities are obvious (e.g. prefer Dataset to Data)
@@ -95,7 +95,7 @@ class Tool( object ):
REJECTED = 'rejected',
ARCHIVED = 'archived' )
def __init__( self, guid=None, tool_id=None, name=None, description=None, user_description=None,
- category=None, version=None, user_id=None, external_filename=None ):
+ category=None, version=None, user_id=None, external_filename=None, suite=False ):
self.guid = guid
self.tool_id = tool_id
self.name = name or "Unnamed tool"
@@ -106,6 +106,7 @@ class Tool( object ):
self.external_filename = external_filename
self.deleted = False
self.__extension = None
+ self.suite = suite
def get_file_name( self ):
if not self.external_filename:
assert self.id is not None, "ID must be set before filename used (commit the object)"
@@ -133,6 +134,8 @@ class Tool( object ):
self.description = datatype_bunch.description
self.version = datatype_bunch.version
self.user_id = datatype_bunch.user.id
+ self.suite = datatype_bunch.suite
+ @property
def state( self ):
if self.events:
# Sort the events in ascending order by update_time
@@ -150,35 +153,47 @@ class Tool( object ):
else:
return ''
return 'No comment'
+ # Tool states
+ @property
def is_new( self ):
- return self.state() == self.states.NEW
+ return self.state == self.states.NEW
+ @property
def is_error( self ):
- return self.state() == self.states.ERROR
+ return self.state == self.states.ERROR
+ @property
def is_deleted( self ):
- return self.state() == self.states.DELETED
+ return self.state == self.states.DELETED
+ @property
def is_waiting( self ):
- return self.state() == self.states.WAITING
+ return self.state == self.states.WAITING
+ @property
def is_approved( self ):
- return self.state() == self.states.APPROVED
+ return self.state == self.states.APPROVED
+ @property
def is_rejected( self ):
- return self.state() == self.states.REJECTED
+ return self.state == self.states.REJECTED
+ @property
def is_archived( self ):
- return self.state() == self.states.ARCHIVED
+ return self.state == self.states.ARCHIVED
def get_state_message( self ):
- if self.is_new():
- return '<font color="red"><b><i>This is an unsubmitted version of this tool</i></b></font>'
- if self.is_error():
- return '<font color="red"><b><i>This tool is in an error state</i></b></font>'
- if self.is_deleted():
- return '<font color="red"><b><i>This is a deleted version of this tool</i></b></font>'
- if self.is_waiting():
- return '<font color="red"><b><i>This version of this tool is awaiting administrative approval</i></b></font>'
- if self.is_approved():
- return '<b><i>This is the latest approved version of this tool</i></b>'
- if self.is_rejected():
- return '<font color="red"><b><i>This version of this tool has been rejected by an administrator</i></b></font>'
- if self.is_archived():
- return '<font color="red"><b><i>This is an archived version of this tool</i></b></font>'
+ if self.is_suite:
+ label = 'tool suite'
+ else:
+ label = 'tool'
+ if self.is_new:
+ return '<font color="red"><b><i>This is an unsubmitted version of this %s</i></b></font>' % label
+ if self.is_error:
+ return '<font color="red"><b><i>This %s is in an error state</i></b></font>' % label
+ if self.is_deleted:
+ return '<font color="red"><b><i>This is a deleted version of this %s</i></b></font>' % label
+ if self.is_waiting:
+ return '<font color="red"><b><i>This version of this %s is awaiting administrative approval</i></b></font>' % label
+ if self.is_approved:
+ return '<b><i>This is the latest approved version of this %s</i></b>' % label
+ if self.is_rejected:
+ return '<font color="red"><b><i>This version of this %s has been rejected by an administrator</i></b></font>' % label
+ if self.is_archived:
+ return '<font color="red"><b><i>This is an archived version of this %s</i></b></font>' % label
@property
def extension( self ):
# if instantiated via a query, this unmapped property won't exist
@@ -202,6 +217,15 @@ class Tool( object ):
self.__extension = 'tar'
return self.__extension
@property
+ def is_suite( self ):
+ return self.suite
+ @property
+ def label( self ):
+ if self.is_suite:
+ return 'tool suite'
+ else:
+ return 'tool'
+ @property
def download_file_name( self ):
return '%s_%s.%s' % ( self.tool_id, self.version, self.extension )
@property
--- a/templates/webapps/community/base_panels.mako
+++ b/templates/webapps/community/base_panels.mako
@@ -1,7 +1,7 @@
<%inherit file="/base_panels.mako"/>
## Default title
-<%def name="title()">Galaxy Community Space</%def>
+<%def name="title()">Galaxy Tool Shed</%def>
## Masthead
<%def name="masthead()">
@@ -94,7 +94,7 @@
<div class="title" style="position: absolute; top: 0; left: 0;"><a href="${app.config.get( 'logo_url', '/' )}"><img border="0" src="${h.url_for('/static/images/galaxyIcon_noText.png')}" style="width: 26px; vertical-align: top;">
- Galaxy Community
+ Galaxy Tool Shed
%if app.config.brand:
<span class='brand'>/ ${app.config.brand}</span>
%endif
--- a/lib/galaxy/webapps/community/model/mapping.py
+++ b/lib/galaxy/webapps/community/model/mapping.py
@@ -113,7 +113,8 @@ Tool.table = Table( "tool", metadata,
Column( "version", TrimmedString( 255 ) ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "external_filename" , TEXT ),
- Column( "deleted", Boolean, index=True, default=False ) )
+ Column( "deleted", Boolean, index=True, default=False ),
+ Column( "suite", Boolean, default=False, index=True ) )
Category.table = Table( "category", metadata,
Column( "id", Integer, primary_key=True ),
--- a/lib/galaxy/webapps/community/controllers/tool.py
+++ b/lib/galaxy/webapps/community/controllers/tool.py
@@ -15,7 +15,7 @@ class ApprovedToolListGrid( ToolListGrid
class MyToolsListGrid( ToolListGrid ):
class StateColumn( grids.TextColumn ):
def get_value( self, trans, grid, tool ):
- state = tool.state()
+ state = tool.state
if state == 'approved':
state_color = 'ok'
elif state == 'rejected':
@@ -64,7 +64,7 @@ class ToolCategoryListGrid( CategoryList
viewable_tools = 0
for tca in category.tools:
tool = tca.tool
- if tool.is_approved():
+ if tool.is_approved:
viewable_tools += 1
return viewable_tools
return 0
--- a/lib/galaxy/webapps/community/datatypes/__init__.py
+++ b/lib/galaxy/webapps/community/datatypes/__init__.py
@@ -17,7 +17,6 @@ class DatatypeVerificationError( Excepti
class Registry( object ):
def __init__( self, root_dir=None, config=None ):
self.datatypes_by_extension = {}
- self.sniff_order = []
if root_dir and config:
# Parse datatypes_conf.xml
tree = galaxy.util.parse_xml( config )
@@ -50,98 +49,143 @@ class Registry( object ):
log.debug( 'Added model class: %s to datatype: %s' % ( model_class, dtype ) )
except Exception, e:
log.warning( 'Error loading datatype "%s", problem: %s' % ( extension, str( e ) ) )
- # Load datatype sniffers from the config
- sniff_order = []
- sniffers = root.find( 'sniffers' )
- for elem in sniffers.findall( 'sniffer' ):
- dtype = elem.get( 'type', None )
- if dtype:
- sniff_order.append( dtype )
- for dtype in sniff_order:
- try:
- fields = dtype.split( ":" )
- datatype_module = fields[0]
- datatype_class = fields[1]
- fields = datatype_module.split( "." )
- module = __import__( fields.pop(0) )
- for mod in fields:
- module = getattr( module, mod )
- aclass = getattr( module, datatype_class )()
- included = False
- for atype in self.sniff_order:
- if not issubclass( atype.__class__, aclass.__class__ ) and isinstance( atype, aclass.__class__ ):
- included = True
- break
- if not included:
- self.sniff_order.append( aclass )
- log.debug( 'Loaded sniffer for datatype: %s' % dtype )
- except Exception, exc:
- log.warning( 'Error appending datatype %s to sniff_order, problem: %s' % ( dtype, str( exc ) ) )
def get_datatype_by_extension( self, ext ):
return self.datatypes_by_extension.get( ext, None )
- def get_datatypes_for_select_list( self ):
+ def get_datatype_labels( self ):
rval = []
for ext, datatype in self.datatypes_by_extension.items():
- rval.append( ( ext, datatype.select_name ) )
+ rval.append( ( ext, datatype.label ) )
return rval
- def sniff( self, fname ):
- for datatype in sniff_order:
- try:
- datatype.sniff( fname )
- return datatype.file_ext
- except:
- pass
class Tool( object ):
- select_name = 'Tool'
def __init__( self, model_object=None ):
self.model_object = model_object
- def verify( self, file ):
- msg = ''
+ self.label = 'Tool'
+ def verify( self, f, xml_files=[], tool_tags={} ):
+ # xml_files and tool_tags will only be received if we're called from the ToolSuite.verify() method.
try:
- tar = tarfile.open( file.name )
+ tar = tarfile.open( f.name )
except tarfile.ReadError:
- raise DatatypeVerificationError( 'The tool file is not a readable tar file' )
- xml_names = filter( lambda x: x.lower().endswith( '.xml' ), tar.getnames() )
- if not xml_names:
- raise DatatypeVerificationError( 'The tool file does not contain an XML file' )
- for xml_name in xml_names:
+ raise DatatypeVerificationError( 'The archive is not a readable tar file.' )
+ if not xml_files:
+ # Make sure we're not uploading a tool suite
+ if filter( lambda x: x.lower() == 'tool_suite.xml', tar.getnames() ):
+ raise DatatypeVerificationError( 'The archive includes a tool_suite.xml file, so set the upload type to "Tool Suite".' )
+ xml_files = filter( lambda x: x.lower().endswith( '.xml' ), tar.getnames() )
+ if not xml_files:
+ raise DatatypeVerificationError( 'The archive does not contain any xml config files.' )
+ for xml_file in xml_files:
try:
- tree = ElementTree.parse( tar.extractfile( xml_name ) )
+ tree = ElementTree.parse( tar.extractfile( xml_file ) )
root = tree.getroot()
except:
log.exception( 'fail:' )
continue
if root.tag == 'tool':
- rval = Bunch()
- try:
- rval.id = root.attrib['id']
- rval.name = root.attrib['name']
- rval.version = root.attrib['version']
- except KeyError, e:
- raise DatatypeVerificationError( 'Tool XML file does not conform to the specification. Missing required <tool> tag attribute: %s' % e )
- rval.description = ''
- desc_tag = root.find( 'description' )
- if desc_tag is not None:
- description = desc_tag.text
- if description:
- rval.description = description.strip()
- rval.message = 'Tool: %s %s, Version: %s, ID: %s' % ( str( rval.name ), str( rval.description ), str( rval.version ), str( rval.id ) )
- return rval
- else:
- raise DatatypeVerificationError( 'Unable to find a properly formatted tool XML file' )
+ if tool_tags:
+ # We are verifying the tools inside a tool suite, so the current tag should have been found in the suite_config.xml
+ # file parsed in the ToolSuite verify() method. The tool_tags dictionary should include a key matching the current
+ # tool Id, and a tuple value matching the tool name and version.
+ if root.attrib[ 'id' ] not in tool_tags:
+ raise DatatypeVerificationError( 'Tool Id (%s) is not included in the suite_config.xml file.' % \
+ ( str( root.attrib[ 'id' ] ) ) )
+ tup = tool_tags[ root.attrib[ 'id' ] ]
+ if root.attrib[ 'name' ] != tup[ 0 ]:
+ raise DatatypeVerificationError( 'Tool name (%s) differs between suite_config.xml and the tool config file for tool Id (%s).' % \
+ ( str( root.attrib[ 'name' ] ), str( root.attrib[ 'id' ] ) ) )
+ if root.attrib[ 'version' ] != tup[ 1 ]:
+ raise DatatypeVerificationError( 'Tool version (%s) differs between suite_config.xml and the tool config file for tool Id (%s).' % \
+ ( str( root.attrib[ 'version' ] ), str( root.attrib[ 'id' ] ) ) )
+ else:
+ # We are not verifying a tool suite, so we'll create a bunch for returning to the caller.
+ tool_bunch = Bunch()
+ try:
+ tool_bunch.id = root.attrib['id']
+ tool_bunch.name = root.attrib['name']
+ tool_bunch.version = root.attrib['version']
+ except KeyError, e:
+ raise DatatypeVerificationError( 'Tool XML file does not conform to the specification. Missing required <tool> tag attribute: %s' % str( e ) )
+ tool_bunch.description = ''
+ desc_tag = root.find( 'description' )
+ if desc_tag is not None:
+ description = desc_tag.text
+ if description:
+ tool_bunch.description = description.strip()
+ tool_bunch.message = 'Tool: %s %s, Version: %s, Id: %s' % \
+ ( str( tool_bunch.name ), str( tool_bunch.description ), str( tool_bunch.version ), str( tool_bunch.id ) )
+ return tool_bunch
+ else:
+ # TODO: should we verify files that are not tool configs?
+ log.debug( "The file named (%s) is not a tool config, so skipping verification." % str( xml_file ) )
def create_model_object( self, datatype_bunch ):
if self.model_object is None:
- raise Exception( 'No model object configured for %s, please check the datatype configuration file' % self.__class__.__name__ )
+ raise Exception( 'No model object configured for %s, check the datatype configuration file' % self.__class__.__name__ )
if datatype_bunch is None:
# TODO: do it automatically
raise Exception( 'Unable to create %s model object without passing in data' % self.__class__.__name__ )
o = self.model_object()
o.create_from_datatype( datatype_bunch )
return o
- def sniff( self, fname ):
+
+class ToolSuite( Tool ):
+ def __init__( self, model_object=None ):
+ self.model_object = model_object
+ self.label = 'Tool Suite'
+ def verify( self, f ):
+ """
+ A sample tool suite config:
+ <suite id="onto_toolkit" name="ONTO Toolkit" version="1.0">
+ <description>ONTO-Toolkit is a collection of Galaxy tools which support the manipulation of bio-ontologies.</description>
+ <tool id="get_ancestor_terms" name="Get the ancestor terms of a given OBO term" version="1.0.0">
+ <description>Collects the ancestor terms from a given term in the given OBO ontology</description>
+ </tool>
+ <tool id="get_child_terms" name="Get the child terms of a given OBO term" version="1.0.0">
+ <description>Collects the child terms from a given term in the given OBO ontology</description>
+ </tool>
+ </suite>
+ """
try:
- self.verify( open( fname, 'r' ) )
- return True
+ tar = tarfile.open( f.name )
+ except tarfile.ReadError:
+ raise DatatypeVerificationError( 'The archive is not a readable tar file.' )
+ suite_config = filter( lambda x: x.lower() == 'tool_suite.xml', tar.getnames() )
+ if not suite_config:
+ raise DatatypeVerificationError( 'The archive does not contain the required tool_suite.xml config file. If you are uploading a single tool archive, set the upload type to "Tool".' )
+ suite_config = suite_config[ 0 ]
+ # Parse and verify suite_config
+ archive_ok = False
+ try:
+ tree = ElementTree.parse( tar.extractfile( suite_config ) )
+ root = tree.getroot()
+ archive_ok = True
except:
- return False
+ log.exception( 'fail:' )
+ if archive_ok and root.tag == 'suite':
+ suite_bunch = Bunch()
+ try:
+ suite_bunch.id = root.attrib['id']
+ suite_bunch.name = root.attrib['name']
+ suite_bunch.version = root.attrib['version']
+ except KeyError, e:
+ raise DatatypeVerificationError( 'The file named tool-suite.xml does not conform to the specification. Missing required <suite> tag attribute: %s' % str( e ) )
+ suite_bunch.description = ''
+ desc_tag = root.find( 'description' )
+ if desc_tag is not None:
+ description = desc_tag.text
+ if description:
+ suite_bunch.description = description.strip()
+ suite_bunch.message = 'Tool suite: %s %s, Version: %s, Id: %s' % \
+ ( str( suite_bunch.name ), str( suite_bunch.description ), str( suite_bunch.version ), str( suite_bunch.id ) )
+ # Create a dictionary of the tools in the suite where the keys are tool_ids and the
+ # values are tuples of tool name and version
+ tool_tags = {}
+ for elem in root.findall( 'tool' ):
+ tool_tags[ elem.attrib['id'] ] = ( elem.attrib['name'], elem.attrib['version'] )
+ else:
+ raise DatatypeVerificationError( "The file named %s is not a valid tool suite config." % str( suite_config ) )
+ # Verify all included tool config files
+ xml_files = filter( lambda x: x.lower().endswith( '.xml' ) and x.lower() != 'tool_suite.xml', tar.getnames() )
+ if not xml_files:
+ raise DatatypeVerificationError( 'The archive does not contain any tool config (xml) files.' )
+ Tool.verify( self, f, xml_files=xml_files, tool_tags=tool_tags )
+ return suite_bunch
--- a/lib/galaxy/webapps/community/controllers/admin.py
+++ b/lib/galaxy/webapps/community/controllers/admin.py
@@ -299,7 +299,7 @@ class GroupListGrid( grids.Grid ):
class AdminToolListGrid( ToolListGrid ):
class StateColumn( grids.TextColumn ):
def get_value( self, trans, grid, tool ):
- state = tool.state()
+ state = tool.state
if state == 'approved':
state_color = 'ok'
elif state == 'rejected':
@@ -441,7 +441,7 @@ class AdminController( BaseController, A
# Called from the ToolStateColumn link
tool_id = kwd.get( 'id', None )
tool = get_tool( trans, tool_id )
- kwd[ 'f-state' ] = tool.state()
+ kwd[ 'f-state' ] = tool.state
elif operation == "tools_by_category":
# Eliminate the current filters if any exist.
for k, v in kwd.items():
@@ -547,7 +547,7 @@ class AdminController( BaseController, A
# If we're approving a tool, all previously approved versions must be set to archived
for version in get_versions( tool ):
# TODO: get latest approved version instead of all versions
- if version != tool and version.is_approved():
+ if version != tool and version.is_approved:
# Create an event with state ARCHIVED for the previously approved version of this tool
self.__create_tool_event( trans,
version,
@@ -799,30 +799,30 @@ def get_tools_by_state( trans, state ):
ids = []
if state == trans.model.Tool.states.NEW:
for tool in get_tools( trans ):
- if tool.is_new():
+ if tool.is_new:
ids.append( tool.id )
elif state == trans.model.Tool.states.ERROR:
for tool in get_tools( trans ):
- if tool.is_error():
+ if tool.is_error:
ids.append( tool.id )
elif state == trans.model.Tool.states.DELETED:
for tool in get_tools( trans ):
- if tool.is_deleted():
+ if tool.is_deleted:
ids.append( tool.id )
elif state == trans.model.Tool.states.WAITING:
for tool in get_tools( trans ):
- if tool.is_waiting():
+ if tool.is_waiting:
ids.append( tool.id )
elif state == trans.model.Tool.states.APPROVED:
for tool in get_tools( trans ):
- if tool.is_approved():
+ if tool.is_approved:
ids.append( tool.id )
elif state == trans.model.Tool.states.REJECTED:
for tool in get_tools( trans ):
- if tool.is_rejected():
+ if tool.is_rejected:
ids.append( tool.id )
elif state == trans.model.Tool.states.ARCHIVED:
for tool in get_tools( trans ):
- if tool.is_archived():
+ if tool.is_archived:
ids.append( tool.id )
return ids
# HG changeset patch -- Bitbucket.org
# Project galaxy-dist
# URL http://bitbucket.org/galaxy/galaxy-dist/overview
# User Nate Coraor <nate(a)bx.psu.edu>
# Date 1279818642 14400
# Node ID db1ae0bc8995cb30deef73ce9d14ef175c512f90
# Parent 79d38503db4d860f9c9d62560c156fb0a2122b1b
Attempt to reduce costly queries in the job runner, especially when tracking jobs in the database.
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -18,7 +18,7 @@ from Queue import Queue, Empty
log = logging.getLogger( __name__ )
# States for running a job. These are NOT the same as data states
-JOB_WAIT, JOB_ERROR, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_OK, JOB_READY, JOB_DELETED, JOB_ADMIN_DELETED = 'wait', 'error', 'input_error', 'input_deleted', 'ok', 'ready', 'deleted', 'admin_deleted'
+JOB_WAIT, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_READY, JOB_DELETED, JOB_ADMIN_DELETED = 'wait', 'input_error', 'input_deleted', 'ready', 'deleted', 'admin_deleted'
# This file, if created in the job's working directory, will be used for
# setting advanced metadata properties on the job and its associated outputs.
@@ -106,14 +106,14 @@ class JobQueue( object ):
for job in self.sa_session.query( model.Job ).filter( model.Job.state == model.Job.states.NEW ):
if job.tool_id not in self.app.toolbox.tools_by_id:
log.warning( "Tool '%s' removed from tool config, unable to recover job: %s" % ( job.tool_id, job.id ) )
- JobWrapper( job, None, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator, or' )
+ JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator, or' )
else:
log.debug( "no runner: %s is still in new state, adding to the jobs queue" %job.id )
self.queue.put( ( job.id, job.tool_id ) )
for job in self.sa_session.query( model.Job ).filter( ( model.Job.state == model.Job.states.RUNNING ) | ( model.Job.state == model.Job.states.QUEUED ) ):
if job.tool_id not in self.app.toolbox.tools_by_id:
log.warning( "Tool '%s' removed from tool config, unable to recover job: %s" % ( job.tool_id, job.id ) )
- JobWrapper( job, None, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator, or' )
+ JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator, or' )
elif job.job_runner_name is None:
log.debug( "no runner: %s is still in queued state, adding to the jobs queue" %job.id )
if self.track_jobs_in_database:
@@ -121,7 +121,7 @@ class JobQueue( object ):
else:
self.queue.put( ( job.id, job.tool_id ) )
else:
- job_wrapper = JobWrapper( job, self.app.toolbox.tools_by_id[ job.tool_id ], self )
+ job_wrapper = JobWrapper( job, self )
self.dispatcher.recover( job, job_wrapper )
if self.sa_session.dirty:
self.sa_session.flush()
@@ -154,11 +154,12 @@ class JobQueue( object ):
# Pull all new jobs from the queue at once
new_jobs = []
if self.track_jobs_in_database:
- for j in self.sa_session.query( model.Job ) \
+ # Clear the session so we get fresh states for job and all datasets
+ self.sa_session.expunge_all()
+ # Fetch all new jobs
+ new_jobs = self.sa_session.query( model.Job ) \
.options( lazyload( "external_output_metadata" ), lazyload( "parameters" ) ) \
- .filter( model.Job.state == model.Job.states.NEW ):
- job = JobWrapper( j, self.app.toolbox.tools_by_id[ j.tool_id ], self )
- new_jobs.append( job )
+ .filter( model.Job.state == model.Job.states.NEW ).all()
else:
try:
while 1:
@@ -168,8 +169,7 @@ class JobQueue( object ):
# Unpack the message
job_id, tool_id = message
# Create a job wrapper from it
- job_entity = self.sa_session.query( model.Job ).get( job_id )
- job = JobWrapper( job_entity, self.app.toolbox.tools_by_id[ tool_id ], self )
+ job = self.sa_session.query( model.Job ).get( job_id )
# Append to watch queue
new_jobs.append( job )
except Empty:
@@ -179,52 +179,43 @@ class JobQueue( object ):
new_waiting = []
for job in ( new_jobs + self.waiting ):
try:
- # Clear the session for each job so we get fresh states for
- # job and all datasets
- self.sa_session.expunge_all()
- # Get the real job entity corresponding to the wrapper (if we
- # are tracking in the database this is probably cached in
- # the session from the origianl query above)
- job_entity = self.sa_session.query( model.Job ).get( job.job_id )
+ # Since we don't expunge when not tracking jobs in the
+ # database, refresh the job here so it's not stale.
+ if not self.track_jobs_in_database:
+ self.sa_session.refresh( job )
# Check the job's dependencies, requeue if they're not done
- job_state = self.__check_if_ready_to_run( job, job_entity )
- if job_state == JOB_WAIT:
+ job_state = self.__check_if_ready_to_run( job )
+ if job_state == JOB_WAIT:
if not self.track_jobs_in_database:
new_waiting.append( job )
- elif job_state == JOB_ERROR:
- log.info( "job %d ended with an error" % job.job_id )
elif job_state == JOB_INPUT_ERROR:
- log.info( "job %d unable to run: one or more inputs in error state" % job.job_id )
+ log.info( "job %d unable to run: one or more inputs in error state" % job.id )
elif job_state == JOB_INPUT_DELETED:
- log.info( "job %d unable to run: one or more inputs deleted" % job.job_id )
+ log.info( "job %d unable to run: one or more inputs deleted" % job.id )
elif job_state == JOB_READY:
if self.job_lock:
- log.info("Job dispatch attempted for %s, but prevented by administrative lock." % job.job_id)
+ log.info( "Job dispatch attempted for %s, but prevented by administrative lock." % job.id )
if not self.track_jobs_in_database:
new_waiting.append( job )
else:
- self.dispatcher.put( job )
- log.debug( "job %d dispatched" % job.job_id)
+ self.dispatcher.put( JobWrapper( job, self ) )
+ log.info( "job %d dispatched" % job.id )
elif job_state == JOB_DELETED:
- msg = "job %d deleted by user while still queued" % job.job_id
- job.info = msg
- log.debug( msg )
+ log.info( "job %d deleted by user while still queued" % job.id )
elif job_state == JOB_ADMIN_DELETED:
- job.fail( job_entity.info )
- log.info( "job %d deleted by admin while still queued" % job.job_id )
+ job.info( "job %d deleted by admin while still queued" % job.id )
else:
- msg = "unknown job state '%s' for job %d" % ( job_state, job.job_id )
- job.info = msg
- log.error( msg )
+ log.error( "unknown job state '%s' for job %d" % ( job_state, job.id ) )
+ if not self.track_jobs_in_database:
+ new_waiting.append( job )
except Exception, e:
- job.info = "failure running job %d: %s" % ( job.job_id, str( e ) )
- log.exception( "failure running job %d" % job.job_id )
+ log.exception( "failure running job %d" % job.id )
# Update the waiting list
self.waiting = new_waiting
# Done with the session
self.sa_session.remove()
- def __check_if_ready_to_run( self, job_wrapper, job ):
+ def __check_if_ready_to_run( self, job ):
"""
Check if a job is ready to run by verifying that each of its input
datasets is ready (specifically in the OK state). If any input dataset
@@ -244,11 +235,11 @@ class JobQueue( object ):
continue
# don't run jobs for which the input dataset was deleted
if idata.deleted:
- job_wrapper.fail( "input data %d (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) )
+ JobWrapper( job, self ).fail( "input data %d (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) )
return JOB_INPUT_DELETED
# an error in the input data causes us to bail immediately
elif idata.state == idata.states.ERROR:
- job_wrapper.fail( "input data %d is in error state" % ( idata.hid ) )
+ JobWrapper( job, self ).fail( "input data %d is in error state" % ( idata.hid ) )
return JOB_INPUT_ERROR
elif idata.state != idata.states.OK and not ( idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id ):
# need to requeue
@@ -280,11 +271,11 @@ class JobWrapper( object ):
Wraps a 'model.Job' with convience methods for running processes and
state management.
"""
- def __init__(self, job, tool, queue ):
+ def __init__( self, job, queue ):
self.job_id = job.id
self.session_id = job.session_id
self.user_id = job.user_id
- self.tool = tool
+ self.tool = queue.app.toolbox.tools_by_id.get( job.tool_id, None )
self.queue = queue
self.app = queue.app
self.sa_session = self.app.model.context
--- a/lib/galaxy/tools/actions/__init__.py
+++ b/lib/galaxy/tools/actions/__init__.py
@@ -5,7 +5,6 @@ from galaxy.tools.parameters.grouping im
from galaxy.util.template import fill_template
from galaxy.util.none_like import NoneDataset
from galaxy.web import url_for
-from galaxy.jobs import JOB_OK
import galaxy.tools
from types import *
@@ -353,7 +352,7 @@ class DefaultToolAction( object ):
assert GALAXY_URL is not None, "GALAXY_URL parameter missing in tool config."
redirect_url += "&GALAXY_URL=%s" % GALAXY_URL
# Job should not be queued, so set state to ok
- job.state = JOB_OK
+ job.state = trans.app.model.Job.states.OK
job.info = "Redirected to: %s" % redirect_url
trans.sa_session.add( job )
trans.sa_session.flush()
# HG changeset patch -- Bitbucket.org
# Project galaxy-dist
# URL http://bitbucket.org/galaxy/galaxy-dist/overview
# User Nate Coraor <nate(a)bx.psu.edu>
# Date 1279659120 14400
# Node ID 8ffcaf3d9a0c16d080801f3f81104e829403da41
# Parent 7e63649e96b5788bdd72e3d639fca1e8153c2775
Patch from Brad Chapman: Handle URLs for uploading in the case where Galaxy is served under a prefix and nginx is used to handle the uploads.
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -437,10 +437,16 @@ class Tool:
self.check_values = util.string_as_bool( input_elem.get("check_values", "true") )
self.nginx_upload = util.string_as_bool( input_elem.get( "nginx_upload", "false" ) )
self.action = input_elem.get( 'action', '/tool_runner/index' )
+ # If we have an nginx upload, save the action as a tuple instead of
+ # a string. The actual action needs to get url_for run to add any
+ # prefixes, and we want to avoid adding the prefix to the
+ # nginx_upload_path. This logic is handled in the tool_form.mako
+ # template.
if self.nginx_upload and self.app.config.nginx_upload_path:
if '?' in urllib.unquote_plus( self.action ):
raise Exception( 'URL parameters in a non-default tool action can not be used in conjunction with nginx upload. Please convert them to hidden POST parameters' )
- self.action = self.app.config.nginx_upload_path + '?nginx_redir=' + urllib.unquote_plus( self.action )
+ self.action = (self.app.config.nginx_upload_path + '?nginx_redir=',
+ urllib.unquote_plus(self.action))
self.target = input_elem.get( "target", "galaxy_main" )
self.method = input_elem.get( "method", "post" )
# Parse the actual parameters
--- a/templates/tool_form.mako
+++ b/templates/tool_form.mako
@@ -200,14 +200,22 @@ function checkUncheckAll( name, check )
<br/>
%endif
- <div class="toolForm" id="${tool.id}">
+ ## handle calculating the redict url for the special case where we have nginx proxy
+ ## upload and need to do url_for on the redirect portion of the tool action
+ <%
+ try:
+ tool_url = h.url_for(tool.action)
+ except AttributeError:
+ assert len(tool.action) == 2
+ tool_url = tool.action[0] + h.url_for(tool.action[1])
+ %><div class="toolForm" id="${tool.id}">
%if tool.has_multiple_pages:
<div class="toolFormTitle">${tool.name} (step ${tool_state.page+1} of ${tool.npages})</div>
%else:
<div class="toolFormTitle">${tool.name}</div>
%endif
<div class="toolFormBody">
- <form id="tool_form" name="tool_form" action="${h.url_for( tool.action )}" enctype="${tool.enctype}" target="${tool.target}" method="${tool.method}">
+ <form id="tool_form" name="tool_form" action="${tool_url}" enctype="${tool.enctype}" target="${tool.target}" method="${tool.method}"><input type="hidden" name="tool_id" value="${tool.id}"><input type="hidden" name="tool_state" value="${util.object_to_string( tool_state.encode( tool, app ) )}"><input type="hidden" name="tool_id" value="${tool.id}"><input type="hidden" name="tool_state" value="${util.object_to_string( tool_state.encode( tool, app ) )}">
%if tool.display_by_page[tool_state.page]: