May 2012 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: dan: Update GenomeSpace exporter tool to use the newly introduced /personaldirectory as the root for auto-generated export paths.
by Bitbucket 04 May '12

04 May '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/dd777fbcc5fa/ changeset: dd777fbcc5fa user: dan date: 2012-05-04 20:31:22 summary: Update GenomeSpace exporter tool to use the newly introduced /personaldirectory as the root for auto-generated export paths. affected #: 2 files diff -r 1eb32dbab2c1a0bc308eb4d3a069fa55ff6ecb83 -r dd777fbcc5fa1fee5939db5d914611b8b466eed6 tools/genomespace/genomespace_exporter.py --- a/tools/genomespace/genomespace_exporter.py +++ b/tools/genomespace/genomespace_exporter.py @@ -50,6 +50,7 @@ def get_directory( url_opener, dm_url, path ): url = dm_url i = None + dir_dict = {} for i, sub_path in enumerate( path ): url = "%s/%s" % ( url, sub_path ) dir_request = urllib2.Request( url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } ) @@ -57,7 +58,7 @@ try: dir_dict = simplejson.loads( url_opener.open( dir_request ).read() ) except urllib2.HTTPError, e: - #print "e", e, url #punting, assuming lack of permisions at this low of a level... + #print "e", e, url #punting, assuming lack of permissions at this low of a level... continue break if i is not None: @@ -69,6 +70,9 @@ def get_default_directory( url_opener, dm_url ): return get_directory( url_opener, dm_url, ["defaultdirectory"] )[0] +def get_personal_directory( url_opener, dm_url ): + return get_directory( url_opener, dm_url, [ "%s/personaldirectory" % ( GENOMESPACE_API_VERSION_STRING ) ] )[0] + def create_directory( url_opener, directory_dict, new_dir, dm_url ): payload = { "isDirectory": True } for dir_slice in new_dir: @@ -134,7 +138,9 @@ genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ] dm_url = genomespace_site_dict['dmServer'] #get default directory - directory_dict = get_default_directory( url_opener, dm_url )['directory'] + directory_dict = get_default_directory( url_opener, dm_url ).get( 'directory', None ) + if directory_dict is None: + return [] #what directory to stuff this in recurse_directory_dict( url_opener, rval, directory_dict.get( 'url' ) ) @@ -150,7 +156,7 @@ directory_dict, target_directory = get_directory( url_opener, dm_url, [ "%s/%s/%s" % ( GENOMESPACE_API_VERSION_STRING, 'file', target_directory[1] ) ] + target_directory[2:] ) directory_dict = directory_dict['directory'] else: - directory_dict = get_default_directory( url_opener, dm_url )['directory'] + directory_dict = get_personal_directory( url_opener, dm_url )['directory'] #this is the base for the auto-generated galaxy export directories #what directory to stuff this in target_directory_dict = create_directory( url_opener, directory_dict, target_directory, dm_url ) #get upload url @@ -208,6 +214,6 @@ (options, args) = parser.parse_args() - send_file_to_genomespace( options.genomespace_site, options.username, options.token, options.dataset, map( binascii.unhexlify, options.subdirectory ), options.filename, options.file_type, options.content_type, options.log ) + send_file_to_genomespace( options.genomespace_site, options.username, options.token, options.dataset, map( binascii.unhexlify, options.subdirectory ), binascii.unhexlify( options.filename ), options.file_type, options.content_type, options.log ) diff -r 1eb32dbab2c1a0bc308eb4d3a069fa55ff6ecb83 -r dd777fbcc5fa1fee5939db5d914611b8b466eed6 tools/genomespace/genomespace_exporter.xml --- a/tools/genomespace/genomespace_exporter.xml +++ b/tools/genomespace/genomespace_exporter.xml @@ -25,9 +25,9 @@ --subdirectory "${ binascii.hexlify( str( $base_url ).split( '://', 1 )[-1] ) }" ##Protocol removed by request #end if #if $filename: - --filename "${filename}" + --filename "${ binascii.hexlify( str( $filename ) ) }" #else: - --filename "Galaxy History Item ${__app__.security.encode_id( $input1.id )} (${__app__.security.encode_id( $output_log.id )}) - ${input1.hid}: ${input1.name}.${input1.ext}" + --filename "${ binascii.hexlify( "Galaxy History Item %s (%s) - %s: %s.%s" % ( $__app__.security.encode_id( $input1.id ), $__app__.security.encode_id( $output_log.id ), $input1.hid, $input1.name, $input1.ext ) ) }" #end if --file_type "${input1.ext}" --content_type "${input1.get_mime()}" @@ -36,7 +36,6 @@ <inputs><param format="data" name="input1" type="data" label="Send this dataset to GenomeSpace" /><param name="base_url" type="baseurl" /> - <param name="subdirectory" type="drill_down" display="radio" hierarchy="exact" multiple="False" label="Choose Target Directory" dynamic_options="galaxy_code_get_genomespace_folders( genomespace_site = 'prod', trans=__trans__, value=__value__, input_dataset=input1 )" help="Leave blank to generate automatically"/><param name="filename" type="text" size="80" help="Leave blank to generate automatically" /></inputs> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Clarify host string value in community_wsgi.ini.
by Bitbucket 03 May '12

03 May '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/1eb32dbab2c1/ changeset: 1eb32dbab2c1 user: greg date: 2012-05-03 20:00:32 summary: Clarify host string value in community_wsgi.ini. affected #: 1 file diff -r ea9f3c25a41576216f3c328e3333ebdb55250ed1 -r 1eb32dbab2c1a0bc308eb4d3a069fa55ff6ecb83 community_wsgi.ini.sample --- a/community_wsgi.ini.sample +++ b/community_wsgi.ini.sample @@ -4,7 +4,12 @@ use = egg:Paste#http port = 9009 + +# The address on which to listen. By default, only listen to localhost (the tool shed will not +# be accessible over the network). Use '0.0.0.0' to listen on all available network interfaces. +#host = 0.0.0.0 host = 127.0.0.1 + use_threadpool = true threadpool_workers = 10 Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance sample file handling for installed tool shed repositories to handle more than just .loc.sample files in the tool-data directory.
by Bitbucket 03 May '12

03 May '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/ea9f3c25a415/ changeset: ea9f3c25a415 user: greg date: 2012-05-03 19:37:33 summary: Enhance sample file handling for installed tool shed repositories to handle more than just .loc.sample files in the tool-data directory. affected #: 4 files diff -r 71dfeecdd019554b5cccc047546ae26ed7b50afa -r ea9f3c25a41576216f3c328e3333ebdb55250ed1 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -156,9 +156,15 @@ # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = handle_missing_data_table_entry( self.app, self.tool_path, sample_files, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups ) - # Handle tools that use fabric scripts to install dependencies. - handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) + repository_tools_tups, sample_files_copied = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + copy_sample_files( self.app, sample_files, sample_files_copied=sample_files_copied ) + if 'tool_dependencies_config' in metadata_dict: + # Install tool dependencies. + status, message = handle_tool_dependencies( self.app, repository_clone_url, metadata_dict[ 'tool_dependencies_config' ] ) + if status != 'ok' and message: + print 'The following error occurred while installing tool dependencies:' + print message add_to_tool_panel( self.app, repository_name, repository_clone_url, @@ -215,7 +221,7 @@ ctx_rev ) if 'tools' in metadata_dict: # Get the tool_versions from the tool shed for each tool in the installed change set. - url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ ( tool_shed_url, tool_shed_repository.name, self.repository_owner, changeset_revision ) response = urllib2.urlopen( url ) text = response.read() diff -r 71dfeecdd019554b5cccc047546ae26ed7b50afa -r ea9f3c25a41576216f3c328e3333ebdb55250ed1 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -1,4 +1,4 @@ -import os, tempfile, shutil, subprocess, logging, string, urllib2 +import sys, os, tempfile, shutil, subprocess, logging, string, urllib2 from datetime import date, datetime, timedelta from time import strftime, gmtime from galaxy import util @@ -7,6 +7,9 @@ from galaxy.tools.search import ToolBoxSearch from galaxy.model.orm import * +from galaxy import eggs +import pkg_resources + pkg_resources.require( 'mercurial' ) from mercurial import ui, commands @@ -215,7 +218,6 @@ # The value of proprietary_path must be an absolute path due to job_working_directory. elem.attrib[ 'proprietary_path' ] = os.path.abspath( relative_head ) elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module - sniffers = datatypes_config_root.find( 'sniffers' ) else: sniffers = None @@ -270,8 +272,7 @@ return tool_shed_url.rstrip( '/' ) def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """ - Clone the repository up to the specified changeset_revision. No subsequent revisions will be present - in the cloned repository. + Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository. """ commands.clone( get_configured_ui(), repository_clone_url, @@ -279,17 +280,28 @@ pull=True, noupdate=False, rev=[ ctx_rev ] ) -def copy_sample_loc_file( app, filename ): - """Copy xxx.loc.sample to ~/tool-data/xxx.loc.sample and ~/tool-data/xxx.loc""" - head, sample_loc_file = os.path.split( filename ) - loc_file = sample_loc_file.replace( '.sample', '' ) - tool_data_path = os.path.abspath( app.config.tool_data_path ) +def copy_sample_file( app, filename, dest_path=None ): + """ + Copy xxx.loc.sample to dest_path/xxx.loc.sample and dest_path/xxx.loc. The default value for dest_path is ~/tool-data. + """ + if dest_path is None: + dest_path = os.path.abspath( app.config.tool_data_path ) + sample_file_path, sample_file_name = os.path.split( filename ) + copied_file = sample_file_name.replace( '.sample', '' ) # It's ok to overwrite the .sample version of the file. - shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, sample_loc_file ) ) - # Only create the .loc file if it does not yet exist. We don't - # overwrite it in case it contains stuff proprietary to the local instance. - if not os.path.exists( os.path.join( tool_data_path, loc_file ) ): - shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, loc_file ) ) + shutil.copy( os.path.abspath( filename ), os.path.join( dest_path, sample_file_name ) ) + # Only create the .loc file if it does not yet exist. We don't overwrite it in case it contains stuff proprietary to the local instance. + if not os.path.exists( os.path.join( dest_path, copied_file ) ): + shutil.copy( os.path.abspath( filename ), os.path.join( dest_path, copied_file ) ) +def copy_sample_files( app, sample_files, sample_files_copied=None, dest_path=None ): + """ + Copy all files to dest_path in the local Galaxy environment that have not already been copied. Those that have been copied + are contained in sample_files_copied. The default value for dest_path is ~/tool-data. + """ + sample_files_copied = util.listify( sample_files_copied ) + for filename in sample_files: + if filename not in sample_files_copied: + copy_sample_file( app, filename, dest_path=dest_path ) def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): return dict( tool_shed=tool_shed, repository_name=name, @@ -336,7 +348,7 @@ tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) def generate_datatypes_metadata( datatypes_config, metadata_dict ): - """Update the received metadata_dict with changes that have been applied to the received datatypes_config.""" + """Update the received metadata_dict with information from the parsed datatypes_config.""" tree = ElementTree.parse( datatypes_config ) root = tree.getroot() ElementInclude.include( root ) @@ -454,21 +466,9 @@ # Handle tool.requirements. tool_requirements = [] for tr in tool.requirements: - name=tr.name - type=tr.type - if type == 'fabfile': - version = None - fabfile = tr.fabfile - method = tr.method - else: - version = tr.version - fabfile = None - method = None - requirement_dict = dict( name=name, - type=type, - version=version, - fabfile=fabfile, - method=method ) + requirement_dict = dict( name=tr.name, + type=tr.type, + version=tr.version ) tool_requirements.append( requirement_dict ) # Handle tool.tests. tool_tests = [] @@ -732,7 +732,7 @@ break return converter_path, display_path def get_ctx_rev( tool_shed_url, name, owner, changeset_revision ): - url = '%s/repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % ( tool_shed_url, name, owner, changeset_revision ) + url = '%s/repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, name, owner, changeset_revision ) response = urllib2.urlopen( url ) ctx_rev = response.read() response.close() @@ -893,27 +893,27 @@ return repository_tools_tups def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups ): """Inspect each tool to see if it has any input parameters that are dynamically generated select lists that depend on a .loc file.""" - missing_files_handled = [] + sample_files_copied = [] for index, repository_tools_tup in enumerate( repository_tools_tups ): tup_path, guid, repository_tool = repository_tools_tup params_with_missing_index_file = repository_tool.params_with_missing_index_file for param in params_with_missing_index_file: options = param.options - missing_head, missing_tail = os.path.split( options.missing_index_file ) - if missing_tail not in missing_files_handled: + missing_file_path, missing_file_name = os.path.split( options.missing_index_file ) + if missing_file_name not in sample_files_copied: # The repository must contain the required xxx.loc.sample file. for sample_file in sample_files: - sample_head, sample_tail = os.path.split( sample_file ) - if sample_tail == '%s.sample' % missing_tail: - copy_sample_loc_file( app, sample_file ) + sample_file_path, sample_file_name = os.path.split( sample_file ) + if sample_file_name == '%s.sample' % missing_file_name: + copy_sample_file( app, sample_file ) if options.tool_data_table and options.tool_data_table.missing_index_file: options.tool_data_table.handle_found_index_file( options.missing_index_file ) - missing_files_handled.append( missing_tail ) + sample_files_copied.append( options.missing_index_file ) break # Reload the tool into the local list of repository_tools_tups. repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) - return repository_tools_tups + return repository_tools_tups, sample_files_copied def handle_sample_tool_data_table_conf_file( app, filename ): """ Parse the incoming filename and add new entries to the in-memory @@ -1038,7 +1038,8 @@ # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate ) def load_repository_contents( trans, repository_name, description, owner, changeset_revision, ctx_rev, tool_path, repository_clone_url, - relative_install_dir, current_working_dir, tool_shed=None, tool_section=None, shed_tool_conf=None ): + relative_install_dir, current_working_dir, tool_shed=None, tool_section=None, shed_tool_conf=None, + install_tool_dependencies=False ): """Generate the metadata for the installed tool shed repository, among other things.""" # It is critical that the installed repository is updated to the desired changeset_revision before metadata is set because the # process for setting metadata uses the repository files on disk. This method is called when an admin is installing a new repository @@ -1063,8 +1064,9 @@ # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = handle_missing_data_table_entry( trans.app, tool_path, sample_files, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) - # Handle tools that use fabric scripts to install dependencies. + repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + copy_sample_files( trans.app, sample_files, sample_files_copied=sample_files_copied ) handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) add_to_tool_panel( app=trans.app, repository_name=repository_name, diff -r 71dfeecdd019554b5cccc047546ae26ed7b50afa -r ea9f3c25a41576216f3c328e3333ebdb55250ed1 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -5,7 +5,7 @@ from galaxy.tools import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.hash_util import * -from galaxy.util.shed_util import copy_sample_loc_file, get_configured_ui, generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata +from galaxy.util.shed_util import copy_sample_file, get_configured_ui, generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file, to_html_escaped, to_html_str, update_repository from galaxy.web.base.controller import * from galaxy.webapps.community import model @@ -209,13 +209,13 @@ if options.index_file or options.missing_index_file: # Make sure the repository contains the required xxx.loc.sample file. index_file = options.index_file or options.missing_index_file - index_head, index_tail = os.path.split( index_file ) + index_file_path, index_file_name = os.path.split( index_file ) sample_found = False for sample_file in sample_files: - sample_head, sample_tail = os.path.split( sample_file ) - if sample_tail == '%s.sample' % index_tail: - copy_sample_loc_file( trans.app, sample_file ) - options.index_file = index_tail + sample_file_path, sample_file_name = os.path.split( sample_file ) + if sample_file_name == '%s.sample' % index_file_name: + copy_sample_file( trans.app, sample_file ) + options.index_file = index_file_name options.missing_index_file = None if options.tool_data_table: options.tool_data_table.missing_index_file = None @@ -224,7 +224,7 @@ if not sample_found: can_set_metadata = False correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file ) - correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_tail ) + correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name ) invalid_files.append( ( name, correction_msg ) ) return can_set_metadata, invalid_files def new_tool_metadata_required( trans, id, metadata_dict ): @@ -283,8 +283,7 @@ else: # There is no saved repository metadata, so we need to create a new repository_metadata table record. return True - # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table - # record is not needed. + # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table record is not needed. return False def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo, repo_dir ): """ diff -r 71dfeecdd019554b5cccc047546ae26ed7b50afa -r ea9f3c25a41576216f3c328e3333ebdb55250ed1 lib/galaxy/webapps/community/controllers/upload.py --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -139,9 +139,8 @@ if error: message = '%s<br/>%s' % ( message, error_message ) if full_path.endswith( '.loc.sample' ): - # Handle the special case where a xxx.loc.sample file is - # being uploaded by copying it to ~/tool-data/xxx.loc. - copy_sample_loc_file( trans.app, full_path ) + # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc. + copy_sample_file( trans.app, full_path ) # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions ) != 1 handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) @@ -295,9 +294,8 @@ if error: return False, message, files_to_remove, content_alert_str if filename_in_archive.endswith( '.loc.sample' ): - # Handle the special case where a xxx.loc.sample file is - # being uploaded by copying it to ~/tool-data/xxx.loc. - copy_sample_loc_file( trans.app, filename_in_archive ) + # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc. + copy_sample_file( trans.app, filename_in_archive ) try: commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) except Exception, e: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Allow GenomeSpace importer tool to sniff filetype when GSMetadata object is not available.
by Bitbucket 03 May '12

03 May '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/71dfeecdd019/ changeset: 71dfeecdd019 user: dan date: 2012-05-03 18:25:06 summary: Allow GenomeSpace importer tool to sniff filetype when GSMetadata object is not available. affected #: 1 file diff -r 4d7077d8d2b875feed71fa8c9ab7bd701957b909 -r 71dfeecdd019554b5cccc047546ae26ed7b50afa tools/genomespace/genomespace_importer.py --- a/tools/genomespace/genomespace_importer.py +++ b/tools/genomespace/genomespace_importer.py @@ -1,6 +1,6 @@ #Dan Blankenberg -import optparse, os, urllib2, urllib, cookielib, urlparse +import optparse, os, urllib2, urllib, cookielib, urlparse, tempfile, shutil from galaxy import eggs import pkg_resources @@ -8,6 +8,10 @@ pkg_resources.require( "simplejson" ) import simplejson +import galaxy.model # need to import model before sniff to resolve a circular import dependency +from galaxy.datatypes import sniff +from galaxy.datatypes.registry import Registry + GENOMESPACE_API_VERSION_STRING = "v1.0" GENOMESPACE_SERVER_URL_PROPERTIES = "http://www.genomespace.org/sites/genomespacefiles/config/serverurl.properti…" @@ -61,13 +65,13 @@ cookie_opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cj ) ) return cookie_opener -def get_galaxy_ext_from_genomespace_format_url( url_opener, file_format_url ): +def get_galaxy_ext_from_genomespace_format_url( url_opener, file_format_url, default = DEFAULT_GALAXY_EXT ): ext = GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT.get( file_format_url, None ) if ext is not None: ext = GENOMESPACE_EXT_TO_GALAXY_EXT.get( ext, None ) if ext is None: #could check content type, etc here - ext = DEFAULT_GALAXY_EXT + ext = default return ext def get_genomespace_site_urls(): @@ -94,8 +98,6 @@ def download_from_genomespace_importer( username, token, json_parameter_file, genomespace_site ): json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() ) datasource_params = json_params.get( 'param_dict' ) - #username = datasource_params.get( "gs-username", None ) - #token = datasource_params.get( "gs-token", None ) assert None not in [ username, token ], "Missing GenomeSpace username or token." output_filename = datasource_params.get( "output_file1", None ) dataset_id = json_params['output_data'][0]['dataset_id'] @@ -106,17 +108,15 @@ set_genomespace_format_identifiers( url_opener, genomespace_site_dict['dmServer'] ) file_url_name = "URL" metadata_parameter_file = open( json_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'wb' ) + #setup datatypes registry for sniffing + datatypes_registry = Registry() + datatypes_registry.load_datatypes( root_dir = json_params[ 'job_config' ][ 'GALAXY_ROOT_DIR' ], config = json_params[ 'job_config' ][ 'GALAXY_DATATYPES_CONF_FILE' ] ) url_param = datasource_params.get( file_url_name, None ) for download_url in url_param.split( ',' ): + using_temp_file = False parsed_url = urlparse.urlparse( download_url ) query_params = urlparse.parse_qs( parsed_url[4] ) - file_type = DEFAULT_GALAXY_EXT - if 'dataformat' in query_params: - file_type = query_params[ 'dataformat' ][0] - file_type = get_galaxy_ext_from_genomespace_format_url( url_opener, file_type ) - elif '.' in parsed_url[2]: - file_type = parsed_url[2].rsplit( '.', 1 )[-1] - file_type = GENOMESPACE_EXT_TO_GALAXY_EXT.get( file_type, file_type ) + #write file to disk new_file_request = urllib2.Request( download_url ) new_file_request.get_method = lambda: 'GET' target_download_url = url_opener.open( new_file_request ) @@ -130,16 +130,58 @@ query_params = urlparse.parse_qs( parsed_url[4] ) filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] ) if output_filename is None: - output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_output%s_visible_%s' % ( hda_id, ''.join( c in VALID_CHARS and c or '-' for c in filename ), file_type ) ) - else: - if dataset_id is not None: - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', - dataset_id = dataset_id, - ext = file_type, - name = "GenomeSpace importer on %s" % ( filename ) ) ) ) + #need to use a temp file here, because we do not know the ext yet + using_temp_file = True + output_filename = tempfile.NamedTemporaryFile( prefix='tmp-genomespace-importer-' ).name output_file = open( output_filename, 'wb' ) chunk_write( target_download_url, output_file ) output_file.close() + + #determine file format + file_type = None + if 'dataformat' in query_params: #this is a converted dataset + file_type = query_params[ 'dataformat' ][0] + file_type = get_galaxy_ext_from_genomespace_format_url( url_opener, file_type ) + else: + try: + #get and use GSMetadata object + download_file_path = download_url.split( "%s/file/" % ( genomespace_site_dict['dmServer'] ), 1)[-1] #FIXME: This is a very bad way to get the path for determining metadata. There needs to be a way to query API using download URLto get to the metadata object + metadata_request = urllib2.Request( "%s/%s/filemetadata/%s" % ( genomespace_site_dict['dmServer'], GENOMESPACE_API_VERSION_STRING, download_file_path ) ) + metadata_request.get_method = lambda: 'GET' + metadata_url = url_opener.open( metadata_request ) + file_metadata_dict = simplejson.loads( metadata_url.read() ) + metadata_url.close() + file_type = file_metadata_dict.get( 'dataFormat', None ) + if file_type and file_type.get( 'url' ): + file_type = file_type.get( 'url' ) + file_type = get_galaxy_ext_from_genomespace_format_url( url_opener, file_type, default = None ) + except: + pass + if file_type is None: + #try to sniff datatype + try: + file_type = sniff.handle_uploaded_dataset_file( output_filename, datatypes_registry ) + except: + pass #sniff failed + if file_type is None and '.' in parsed_url[2]: + #still no known datatype, fall back to using extension + file_type = parsed_url[2].rsplit( '.', 1 )[-1] + file_type = GENOMESPACE_EXT_TO_GALAXY_EXT.get( file_type, file_type ) + if file_type is None: + #use default extension (e.g. 'data') + file_type = DEFAULT_GALAXY_EXT + + #save json info for single primary dataset + if dataset_id is not None: + metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', + dataset_id = dataset_id, + ext = file_type, + name = "GenomeSpace importer on %s" % ( filename ) ) ) ) + #if using tmp file, move the file to the new file path dir to get scooped up later + if using_temp_file: + shutil.move( output_filename, os.path.join( datasource_params['__new_file_path__'], 'primary_%i_output%s_visible_%s' % ( hda_id, ''.join( c in VALID_CHARS and c or '-' for c in filename ), file_type ) ) ) + + dataset_id = None #only one primary dataset available output_filename = None #only have one filename available metadata_parameter_file.close() return True Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 03 May '12

03 May '12

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/81e6e2f44d32/ changeset: 81e6e2f44d32 user: cjav date: 2012-05-03 01:30:11 summary: expose dataset full path through the API affected #: 1 file diff -r c5f3e218266e8b82cd509b9177461839c74971c3 -r 81e6e2f44d32845b8d48459ea779030d596c7c48 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1167,6 +1167,7 @@ deleted = hda.deleted, visible = hda.visible, state = hda.state, + file_name = hda.file_name, file_size = int( hda.get_size() ), data_type = hda.ext, genome_build = hda.dbkey, https://bitbucket.org/galaxy/galaxy-central/changeset/4d7077d8d2b8/ changeset: 4d7077d8d2b8 user: dannon date: 2012-05-03 16:18:38 summary: Incorporation and slight update to pull request #43. The original contribution allowed access to file_path regardless of admin status and config setting. affected #: 2 files diff -r 81e6e2f44d32845b8d48459ea779030d596c7c48 -r 4d7077d8d2b875feed71fa8c9ab7bd701957b909 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1167,7 +1167,6 @@ deleted = hda.deleted, visible = hda.visible, state = hda.state, - file_name = hda.file_name, file_size = int( hda.get_size() ), data_type = hda.ext, genome_build = hda.dbkey, diff -r 81e6e2f44d32845b8d48459ea779030d596c7c48 -r 4d7077d8d2b875feed71fa8c9ab7bd701957b909 lib/galaxy/web/api/history_contents.py --- a/lib/galaxy/web/api/history_contents.py +++ b/lib/galaxy/web/api/history_contents.py @@ -1,11 +1,9 @@ """ API operations on the contents of a history. """ -import logging, os, string, shutil, urllib, re, socket -from cgi import escape, FieldStorage -from galaxy import util, datatypes, jobs, web, util +import logging +from galaxy import web from galaxy.web.base.controller import * -from galaxy.util.sanitize_html import sanitize_html from galaxy.model.orm import * import pkg_resources @@ -56,6 +54,8 @@ return str( e ) try: item = content.get_api_value( view='element' ) + if trans.user_is_admin() or trans.app.config.expose_dataset_path: + item['file_name'] = hda.file_name if not item['deleted']: # Problem: Method url_for cannot use the dataset controller # Get the environment from DefaultWebTransaction and use default webapp mapper instead of webapp API mapper @@ -76,7 +76,6 @@ POST /api/libraries/{encoded_history_id}/contents Creates a new history content item (file, aka HistoryDatasetAssociation). """ - params = util.Params( payload ) from_ld_id = payload.get( 'from_ld_id', None ) try: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Update WormBase datasource tool to use WormBase 2.
by Bitbucket 02 May '12

02 May '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/c5f3e218266e/ changeset: c5f3e218266e user: dan date: 2012-05-02 23:04:33 summary: Update WormBase datasource tool to use WormBase 2. affected #: 1 file diff -r a781f132e259b60c417c674047535ac0a688b69d -r c5f3e218266e8b82cd509b9177461839c74971c3 tools/data_source/wormbase.xml --- a/tools/data_source/wormbase.xml +++ b/tools/data_source/wormbase.xml @@ -1,8 +1,8 @@ <?xml version="1.0"?> -<tool name="Wormbase" id="wormbase" tool_type="data_source"> +<tool name="WormBase" id="wormbase" tool_type="data_source" version="1.0.1"><description>server</description><command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command> - <inputs action="http://www.wormbase.org/db/seq/gbgff/c_elegans/" check_values="false" target="_top"> + <inputs action="http://www.wormbase.org/tools/genome/gbrowse/c_elegans/" check_values="false" target="_top"><display>go to Wormbase server $GALAXY_URL</display><param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=wormbase" /></inputs> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix typo in BAM slice tool.
by Bitbucket 02 May '12

02 May '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/a781f132e259/ changeset: a781f132e259 user: dan date: 2012-05-02 17:47:04 summary: Fix typo in BAM slice tool. affected #: 1 file diff -r 926c93b412beaaff5579e6faa2d052768e9408d1 -r a781f132e259b60c417c674047535ac0a688b69d tools/samtools/samtools_slice_bam.xml --- a/tools/samtools/samtools_slice_bam.xml +++ b/tools/samtools/samtools_slice_bam.xml @@ -20,7 +20,7 @@ <test><param name="input_bam" value="gatk/fake_phiX_reads_1.bam" ftype="bam" /><param name="input_interval" value="gatk/fake_phiX_variant_locations.bed" ftype="bed" /> - <output name="output_log" file="gatk/fake_phiX_reads_1.bam" ftype="bam" /> + <output name="output_bam" file="gatk/fake_phiX_reads_1.bam" ftype="bam" /></test></tests><help> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Add Slice BAM by regions tool.
by Bitbucket 02 May '12

02 May '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/926c93b412be/ changeset: 926c93b412be user: dan date: 2012-05-02 17:32:19 summary: Add Slice BAM by regions tool. affected #: 3 files diff -r 5e53e5d824009984629a269b9d768b99b5683310 -r 926c93b412beaaff5579e6faa2d052768e9408d1 tool_conf.xml.sample --- a/tool_conf.xml.sample +++ b/tool_conf.xml.sample @@ -384,6 +384,7 @@ <tool file="samtools/pileup_interval.xml" /><tool file="samtools/samtools_flagstat.xml" /><tool file="samtools/samtools_rmdup.xml" /> + <tool file="samtools/samtools_slice_bam.xml" /></section><section name="NGS: GATK Tools (beta)" id="gatk"><label text="Alignment Utilities" id="gatk_bam_utilities"/> diff -r 5e53e5d824009984629a269b9d768b99b5683310 -r 926c93b412beaaff5579e6faa2d052768e9408d1 tools/samtools/samtools_slice_bam.py --- /dev/null +++ b/tools/samtools/samtools_slice_bam.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +#Dan Blankenberg + +""" +A wrapper script for slicing a BAM file by provided BED file using SAMTools. +%prog input_filename.sam output_filename.bam +""" +#TODO: Confirm that the sort is necessary e.g. if input regions are out of order + + +import sys, optparse, os, tempfile, subprocess, shutil + +CHUNK_SIZE = 2**20 #1mb + +def cleanup_before_exit( tmp_dir ): + if tmp_dir and os.path.exists( tmp_dir ): + shutil.rmtree( tmp_dir ) + +def __main__(): + #Parse Command Line + parser = optparse.OptionParser() + (options, args) = parser.parse_args() + + assert len( args ) == 4, "Invalid command line: samtools_slice_bam.py input.bam input.bam.bai input.interval output.bam" + input_bam_filename, input_index_filename, input_interval_filename, output_bam_filename = args + + tmp_dir = tempfile.mkdtemp( prefix='tmp-samtools_slice_bam-' ) + + tmp_input_bam_filename = os.path.join( tmp_dir, 'input_bam.bam' ) + os.symlink( input_bam_filename, tmp_input_bam_filename ) + os.symlink( input_index_filename, "%s.bai" % tmp_input_bam_filename ) + + #Slice BAM + unsorted_bam_filename = os.path.join( tmp_dir, 'unsorted.bam' ) + unsorted_stderr_filename = os.path.join( tmp_dir, 'unsorted.stderr' ) + cmd = 'samtools view -b -L "%s" "%s" > "%s"' % ( input_interval_filename, tmp_input_bam_filename, unsorted_bam_filename ) + proc = subprocess.Popen( args=cmd, stderr=open( unsorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) + return_code = proc.wait() + if return_code: + stderr_target = sys.stderr + else: + stderr_target = sys.stdout + stderr = open( unsorted_stderr_filename ) + while True: + chunk = stderr.read( CHUNK_SIZE ) + if chunk: + stderr_target.write( chunk ) + else: + break + stderr.close() + + #sort sam, so indexing will not fail + #TODO: confirm if sorting is necessary (is original BAM order maintained, or does the output follow the order of input intervals?) + sorted_stderr_filename = os.path.join( tmp_dir, 'sorted.stderr' ) + sorting_prefix = os.path.join( tmp_dir, 'sorted_bam' ) + cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_bam_filename ) + proc = subprocess.Popen( args=cmd, stderr=open( sorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) + return_code = proc.wait() + + if return_code: + stderr_target = sys.stderr + else: + stderr_target = sys.stdout + stderr = open( sorted_stderr_filename ) + while True: + chunk = stderr.read( CHUNK_SIZE ) + if chunk: + stderr_target.write( chunk ) + else: + break + stderr.close() + + cleanup_before_exit( tmp_dir ) + +if __name__=="__main__": __main__() diff -r 5e53e5d824009984629a269b9d768b99b5683310 -r 926c93b412beaaff5579e6faa2d052768e9408d1 tools/samtools/samtools_slice_bam.xml --- /dev/null +++ b/tools/samtools/samtools_slice_bam.xml @@ -0,0 +1,40 @@ +<tool id="samtools_slice_bam" name="Slice BAM" version="0.0.1"> + <description>by provided regions</description> + <requirements> + <requirement type="package">samtools</requirement> + </requirements> + <command interpreter="python">samtools_slice_bam.py + "${input_bam}" + "${input_bam.metadata.bam_index}" + "${input_interval}" + "${output_bam}" + </command> + <inputs> + <param name="input_bam" type="data" format="bam" label="BAM file" /> + <param name="input_interval" type="data" format="bed" label="BED file" /> + </inputs> + <outputs> + <data format="bam" name="output_bam"/> + </outputs> + <tests> + <test> + <param name="input_bam" value="gatk/fake_phiX_reads_1.bam" ftype="bam" /> + <param name="input_interval" value="gatk/fake_phiX_variant_locations.bed" ftype="bed" /> + <output name="output_log" file="gatk/fake_phiX_reads_1.bam" ftype="bam" /> + </test> + </tests> + <help> +**What it does** + + Accepts an input BAM file and an input BED file and creates an output BAM file containing only those alignments that overlap the provided BED intervals. + +------ + +**Citation** + +For the underlying tool, please cite `Li H, Handsaker B, Wysoker A, Fennell T, Ruan J, Homer N, Marth G, Abecasis G, Durbin R; 1000 Genome Project Data Processing Subgroup. The Sequence Alignment/Map format and SAMtools. Bioinformatics. 2009 Aug 15;25(16):2078-9. <http://www.ncbi.nlm.nih.gov/pubmed/19505943>`_ + +If you use this tool in Galaxy, please cite Blankenberg D, et al. *In preparation.* + + </help> +</tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix popup menu for viewing tool metadata in a tool shed repository to be compatible with new styles.
by Bitbucket 01 May '12

01 May '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/5e53e5d82400/ changeset: 5e53e5d82400 user: greg date: 2012-05-01 18:17:07 summary: Fix popup menu for viewing tool metadata in a tool shed repository to be compatible with new styles. affected #: 1 file diff -r 49b5dc00dfb6ef3bf0f4bb1f611db33042469181 -r 5e53e5d824009984629a269b9d768b99b5683310 templates/webapps/community/repository/common.mako --- a/templates/webapps/community/repository/common.mako +++ b/templates/webapps/community/repository/common.mako @@ -108,15 +108,13 @@ <td><b>version</b></td><td><b>requirements</b></td></tr> - %for tool_dict in tool_dicts: + %for index, tool_dict in enumerate( tool_dicts ): <tr><td> - <div style="float: left; margin-left: 1px;" class="menubutton split popup" id="tool-${tool_dict[ 'id' ].replace( ' ', '_' )}-popup"> - <a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( repository.id ), tool_config=tool_dict[ 'tool_config' ], changeset_revision=changeset_revision, webapp=webapp )}"> - ${tool_dict[ 'name' ]} - </a> + <div style="float:left;" class="menubutton split popup" id="tool-${index}-popup"> + <a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( repository.id ), tool_config=tool_dict[ 'tool_config' ], changeset_revision=changeset_revision, webapp=webapp )}">${tool_dict[ 'name' ]}</a></div> - <div popupmenu="tool-${tool_dict[ 'id' ].replace( ' ', '_' )}-popup"> + <div popupmenu="tool-${index}-popup"><a class="action-button" href="${h.url_for( controller='repository', action='view_tool_metadata', repository_id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision, tool_id=tool_dict[ 'id' ], webapp=webapp )}">View tool metadata</a></div></td> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Eliminate the fabfile/method type from ToolRequirement.
by Bitbucket 01 May '12

01 May '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/49b5dc00dfb6/ changeset: 49b5dc00dfb6 user: greg date: 2012-05-01 15:15:59 summary: Eliminate the fabfile/method type from ToolRequirement. affected #: 1 file diff -r 0090f9df64fd84a9fda73d12b816a7d4d044b822 -r 49b5dc00dfb6ef3bf0f4bb1f611db33042469181 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -702,19 +702,13 @@ class ToolRequirement( object ): """ - Represents an external requirement that must be available for the tool to - run (for example, a program, package, or library). Requirements can - optionally assert a specific version, or reference a command to execute a - fabric script. If fabric is used, the type is 'fabfile' and the version - attribute is not used since the fabric script includes all necessary - information for automatic dependency installation. + Represents an external requirement that must be available for the tool to run (for example, a program, package, or library). + Requirements can optionally assert a specific version. """ - def __init__( self, name=None, type=None, version=None, fabfile=None, method=None ): + def __init__( self, name=None, type=None, version=None ): self.name = name self.type = type self.version = version - self.fabfile = fabfile - self.method = method class ToolParallelismInfo(object): """ @@ -1339,19 +1333,8 @@ for requirement_elem in requirements_elem.findall( 'requirement' ): name = util.xml_text( requirement_elem ) type = requirement_elem.get( "type", "package" ) - if type == 'fabfile': - # The fabric script will include all necessary information for - # automatically installing the tool dependencies. - fabfile = requirement_elem.get( "fabfile" ) - method = requirement_elem.get( "method" ) - version = None - else: - # For backward compatibility, requirements tag sets should not require the - # use of a fabric script. - version = requirement_elem.get( "version" ) - fabfile = None - method = None - requirement = ToolRequirement( name=name, type=type, version=version, fabfile=fabfile, method=method ) + version = requirement_elem.get( "version", None ) + requirement = ToolRequirement( name=name, type=type, version=version ) self.requirements.append( requirement ) def check_workflow_compatible( self ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0