- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 7 new changesets
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

7 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e80c212ba514/ Changeset: e80c212ba514 User: dannon Date: 2014-07-23 23:24:33 Summary: Strip whitespace in manage_info template. Affected #: 1 file diff -r 6fbe4d95a8dc64bd222dbf1170bdadcc5321e855 -r e80c212ba514b4960c01f681cf15a2bc8b2b4882 templates/webapps/galaxy/user/manage_info.mako --- a/templates/webapps/galaxy/user/manage_info.mako +++ b/templates/webapps/galaxy/user/manage_info.mako @@ -49,7 +49,7 @@ <div class="form-row"><div class="grid-header"> %for i, filter in enumerate( ['Active', 'Deleted', 'All'] ): - %if i > 0: + %if i > 0: <span>|</span> %endif %if show_filter == filter: @@ -62,10 +62,10 @@ </div><table class="grid"><tbody> - %for index, address in enumerate(addresses): + %for index, address in enumerate(addresses): <tr class="libraryRow libraryOrFolderRow" id="libraryRow"><td> - <div class="form-row"> + <div class="form-row"><label>${address.desc}:</label> ${address.get_html()} </div> @@ -82,7 +82,7 @@ </ul></div></td> - </tr> + </tr> %endfor </tbody></table> https://bitbucket.org/galaxy/galaxy-central/commits/65f2c8c30d47/ Changeset: 65f2c8c30d47 User: dannon Date: 2014-07-24 22:04:39 Summary: Sanitize user addresses. This should be revisited at some point -- not sure why these html stubs (and only these) generated in the model. Affected #: 1 file diff -r e80c212ba514b4960c01f681cf15a2bc8b2b4882 -r 65f2c8c30d477eb1008bf4e3b40c8695f5ac2df6 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -34,6 +34,7 @@ from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash from galaxy.util.directory_hash import directory_hash_id +from galaxy.util.sanitize_html import sanitize_html from galaxy.web.framework.helpers import to_unicode from galaxy.web.form_builder import (AddressField, CheckboxField, HistoryField, PasswordField, SelectField, TextArea, TextField, WorkflowField, @@ -3703,23 +3704,27 @@ self.country = country self.phone = phone def get_html(self): + # This should probably be deprecated eventually. It should currently + # sanitize. + # TODO Find out where else uses this and replace with + # templates html = '' if self.name: - html = html + self.name + html = html + sanitize_html(self.name) if self.institution: - html = html + '<br/>' + self.institution + html = html + '<br/>' + sanitize_html(self.institution) if self.address: - html = html + '<br/>' + self.address + html = html + '<br/>' + sanitize_html(self.address) if self.city: - html = html + '<br/>' + self.city + html = html + '<br/>' + sanitize_html(self.city) if self.state: - html = html + ' ' + self.state + html = html + ' ' + sanitize_html(self.state) if self.postal_code: - html = html + ' ' + self.postal_code + html = html + ' ' + sanitize_html(self.postal_code) if self.country: - html = html + '<br/>' + self.country + html = html + '<br/>' + sanitize_html(self.country) if self.phone: - html = html + '<br/>' + 'Phone: ' + self.phone + html = html + '<br/>' + 'phone: ' + sanitize_html(self.phone) return html class UserOpenID( object ): https://bitbucket.org/galaxy/galaxy-central/commits/e39c3a510cc1/ Changeset: e39c3a510cc1 User: dannon Date: 2014-07-24 22:06:21 Summary: Sanitize address description display in the manage_info field. Affected #: 1 file diff -r 65f2c8c30d477eb1008bf4e3b40c8695f5ac2df6 -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 templates/webapps/galaxy/user/manage_info.mako --- a/templates/webapps/galaxy/user/manage_info.mako +++ b/templates/webapps/galaxy/user/manage_info.mako @@ -66,7 +66,7 @@ <tr class="libraryRow libraryOrFolderRow" id="libraryRow"><td><div class="form-row"> - <label>${address.desc}:</label> + <label>${address.desc | h}:</label> ${address.get_html()} </div><div class="form-row"> https://bitbucket.org/galaxy/galaxy-central/commits/6e6e184ec4e1/ Changeset: 6e6e184ec4e1 User: dannon Date: 2014-07-24 22:06:37 Summary: Merge. Affected #: 6 files diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1712,10 +1712,15 @@ if self.__display_multirun_option(): # Select multiple datasets, run multiple jobs. multirun_key = "%s|__multirun__" % self.name + collection_multirun_key = "%s|__collection_multirun__" % self.name if multirun_key in (other_values or {}): multirun_value = listify( other_values[ multirun_key ] ) if multirun_value and len( multirun_value ) > 1: default_field = "select_multiple" + elif collection_multirun_key in (other_values or {}): + multirun_value = listify( other_values[ collection_multirun_key ] ) + if multirun_value: + default_field = "select_collection" else: multirun_value = value multi_dataset_matcher = DatasetMatcher( trans, self, multirun_value, other_values ) @@ -2014,9 +2019,17 @@ default_field = "select_single_collection" fields = odict() + collection_multirun_key = "%s|__collection_multirun__" % self.name + if collection_multirun_key in (other_values or {}): + multirun_value = other_values[ collection_multirun_key ] + if multirun_value: + default_field = "select_map_over_collections" + else: + multirun_value = value + history = self._get_history( trans ) fields[ "select_single_collection" ] = self._get_single_collection_field( trans=trans, history=history, value=value, other_values=other_values ) - fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=value, other_values=other_values ) + fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=multirun_value, other_values=other_values ) return self._switch_fields( fields, default_field=default_field ) diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae static/scripts/galaxy.tools.js --- a/static/scripts/galaxy.tools.js +++ b/static/scripts/galaxy.tools.js @@ -86,7 +86,7 @@ }).attr( 'title', selectionType['select_by'] - ); + ).data( "index", iIndex ); view.formRow().find( "label" ).append( button ); } }); @@ -114,11 +114,13 @@ } else { $("div#remap-row").css("display", "none"); } - this.formRow().find( "i" ).each(function(index, iElement) { + this.formRow().find( "i" ).each(function(_, iElement) { + var $iElement = $(iElement); + var index = $iElement.data("index"); if(index == enableIndex) { - $(iElement).css('color', 'black'); + $iElement.css('color', 'black'); } else { - $(iElement).css('color', 'Gray'); + $iElement.css('color', 'Gray'); } }); var $select = this.$( "select" ); diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae static/scripts/viz/trackster.js --- a/static/scripts/viz/trackster.js +++ b/static/scripts/viz/trackster.js @@ -103,8 +103,9 @@ var self = this, menu = create_icon_buttons_menu([ { icon_class: 'plus-button', title: 'Add tracks', on_click: function() { - visualization.select_datasets(galaxy_config.root + "visualization/list_current_history_datasets", galaxy_config.root + "api/datasets", { 'f-dbkey': view.dbkey }, function(tracks) { - _.each(tracks, function(track) { + visualization.select_datasets(galaxy_config.root + "visualization/list_current_history_datasets", galaxy_config.root + "api/datasets", { 'f-dbkey': view.dbkey }, + function(new_tracks) { + _.each(new_tracks, function(track) { view.add_drawable( tracks.object_from_template(track, view, view) ); }); }); diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae test/functional/tools/collection_two_paired.xml --- /dev/null +++ b/test/functional/tools/collection_two_paired.xml @@ -0,0 +1,69 @@ +<tool id="collection_two_paired" name="collection_two_paired" version="0.1.0"> + <command> + #if $kind.collection_type == "paired" + cat $kind.f1.forward $kind.f1['reverse'] >> $out1; + cat $kind.f2.forward $kind.f2['reverse'] >> $out1; + #else + #for $i, $_ in enumerate($kind.f1): + cat $kind.f1[$i] $kind.f2[$i] >> $out1; + #end for + #end if + </command> + <inputs> + <conditional name="kind"> + <param type="select" name="collection_type"> + <option value="paired">Paired Datasets</option> + <option value="list">List of Datasets</option> + </param> + <when value="paired"> + <param name="f1" type="data_collection" collection_type="paired" /> + <param name="f2" type="data_collection" collection_type="paired" /> + </when> + <when value="list"> + <param name="f1" type="data_collection" collection_type="paired" /> + <param name="f2" type="data_collection" collection_type="paired" /> + </when> + </conditional> + </inputs> + <outputs> + <data format="txt" name="out1" /> + </outputs> + <tests> + <test> + <conditional name="kind"> + <param name="collection_type" value="paired" /> + <param name="f1"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + <test> + <conditional name="kind"> + <param name="collection_type" value="list" /> + <param name="f1"> + <collection type="list"> + <element name="l11" value="simple_line.txt" /> + <element name="l12" value="simple_line.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="list"> + <element name="l21" value="simple_line_alternative.txt" /> + <element name="l22" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + </tests> +</tool> diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -21,4 +21,5 @@ <tool file="collection_paired_test.xml" /><tool file="collection_nested_test.xml" /><tool file="collection_mixed_param.xml" /> + <tool file="collection_two_paired.xml" /></toolbox> \ No newline at end of file diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae test/unit/tools/test_execution.py --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -16,13 +16,12 @@ eggs.require( "Paste" ) from paste import httpexceptions -# Tool with a repeat parameter, to test state update. -REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"> +BASE_REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"><command>echo "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1</command><inputs><param type="text" name="param1" value="" /><repeat name="repeat1" label="Repeat 1"> - <param type="text" name="param2" value="" /> + %s </repeat></inputs><outputs> @@ -31,6 +30,10 @@ </tool> ''' +# Tool with a repeat parameter, to test state update. +REPEAT_TOOL_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="text" name="param2" value="" />''' +REPEAT_COLLECTION_PARAM_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="data_collection" name="param2" collection_type="paired" />''' + class ToolExecutionTestCase( TestCase, tools_support.UsesApp, tools_support.UsesTools ): @@ -287,13 +290,48 @@ } ) self.__assert_exeuted( template, template_vars ) - def __history_dataset_collection_for( self, hdas, id=1234 ): - collection = galaxy.model.DatasetCollection() + def test_subcollection_multirun_with_state_updates( self ): + self._init_tool( REPEAT_COLLECTION_PARAM_CONTENTS ) + hda1, hda2 = self.__add_dataset( 1 ), self.__add_dataset( 2 ) + collection = self.__history_dataset_collection_for( [ hda1, hda2 ], collection_type="list:paired" ) + collection_id = self.app.security.encode_id( collection.id ) + self.app.dataset_collections_service = Bunch( + match_collections=lambda collections: None + ) + template, template_vars = self.__handle_with_incoming( + repeat1_add="dummy", + ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert len( state.inputs[ "repeat1" ] ) == 1 + template, template_vars = self.__handle_with_incoming( state, **{ + "repeat1_0|param2|__collection_multirun__": "%s|paired" % collection_id, + "repeat1_add": "dummy", + } ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert state.inputs[ "repeat1" ][ 0 ][ "param2|__collection_multirun__" ] == "%s|paired" % collection_id + + def __history_dataset_collection_for( self, hdas, collection_type="list", id=1234 ): + collection = galaxy.model.DatasetCollection( + collection_type=collection_type, + ) to_element = lambda hda: galaxy.model.DatasetCollectionElement( collection=collection, element=hda, ) - collection.datasets = map(to_element, hdas) + elements = map(to_element, hdas) + if collection_type == "list:paired": + paired_collection = galaxy.model.DatasetCollection( + collection_type="paired", + ) + paired_collection.elements = elements + list_dce = galaxy.model.DatasetCollectionElement( + collection=collection, + element=paired_collection, + ) + elements = [ list_dce ] + + collection.elements = elements + history_dataset_collection_association = galaxy.model.HistoryDatasetCollectionAssociation( id=id, collection=collection, @@ -349,13 +387,13 @@ self.history.datasets.append( hda ) return hda - def __add_collection_dataset( self, id, *hdas ): + def __add_collection_dataset( self, id, collection_type="paired", *hdas ): hdca = galaxy.model.HistoryDatasetCollectionAssociation() hdca.id = id collection = galaxy.model.DatasetCollection() hdca.collection = collection collection.elements = [ galaxy.model.DatasetCollectionElement(element=self.__add_dataset( 1 )) ] - + collection.type = collection_type self.trans.sa_session.model_objects[ galaxy.model.HistoryDatasetCollectionAssociation ][ id ] = hdca self.history.dataset_collections.append( hdca ) return hdca https://bitbucket.org/galaxy/galaxy-central/commits/3686de5c8280/ Changeset: 3686de5c8280 User: dannon Date: 2014-07-24 23:29:51 Summary: merge Affected #: 2 files diff -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae -r 3686de5c8280c18c197323948c5e2ea79e642a0b lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1391,7 +1391,7 @@ # Load parameters (optional) input_elem = root.find("inputs") enctypes = set() - if input_elem: + if input_elem is not None: # Handle properties of the input form self.check_values = string_as_bool( input_elem.get("check_values", self.check_values ) ) self.nginx_upload = string_as_bool( input_elem.get( "nginx_upload", self.nginx_upload ) ) diff -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae -r 3686de5c8280c18c197323948c5e2ea79e642a0b test/functional/tools/collection_two_paired.xml --- a/test/functional/tools/collection_two_paired.xml +++ b/test/functional/tools/collection_two_paired.xml @@ -20,8 +20,8 @@ <param name="f2" type="data_collection" collection_type="paired" /></when><when value="list"> - <param name="f1" type="data_collection" collection_type="paired" /> - <param name="f2" type="data_collection" collection_type="paired" /> + <param name="f1" type="data_collection" collection_type="list" /> + <param name="f2" type="data_collection" collection_type="list" /></when></conditional></inputs> https://bitbucket.org/galaxy/galaxy-central/commits/dd39f52ba6fc/ Changeset: dd39f52ba6fc User: dannon Date: 2014-07-24 23:52:30 Summary: Cleanup in galaxy.util Affected #: 1 file diff -r 3686de5c8280c18c197323948c5e2ea79e642a0b -r dd39f52ba6fc0cea9328d1437842904ac1d020c9 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -48,10 +48,10 @@ from .inflection import Inflector, English inflector = Inflector(English) -log = logging.getLogger(__name__) +log = logging.getLogger(__name__) _lock = threading.RLock() -CHUNK_SIZE = 65536 #64k +CHUNK_SIZE = 65536 # 64k DATABASE_MAX_STRING_SIZE = 32768 DATABASE_MAX_STRING_SIZE_PRETTY = '32K' @@ -62,6 +62,7 @@ NULL_CHAR = '\000' BINARY_CHARS = [ NULL_CHAR ] + def is_multi_byte( chars ): for char in chars: try: @@ -69,18 +70,15 @@ except UnicodeDecodeError: # Probably binary return False - if wchartype.is_asian( char ) or \ - wchartype.is_full_width( char ) or \ - wchartype.is_kanji( char ) or \ - wchartype.is_hiragana( char ) or \ - wchartype.is_katakana( char ) or \ - wchartype.is_half_katakana( char ) or \ - wchartype.is_hangul( char ) or \ - wchartype.is_full_digit( char ) or \ - wchartype.is_full_letter( char ): + if ( wchartype.is_asian( char ) or wchartype.is_full_width( char ) or + wchartype.is_kanji( char ) or wchartype.is_hiragana( char ) or + wchartype.is_katakana( char ) or wchartype.is_half_katakana( char ) + or wchartype.is_hangul( char ) or wchartype.is_full_digit( char ) + or wchartype.is_full_letter( char )): return True return False + def is_binary( value, binary_chars=None ): """ File is binary if it contains a null-byte by default (e.g. behavior of grep, etc.). @@ -99,6 +97,7 @@ return True return False + def get_charset_from_http_headers( headers, default=None ): rval = headers.get('content-type', None ) if rval and 'charset=' in rval: @@ -107,16 +106,18 @@ return rval return default + def synchronized(func): """This wrapper will serialize access to 'func' to a single thread. Use it as a decorator.""" def caller(*params, **kparams): - _lock.acquire(True) # Wait + _lock.acquire(True) # Wait try: return func(*params, **kparams) finally: _lock.release() return caller + def file_iter(fname, sep=None): """ This generator iterates over a file and yields its lines @@ -131,6 +132,7 @@ if line and line[0] != '#': yield line.split(sep) + def file_reader( fp, chunk_size=CHUNK_SIZE ): """This generator yields the open fileobject in chunks (default 64k). Closes the file at the end""" while 1: @@ -140,6 +142,7 @@ yield data fp.close() + def unique_id(KEY_SIZE=128): """ Generates an unique id @@ -148,8 +151,8 @@ >>> len(set(ids)) 1000 """ - id = str( random.getrandbits( KEY_SIZE ) ) - return md5(id).hexdigest() + return md5(str( random.getrandbits( KEY_SIZE ) )).hexdigest() + def parse_xml(fname): """Returns a parsed xml tree""" @@ -163,6 +166,7 @@ tree = ElementTree.fromstring(xml_string) return tree + def xml_to_string( elem, pretty=False ): """Returns a string from an xml tree""" if pretty: @@ -170,12 +174,13 @@ try: return ElementTree.tostring( elem ) except TypeError, e: - #assume this is a comment + # we assume this is a comment if hasattr( elem, 'text' ): return "\n" % ( elem.text ) else: raise e + def xml_element_compare( elem1, elem2 ): if not isinstance( elem1, dict ): elem1 = xml_element_to_dict( elem1 ) @@ -183,9 +188,11 @@ elem2 = xml_element_to_dict( elem2 ) return elem1 == elem2 + def xml_element_list_compare( elem_list1, elem_list2 ): return [ xml_element_to_dict( elem ) for elem in elem_list1 ] == [ xml_element_to_dict( elem ) for elem in elem_list2 ] + def xml_element_to_dict( elem ): rval = {} if elem.attrib: @@ -220,7 +227,6 @@ return rval - def pretty_print_xml( elem, level=0 ): pad = ' ' i = "\n" + level * pad @@ -238,26 +244,28 @@ elem.tail = i + pad return elem + def get_file_size( value, default=None ): try: - #try built-in + # try built-in return os.path.getsize( value ) except: try: - #try built-in one name attribute + # try built-in one name attribute return os.path.getsize( value.name ) except: try: - #try tell() of end of object + # try tell() of end of object offset = value.tell() value.seek( 0, 2 ) rval = value.tell() value.seek( offset ) return rval except: - #return default value + # return default value return default + def shrink_stream_by_size( value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False ): rval = '' if get_file_size( value ) > size: @@ -292,6 +300,7 @@ rval += data return rval + def shrink_string_by_size( value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False ): if len( value ) > size: len_join_by = len( join_by ) @@ -311,29 +320,30 @@ value = "%s%s%s" % ( value[:left_index], join_by, value[-right_index:] ) return value + def pretty_print_json(json_data, is_json_string=False): if is_json_string: json_data = json.from_json_string(json_data) return json.to_json_string(json_data, sort_keys=True, indent=4) # characters that are valid -valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") +valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") # characters that are allowed but need to be escaped -mapped_chars = { '>' :'__gt__', - '<' :'__lt__', - "'" :'__sq__', - '"' :'__dq__', - '[' :'__ob__', - ']' :'__cb__', - '{' :'__oc__', - '}' :'__cc__', - '@' : '__at__', - '\n' : '__cn__', - '\r' : '__cr__', - '\t' : '__tc__', - '#' : '__pd__' - } +mapped_chars = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '@': '__at__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + def restore_text(text): """Restores sanitized text""" @@ -343,6 +353,7 @@ text = text.replace(value, key) return text + def sanitize_text(text): """ Restricts the characters that are allowed in text; accepts both strings @@ -353,6 +364,7 @@ elif isinstance( text, list ): return [ _sanitize_text_helper(t) for t in text ] + def _sanitize_text_helper(text): """Restricts the characters that are allowed in a string""" @@ -363,9 +375,10 @@ elif c in mapped_chars: out.append(mapped_chars[c]) else: - out.append('X') # makes debugging easier + out.append('X') # makes debugging easier return ''.join(out) + def sanitize_param(value): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): @@ -373,10 +386,12 @@ elif isinstance( value, list ): return map(sanitize_text, value) else: - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] + + def sanitize_for_filename( text, default=None ): """ Restricts the characters that are allowed in a filename portion; Returns default value or a unique id string if result is not a valid name. @@ -512,7 +527,7 @@ def __init__( self, params, sanitize=True ): if sanitize: for key, value in params.items(): - if key not in self.NEVER_SANITIZE and True not in [ key.endswith( "|%s" % nonsanitize_parameter ) for nonsanitize_parameter in self.NEVER_SANITIZE ]: #sanitize check both ungrouped and grouped parameters by name. Anything relying on NEVER_SANITIZE should be changed to not require this and NEVER_SANITIZE should be removed. + if key not in self.NEVER_SANITIZE and True not in [ key.endswith( "|%s" % nonsanitize_parameter ) for nonsanitize_parameter in self.NEVER_SANITIZE ]: # sanitize check both ungrouped and grouped parameters by name. Anything relying on NEVER_SANITIZE should be changed to not require this and NEVER_SANITIZE should be removed. self.__dict__[ key ] = sanitize_param( value ) else: self.__dict__[ key ] = value @@ -525,7 +540,7 @@ """ flat = [] for key, value in self.__dict__.items(): - if type(value) == type([]): + if isinstance(value, list): for v in value: flat.append( (key, v) ) else: @@ -551,16 +566,19 @@ def update(self, values): self.__dict__.update(values) + def rst_to_html( s ): """Convert a blob of reStructuredText to HTML""" log = logging.getLogger( "docutils" ) + class FakeStream( object ): def write( self, str ): if len( str ) > 0 and not str.isspace(): log.warn( str ) return unicodify( docutils.core.publish_string( s, - writer=docutils.writers.html4css1.Writer(), - settings_overrides={ "embed_stylesheet": False, "template": os.path.join(os.path.dirname(__file__), "docutils_template.txt"), "warning_stream": FakeStream() } ) ) + writer=docutils.writers.html4css1.Writer(), + settings_overrides={ "embed_stylesheet": False, "template": os.path.join(os.path.dirname(__file__), "docutils_template.txt"), "warning_stream": FakeStream() } ) ) + def xml_text(root, name=None): """Returns the text inside an element""" @@ -582,6 +600,8 @@ # asbool implementation pulled from PasteDeploy truthy = frozenset(['true', 'yes', 'on', 'y', 't', '1']) falsy = frozenset(['false', 'no', 'off', 'n', 'f', '0']) + + def asbool(obj): if isinstance(obj, basestring): obj = obj.strip().lower() @@ -600,6 +620,7 @@ else: return False + def string_as_bool_or_none( string ): """ Returns True, None or False based on the argument: @@ -618,6 +639,7 @@ else: return False + def listify( item, do_strip=False ): """ Make a single item a single item list, or return a list if passed a @@ -635,6 +657,7 @@ else: return [ item ] + def commaify(amount): orig = amount new = re.sub("^(-?\d+)(\d{3})", '\g<1>,\g<2>', amount) @@ -643,7 +666,8 @@ else: return commaify(new) -def roundify(amount, sfs = 2): + +def roundify(amount, sfs=2): """ Take a number in string form and truncate to 'sfs' significant figures. """ @@ -652,6 +676,7 @@ else: return amount[0:sfs] + '0'*(len(amount) - sfs) + def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ): """ Returns a unicode string or None @@ -691,6 +716,7 @@ def object_to_string( obj ): return binascii.hexlify( pickle.dumps( obj, 2 ) ) + def string_to_object( s ): return pickle.loads( binascii.unhexlify( s ) ) @@ -743,19 +769,23 @@ return False return True + def get_ucsc_by_build(build): sites = [] for site in ucsc_build_sites: if build in site['builds']: - sites.append((site['name'],site['url'])) + sites.append((site['name'], site['url'])) return sites + + def get_gbrowse_sites_by_build(build): sites = [] for site in gbrowse_build_sites: if build in site['builds']: - sites.append((site['name'],site['url'])) + sites.append((site['name'], site['url'])) return sites + def read_dbnames(filename): """ Read build names from file """ class DBNames( list ): @@ -764,30 +794,34 @@ db_names = DBNames() try: ucsc_builds = {} - man_builds = [] #assume these are integers + man_builds = [] # assume these are integers name_to_db_base = {} for line in open(filename): try: - if line[0:1] == "#": continue - fields = line.replace("\r","").replace("\n","").split("\t") - #Special case of unspecified build is at top of list + if line[0:1] == "#": + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") + # Special case of unspecified build is at top of list if fields[0] == "?": - db_names.insert(0,(fields[0],fields[1])) + db_names.insert(0, (fields[0], fields[1])) continue - try: #manual build (i.e. microbes) + try: # manual build (i.e. microbes) int(fields[0]) man_builds.append((fields[1], fields[0])) - except: #UCSC build + except: # UCSC build db_base = fields[0].rstrip('0123456789') if db_base not in ucsc_builds: ucsc_builds[db_base] = [] name_to_db_base[fields[1]] = db_base - #we want to sort within a species numerically by revision number + # we want to sort within a species numerically by revision number build_rev = re.compile(r'\d+$') - try: build_rev = int(build_rev.findall(fields[0])[0]) - except: build_rev = 0 - ucsc_builds[db_base].append((build_rev, fields[0],fields[1])) - except: continue + try: + build_rev = int(build_rev.findall(fields[0])[0]) + except: + build_rev = 0 + ucsc_builds[db_base].append((build_rev, fields[0], fields[1])) + except: + continue sort_names = name_to_db_base.keys() sort_names.sort() for name in sort_names: @@ -796,16 +830,18 @@ ucsc_builds[db_base].reverse() ucsc_builds[db_base] = [(build, name) for build_rev, build, name in ucsc_builds[db_base]] db_names = DBNames( db_names + ucsc_builds[db_base] ) - if len( db_names ) > 1 and len( man_builds ) > 0: db_names.append( ( db_names.default_value, '----- Additional Species Are Below -----' ) ) + if len( db_names ) > 1 and len( man_builds ) > 0: + db_names.append( ( db_names.default_value, '----- Additional Species Are Below -----' ) ) man_builds.sort() - man_builds = [(build, name) for name, build in man_builds] + man_builds = [(build, name) for name, build in man_builds] db_names = DBNames( db_names + man_builds ) except Exception, e: print "ERROR: Unable to read builds file:", e - if len(db_names)<1: + if len(db_names) < 1: db_names = DBNames( [( db_names.default_value, db_names.default_name )] ) return db_names + def read_ensembl( filename, ucsc ): """ Read Ensembl build names from file """ ucsc_builds = [] @@ -814,47 +850,55 @@ ensembl_builds = list() try: for line in open( filename ): - if line[0:1] in [ '#', '\t' ]: continue - fields = line.replace("\r","").replace("\n","").split("\t") - if fields[0] in ucsc_builds: continue + if line[0:1] in [ '#', '\t' ]: + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") + if fields[0] in ucsc_builds: + continue ensembl_builds.append( dict( dbkey=fields[0], release=fields[1], name=fields[2].replace( '_', ' ' ) ) ) except Exception, e: print "ERROR: Unable to read builds file:", e return ensembl_builds + def read_ncbi( filename ): """ Read NCBI build names from file """ ncbi_builds = list() try: for line in open( filename ): - if line[0:1] in [ '#', '\t' ]: continue - fields = line.replace("\r","").replace("\n","").split("\t") + if line[0:1] in [ '#', '\t' ]: + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") ncbi_builds.append( dict( dbkey=fields[0], name=fields[1] ) ) except Exception, e: print "ERROR: Unable to read builds file:", e return ncbi_builds + def read_build_sites( filename, check_builds=True ): """ read db names to ucsc mappings from file, this file should probably be merged with the one above """ build_sites = [] try: for line in open(filename): try: - if line[0:1] == "#": continue - fields = line.replace("\r","").replace("\n","").split("\t") + if line[0:1] == "#": + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") site_name = fields[0] site = fields[1] if check_builds: site_builds = fields[2].split(",") - site_dict = {'name':site_name, 'url':site, 'builds':site_builds} + site_dict = {'name': site_name, 'url': site, 'builds': site_builds} else: - site_dict = {'name':site_name, 'url':site} + site_dict = {'name': site_name, 'url': site} build_sites.append( site_dict ) - except: continue + except: + continue except: - print "ERROR: Unable to read builds for site file %s" %filename + print "ERROR: Unable to read builds for site file %s" % filename return build_sites + def relativize_symlinks( path, start=None, followlinks=False): for root, dirs, files in os.walk( path, followlinks=followlinks ): rel_start = None @@ -871,23 +915,26 @@ os.remove( symlink_file_name ) os.symlink( rel_path, symlink_file_name ) + def stringify_dictionary_keys( in_dict ): - #returns a new dictionary - #changes unicode keys into strings, only works on top level (does not recurse) - #unicode keys are not valid for expansion into keyword arguments on method calls + # returns a new dictionary + # changes unicode keys into strings, only works on top level (does not recurse) + # unicode keys are not valid for expansion into keyword arguments on method calls out_dict = {} for key, value in in_dict.iteritems(): out_dict[ str( key ) ] = value return out_dict + def recursively_stringify_dictionary_keys( d ): if isinstance(d, dict): - return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()]) + return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k, v in d.iteritems()]) elif isinstance(d, list): return [recursively_stringify_dictionary_keys(x) for x in d] else: return d + def mkstemp_ln( src, prefix='mkstemp_ln_' ): """ From tempfile._mkstemp_inner, generate a hard link in the same dir with a @@ -904,9 +951,10 @@ return (os.path.abspath(file)) except OSError, e: if e.errno == errno.EEXIST: - continue # try again + continue # try again raise - raise IOError, (errno.EEXIST, "No usable temporary file name found") + raise IOError(errno.EEXIST, "No usable temporary file name found") + def umask_fix_perms( path, umask, unmasked_perms, gid=None ): """ @@ -923,7 +971,7 @@ try: os.chmod( path, perms ) except Exception, e: - log.warning( 'Unable to honor umask (%s) for %s, tried to set: %s but mode remains %s, error was: %s' % ( oct( umask ), \ + log.warning( 'Unable to honor umask (%s) for %s, tried to set: %s but mode remains %s, error was: %s' % ( oct( umask ), path, oct( perms ), oct( stat.S_IMODE( st.st_mode ) ), @@ -939,11 +987,12 @@ except: desired_group = gid current_group = st.st_gid - log.warning( 'Unable to honor primary group (%s) for %s, group remains %s, error was: %s' % ( desired_group, \ + log.warning( 'Unable to honor primary group (%s) for %s, group remains %s, error was: %s' % ( desired_group, path, current_group, e ) ) + def docstring_trim(docstring): """Trimming python doc strings. Taken from: http://www.python.org/dev/peps/pep-0257/""" if not docstring: @@ -970,6 +1019,7 @@ # Return a single string: return '\n'.join(trimmed) + def nice_size(size): """ Returns a readably formatted string with the size @@ -989,14 +1039,15 @@ except: return '??? bytes' for ind, word in enumerate(words): - step = 1024 ** (ind + 1) + step = 1024 ** (ind + 1) if step > size: size = size / float(1024 ** ind) - if word == 'bytes': # No decimals for bytes + if word == 'bytes': # No decimals for bytes return "%d bytes" % size return "%.1f %s" % (size, word) return '??? bytes' + def size_to_bytes( size ): """ Returns a number of bytes if given a reasonably formatted string with the size @@ -1023,6 +1074,7 @@ elif multiple.startswith( 'b' ): return int( size ) + def send_mail( frm, to, subject, body, config ): """ Sends an email. @@ -1072,6 +1124,7 @@ s.sendmail( frm, to, msg.as_string() ) s.quit() + def force_symlink( source, link_name ): try: os.symlink( source, link_name ) @@ -1082,12 +1135,13 @@ else: raise e + def move_merge( source, target ): - #when using shutil and moving a directory, if the target exists, - #then the directory is placed inside of it - #if the target doesn't exist, then the target is made into the directory - #this makes it so that the target is always the target, and if it exists, - #the source contents are moved into the target + # when using shutil and moving a directory, if the target exists, + # then the directory is placed inside of it + # if the target doesn't exist, then the target is made into the directory + # this makes it so that the target is always the target, and if it exists, + # the source contents are moved into the target if os.path.isdir( source ) and os.path.exists( target ) and os.path.isdir( target ): for name in os.listdir( source ): move_merge( os.path.join( source, name ), os.path.join( target, name ) ) @@ -1103,7 +1157,7 @@ rv |= ord(x) ^ ord(y) return rv == 0 -galaxy_root_path = os.path.join(__path__[0], "..","..","..") +galaxy_root_path = os.path.join(__path__[0], "..", "..", "..") # The dbnames list is used in edit attributes and the upload tool dbnames = read_dbnames( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "builds.txt" ) ) @@ -1114,6 +1168,7 @@ gbrowse_build_sites = read_build_sites( os.path.join( galaxy_root_path, "tool-data", "shared", "gbrowse", "gbrowse_build_sites.txt" ) ) dlnames = dict(ucsc=ucsc_names, ensembl=ensembl_names, ncbi=ncbi_names) + def galaxy_directory(): return os.path.abspath(galaxy_root_path) https://bitbucket.org/galaxy/galaxy-central/commits/f03f9c5a5efc/ Changeset: f03f9c5a5efc User: dannon Date: 2014-07-24 23:53:37 Summary: Prevent redefinition of build_rev in list comprehension -- doesn't get used anyway Affected #: 1 file diff -r dd39f52ba6fc0cea9328d1437842904ac1d020c9 -r f03f9c5a5efc7fecb4ca131f66c441d55cc5d353 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -828,7 +828,7 @@ db_base = name_to_db_base[name] ucsc_builds[db_base].sort() ucsc_builds[db_base].reverse() - ucsc_builds[db_base] = [(build, name) for build_rev, build, name in ucsc_builds[db_base]] + ucsc_builds[db_base] = [(build, name) for _, build, name in ucsc_builds[db_base]] db_names = DBNames( db_names + ucsc_builds[db_base] ) if len( db_names ) > 1 and len( man_builds ) > 0: db_names.append( ( db_names.default_value, '----- Additional Species Are Below -----' ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for parsing tool <input> attributes when it contains no parameters.
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/832bbb3fa9fb/ Changeset: 832bbb3fa9fb User: dan Date: 2014-07-24 22:46:24 Summary: Fix for parsing tool <input> attributes when it contains no parameters. Affected #: 1 file diff -r 6f7d70f72da237e0d8d37a20b14a4eab1828803d -r 832bbb3fa9fb62d53f2be23cf880755fa2001f46 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1391,7 +1391,7 @@ # Load parameters (optional) input_elem = root.find("inputs") enctypes = set() - if input_elem: + if input_elem is not None: # Handle properties of the input form self.check_values = string_as_bool( input_elem.get("check_values", self.check_values ) ) self.nginx_upload = string_as_bool( input_elem.get( "nginx_upload", self.nginx_upload ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix in test tool demoing two collection params.
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6f7d70f72da2/ Changeset: 6f7d70f72da2 User: jmchilton Date: 2014-07-24 22:30:51 Summary: Bugfix in test tool demoing two collection params. Bit problematic functional tests passed despite this bug. Affected #: 1 file diff -r b2700f28372549a17f5ca03af60e86964665c3ff -r 6f7d70f72da237e0d8d37a20b14a4eab1828803d test/functional/tools/collection_two_paired.xml --- a/test/functional/tools/collection_two_paired.xml +++ b/test/functional/tools/collection_two_paired.xml @@ -20,8 +20,8 @@ <param name="f2" type="data_collection" collection_type="paired" /></when><when value="list"> - <param name="f1" type="data_collection" collection_type="paired" /> - <param name="f2" type="data_collection" collection_type="paired" /> + <param name="f1" type="data_collection" collection_type="list" /> + <param name="f2" type="data_collection" collection_type="list" /></when></conditional></inputs> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Add functional tool tests exercising multiple collection parameters at once.
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b2700f283725/ Changeset: b2700f283725 User: jmchilton Date: 2014-07-24 22:06:04 Summary: Add functional tool tests exercising multiple collection parameters at once. Affected #: 2 files diff -r 2be89285509068e862d27d41c3ba02c5a59e5b00 -r b2700f28372549a17f5ca03af60e86964665c3ff test/functional/tools/collection_two_paired.xml --- /dev/null +++ b/test/functional/tools/collection_two_paired.xml @@ -0,0 +1,69 @@ +<tool id="collection_two_paired" name="collection_two_paired" version="0.1.0"> + <command> + #if $kind.collection_type == "paired" + cat $kind.f1.forward $kind.f1['reverse'] >> $out1; + cat $kind.f2.forward $kind.f2['reverse'] >> $out1; + #else + #for $i, $_ in enumerate($kind.f1): + cat $kind.f1[$i] $kind.f2[$i] >> $out1; + #end for + #end if + </command> + <inputs> + <conditional name="kind"> + <param type="select" name="collection_type"> + <option value="paired">Paired Datasets</option> + <option value="list">List of Datasets</option> + </param> + <when value="paired"> + <param name="f1" type="data_collection" collection_type="paired" /> + <param name="f2" type="data_collection" collection_type="paired" /> + </when> + <when value="list"> + <param name="f1" type="data_collection" collection_type="paired" /> + <param name="f2" type="data_collection" collection_type="paired" /> + </when> + </conditional> + </inputs> + <outputs> + <data format="txt" name="out1" /> + </outputs> + <tests> + <test> + <conditional name="kind"> + <param name="collection_type" value="paired" /> + <param name="f1"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + <test> + <conditional name="kind"> + <param name="collection_type" value="list" /> + <param name="f1"> + <collection type="list"> + <element name="l11" value="simple_line.txt" /> + <element name="l12" value="simple_line.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="list"> + <element name="l21" value="simple_line_alternative.txt" /> + <element name="l22" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + </tests> +</tool> diff -r 2be89285509068e862d27d41c3ba02c5a59e5b00 -r b2700f28372549a17f5ca03af60e86964665c3ff test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -21,4 +21,5 @@ <tool file="collection_paired_test.xml" /><tool file="collection_nested_test.xml" /><tool file="collection_mixed_param.xml" /> + <tool file="collection_two_paired.xml" /></toolbox> \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/20dcff34ee3b/ Changeset: 20dcff34ee3b User: jmchilton Date: 2014-07-24 21:30:11 Summary: More collection-y state update tests. Affected #: 1 file diff -r 9d8967752787556b3d814e086fe2bd9eb1992170 -r 20dcff34ee3bb598969c51a7c5b53e9dd679c111 test/unit/tools/test_execution.py --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -16,13 +16,12 @@ eggs.require( "Paste" ) from paste import httpexceptions -# Tool with a repeat parameter, to test state update. -REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"> +BASE_REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"><command>echo "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1</command><inputs><param type="text" name="param1" value="" /><repeat name="repeat1" label="Repeat 1"> - <param type="text" name="param2" value="" /> + %s </repeat></inputs><outputs> @@ -31,6 +30,10 @@ </tool> ''' +# Tool with a repeat parameter, to test state update. +REPEAT_TOOL_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="text" name="param2" value="" />''' +REPEAT_COLLECTION_PARAM_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="data_collection" name="param2" collection_type="paired" />''' + class ToolExecutionTestCase( TestCase, tools_support.UsesApp, tools_support.UsesTools ): @@ -287,13 +290,48 @@ } ) self.__assert_exeuted( template, template_vars ) - def __history_dataset_collection_for( self, hdas, id=1234 ): - collection = galaxy.model.DatasetCollection() + def test_subcollection_multirun_with_state_updates( self ): + self._init_tool( REPEAT_COLLECTION_PARAM_CONTENTS ) + hda1, hda2 = self.__add_dataset( 1 ), self.__add_dataset( 2 ) + collection = self.__history_dataset_collection_for( [ hda1, hda2 ], collection_type="list:paired" ) + collection_id = self.app.security.encode_id( collection.id ) + self.app.dataset_collections_service = Bunch( + match_collections=lambda collections: None + ) + template, template_vars = self.__handle_with_incoming( + repeat1_add="dummy", + ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert len( state.inputs[ "repeat1" ] ) == 1 + template, template_vars = self.__handle_with_incoming( state, **{ + "repeat1_0|param2|__collection_multirun__": "%s|paired" % collection_id, + "repeat1_add": "dummy", + } ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert state.inputs[ "repeat1" ][ 0 ][ "param2|__collection_multirun__" ] == "%s|paired" % collection_id + + def __history_dataset_collection_for( self, hdas, collection_type="list", id=1234 ): + collection = galaxy.model.DatasetCollection( + collection_type=collection_type, + ) to_element = lambda hda: galaxy.model.DatasetCollectionElement( collection=collection, element=hda, ) - collection.datasets = map(to_element, hdas) + elements = map(to_element, hdas) + if collection_type == "list:paired": + paired_collection = galaxy.model.DatasetCollection( + collection_type="paired", + ) + paired_collection.elements = elements + list_dce = galaxy.model.DatasetCollectionElement( + collection=collection, + element=paired_collection, + ) + elements = [ list_dce ] + + collection.elements = elements + history_dataset_collection_association = galaxy.model.HistoryDatasetCollectionAssociation( id=id, collection=collection, @@ -349,13 +387,13 @@ self.history.datasets.append( hda ) return hda - def __add_collection_dataset( self, id, *hdas ): + def __add_collection_dataset( self, id, collection_type="paired", *hdas ): hdca = galaxy.model.HistoryDatasetCollectionAssociation() hdca.id = id collection = galaxy.model.DatasetCollection() hdca.collection = collection collection.elements = [ galaxy.model.DatasetCollectionElement(element=self.__add_dataset( 1 )) ] - + collection.type = collection_type self.trans.sa_session.model_objects[ galaxy.model.HistoryDatasetCollectionAssociation ][ id ] = hdca self.history.dataset_collections.append( hdca ) return hdca https://bitbucket.org/galaxy/galaxy-central/commits/9c2cbf6c7f3d/ Changeset: 9c2cbf6c7f3d User: jmchilton Date: 2014-07-24 21:30:11 Summary: Fix multirun and collection multirun state updates in tool param GUI. 4139740 fixed backend and had tests so I guess I didn't even manually test, turns out tool parameter GUI generation needed to fixed for these changes though. Affected #: 1 file diff -r 20dcff34ee3bb598969c51a7c5b53e9dd679c111 -r 9c2cbf6c7f3db7cb59ebd53b9c2829404077993a lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1712,10 +1712,15 @@ if self.__display_multirun_option(): # Select multiple datasets, run multiple jobs. multirun_key = "%s|__multirun__" % self.name + collection_multirun_key = "%s|__collection_multirun__" % self.name if multirun_key in (other_values or {}): multirun_value = listify( other_values[ multirun_key ] ) if multirun_value and len( multirun_value ) > 1: default_field = "select_multiple" + elif collection_multirun_key in (other_values or {}): + multirun_value = listify( other_values[ collection_multirun_key ] ) + if multirun_value: + default_field = "select_collection" else: multirun_value = value multi_dataset_matcher = DatasetMatcher( trans, self, multirun_value, other_values ) @@ -2014,9 +2019,17 @@ default_field = "select_single_collection" fields = odict() + collection_multirun_key = "%s|__collection_multirun__" % self.name + if collection_multirun_key in (other_values or {}): + multirun_value = other_values[ collection_multirun_key ] + if multirun_value: + default_field = "select_map_over_collections" + else: + multirun_value = value + history = self._get_history( trans ) fields[ "select_single_collection" ] = self._get_single_collection_field( trans=trans, history=history, value=value, other_values=other_values ) - fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=value, other_values=other_values ) + fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=multirun_value, other_values=other_values ) return self._switch_fields( fields, default_field=default_field ) https://bitbucket.org/galaxy/galaxy-central/commits/2be892855090/ Changeset: 2be892855090 User: jmchilton Date: 2014-07-24 21:30:11 Summary: Small UI fix for data param multirun options. Index-based logic wasn't updated after made it so certain options wouldn't be visible if they were not valid for a given history. Affected #: 1 file diff -r 9c2cbf6c7f3db7cb59ebd53b9c2829404077993a -r 2be89285509068e862d27d41c3ba02c5a59e5b00 static/scripts/galaxy.tools.js --- a/static/scripts/galaxy.tools.js +++ b/static/scripts/galaxy.tools.js @@ -86,7 +86,7 @@ }).attr( 'title', selectionType['select_by'] - ); + ).data( "index", iIndex ); view.formRow().find( "label" ).append( button ); } }); @@ -114,11 +114,13 @@ } else { $("div#remap-row").css("display", "none"); } - this.formRow().find( "i" ).each(function(index, iElement) { + this.formRow().find( "i" ).each(function(_, iElement) { + var $iElement = $(iElement); + var index = $iElement.data("index"); if(index == enableIndex) { - $(iElement).css('color', 'black'); + $iElement.css('color', 'black'); } else { - $(iElement).css('color', 'Gray'); + $iElement.css('color', 'Gray'); } }); var $select = this.$( "select" ); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Trackster: resolve name collision due to refactoring.
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9d8967752787/ Changeset: 9d8967752787 User: jgoecks Date: 2014-07-24 15:26:39 Summary: Trackster: resolve name collision due to refactoring. Affected #: 1 file diff -r 6fbe4d95a8dc64bd222dbf1170bdadcc5321e855 -r 9d8967752787556b3d814e086fe2bd9eb1992170 static/scripts/viz/trackster.js --- a/static/scripts/viz/trackster.js +++ b/static/scripts/viz/trackster.js @@ -103,8 +103,9 @@ var self = this, menu = create_icon_buttons_menu([ { icon_class: 'plus-button', title: 'Add tracks', on_click: function() { - visualization.select_datasets(galaxy_config.root + "visualization/list_current_history_datasets", galaxy_config.root + "api/datasets", { 'f-dbkey': view.dbkey }, function(tracks) { - _.each(tracks, function(track) { + visualization.select_datasets(galaxy_config.root + "visualization/list_current_history_datasets", galaxy_config.root + "api/datasets", { 'f-dbkey': view.dbkey }, + function(new_tracks) { + _.each(new_tracks, function(track) { view.add_drawable( tracks.object_from_template(track, view, view) ); }); }); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix missing import in the Tool Shed's ToolValidator.
by commits-noreply＠bitbucket.org 23 Jul '14

23 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6fbe4d95a8dc/ Changeset: 6fbe4d95a8dc User: greg Date: 2014-07-23 22:31:59 Summary: Fix missing import in the Tool Shed's ToolValidator. Affected #: 1 file diff -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 -r 6fbe4d95a8dc64bd222dbf1170bdadcc5321e855 lib/tool_shed/tools/tool_validator.py --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -12,6 +12,7 @@ from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util +from tool_shed.util import xml_util log = logging.getLogger( __name__ ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 23 Jul '14

23 Jul '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2ae881148949/ Changeset: 2ae881148949 User: dannon Date: 2014-07-23 21:41:14 Summary: Fix 'decod_id' invocation to use the correct function when deleting user addresses. Affected #: 1 file diff -r 053943b668af334c480b464fe7351b50284b7099 -r 2ae881148949596af9c30f2c559dc8f627dda402 lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -1514,7 +1514,7 @@ @web.expose def delete_address( self, trans, cntrller, address_id=None, user_id=None ): try: - user_address = trans.sa_session.query( trans.app.model.UserAddress ).get( trans.security.decod_id( address_id ) ) + user_address = trans.sa_session.query( trans.app.model.UserAddress ).get( trans.security.decode_id( address_id ) ) except: message = 'Invalid address is (%s)' % address_id status = 'error' https://bitbucket.org/galaxy/galaxy-central/commits/4974ffd8b2cb/ Changeset: 4974ffd8b2cb User: dannon Date: 2014-07-23 21:42:23 Summary: Fix another variable definition issue in user address deletion. Affected #: 1 file diff -r 2ae881148949596af9c30f2c559dc8f627dda402 -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 lib/galaxy/webapps/galaxy/controllers/user.py --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -1522,7 +1522,7 @@ user_address.deleted = True trans.sa_session.add( user_address ) trans.sa_session.flush() - 'Address (%s) deleted' % user_address.desc + message = 'Address (%s) deleted' % user_address.desc status = 'done' return trans.response.send_redirect( web.url_for( controller='user', action='manage_user_info', https://bitbucket.org/galaxy/galaxy-central/commits/7aade8875ea4/ Changeset: 7aade8875ea4 User: dannon Date: 2014-07-23 22:27:42 Summary: Merge. Affected #: 60 files diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -12,7 +12,6 @@ import tool_shed.repository_types.util as rt_util from tool_shed.util import common_util -from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import readme_util @@ -25,6 +24,7 @@ from tool_shed.galaxy_install import dependency_display from tool_shed.galaxy_install import install_manager +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.grids import admin_toolshed_grids from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager @@ -256,17 +256,17 @@ dmh.remove_from_data_manager( tool_shed_repository ) if tool_shed_repository.includes_datatypes: # Deactivate proprietary datatypes. - installed_repository_dict = datatype_util.load_installed_datatypes( trans.app, - tool_shed_repository, - repository_install_dir, - deactivate=True ) + cdl = custom_datatype_manager.CustomDatatypeLoader( trans.app ) + installed_repository_dict = cdl.load_installed_datatypes( tool_shed_repository, + repository_install_dir, + deactivate=True ) if installed_repository_dict: converter_path = installed_repository_dict.get( 'converter_path' ) if converter_path is not None: - datatype_util.load_installed_datatype_converters( trans.app, installed_repository_dict, deactivate=True ) + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=True ) display_path = installed_repository_dict.get( 'display_path' ) if display_path is not None: - datatype_util.load_installed_display_applications( trans.app, installed_repository_dict, deactivate=True ) + cdl.load_installed_display_applications( installed_repository_dict, deactivate=True ) if remove_from_disk_checked: try: # Remove the repository from disk. diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -195,6 +195,7 @@ class HistoryController( BaseUIController, SharableMixin, UsesAnnotations, UsesItemRatings, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, ExportsHistoryMixin, ImportsHistoryMixin ): + def __init__( self, app ): super( HistoryController, self ).__init__( app ) self.mgrs = util.bunch.Bunch( diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/galaxy/webapps/galaxy/controllers/page.py --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -1,5 +1,6 @@ from sqlalchemy import desc, and_ from galaxy import model, web +from galaxy import managers from galaxy.web import error, url_for from galaxy.model.item_attrs import UsesItemRatings from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesHistoryMixin, UsesStoredWorkflowMixin, UsesVisualizationMixin @@ -285,6 +286,12 @@ _page_selection_grid = PageSelectionGrid() _visualization_selection_grid = VisualizationSelectionGrid() + def __init__( self, app ): + super( PageController, self ).__init__( app ) + self.mgrs = util.bunch.Bunch( + histories=managers.histories.HistoryManager() + ) + @web.expose @web.require_login() def list( self, trans, *args, **kwargs ): @@ -718,6 +725,7 @@ """ Returns html suitable for embedding in another page. """ + #TODO: should be moved to history controller and/or called via ajax from the template history = self.get_history( trans, id, False, True ) if not history: return None @@ -729,15 +737,22 @@ hda_dicts = [] datasets = self.get_history_datasets( trans, history ) - for hda in datasets: - hda_dict = self.get_hda_dict( trans, hda ) - hda_dict[ 'annotation' ] = self.get_item_annotation_str( trans.sa_session, history.user, hda ) - hda_dicts.append( hda_dict ) - history_dict = self.get_history_dict( trans, history, hda_dictionaries=hda_dicts ) - history_dict[ 'annotation' ] = history.annotation + #for hda in datasets: + # hda_dict = self.get_hda_dict( trans, hda ) + # hda_dict[ 'annotation' ] = self.get_item_annotation_str( trans.sa_session, history.user, hda ) + # hda_dicts.append( hda_dict ) + #history_dict = self.get_history_dict( trans, history, hda_dictionaries=hda_dicts ) + #history_dict[ 'annotation' ] = history.annotation + + # include all datasets: hidden, deleted, and purged + #TODO!: doubled query (hda_dictionaries + datasets) + history_data = self.mgrs.histories._get_history_data( trans, history ) + history_dictionary = history_data[ 'history' ] + hda_dictionaries = history_data[ 'contents' ] + history_dictionary[ 'annotation' ] = history.annotation filled = trans.fill_template( "history/embed.mako", item=history, item_data=datasets, - user_is_owner=user_is_owner, history_dict=history_dict, hda_dicts=hda_dicts ) + user_is_owner=user_is_owner, history_dict=history_dictionary, hda_dicts=hda_dictionaries ) return filled def _get_embed_html( self, trans, item_class, item_id ): diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1062,7 +1062,7 @@ action='browse_repositories', message=message, status=status ) ) - name = kwd.get( 'name', '' ) + name = kwd.get( 'name', '' ).strip() description = kwd.get( 'description', '' ) long_description = kwd.get( 'long_description', '' ) category_ids = util.listify( kwd.get( 'category_id', '' ) ) @@ -1262,7 +1262,11 @@ repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) metadata = repository_metadata.metadata toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + # Initialize the repository dependency RelationBuilder. rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + # Work-around to ensure repositories that contain packages needed only for compiling + # a dependent package are included in the capsule. + rb.set_filter_dependencies_needed_for_compiling( False ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if repository_dependencies: diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -245,9 +245,13 @@ repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, self.repository_id, self.changeset_revision ) - # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. + # Get a dictionary of all repositories upon which the contents of the current + # repository_metadata record depend. toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) rb = RelationBuilder( self.app, repository, repository_metadata, toolshed_base_url ) + # Work-around to ensure repositories that contain packages needed only for compiling + # a dependent package are included in the capsule. + rb.set_filter_dependencies_needed_for_compiling( False ) repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() repo = hg_util.get_repo_for_repository( self.app, repository=self.repository, diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/tool_shed/dependencies/repository/relation_builder.py --- a/lib/tool_shed/dependencies/repository/relation_builder.py +++ b/lib/tool_shed/dependencies/repository/relation_builder.py @@ -23,6 +23,11 @@ self.handled_key_rd_dicts = [] self.key_rd_dicts_to_be_processed = [] self.tool_shed_url = tool_shed_url + # This is a temporary work-around for handling repository dependencies that are needed + # only if compiling a dependent package. This value should be True unless exporting + # a repository capsule, in which case the set_filter_dependencies_needed_for_compiling() + # function is called. + self.filter_dependencies_needed_for_compiling = True def can_add_to_key_rd_dicts( self, key_rd_dict, key_rd_dicts ): """Handle the case where an update to the changeset revision was done.""" @@ -389,9 +394,12 @@ current_repository_key_rd_dicts = \ self.get_updated_changeset_revisions_for_repository_dependencies( current_repository_key_rd_dicts ) for key_rd_dict in current_repository_key_rd_dicts: - # Filter out repository dependencies that are required only if compiling the dependent - # repository's tool dependency. - key_rd_dict = self.filter_only_if_compiling_contained_td( key_rd_dict ) + if self.filter_dependencies_needed_for_compiling: + # Filter out repository dependencies that are required only if compiling the dependent + # repository's tool dependency. + # TODO: this temporary work-around should be removed when the underlying framework + # support for handling only_if_compiling_contained_td-flagged repositories is completed. + key_rd_dict = self.filter_only_if_compiling_contained_td( key_rd_dict ) if key_rd_dict: is_circular = False in_handled_key_rd_dicts = self.in_key_rd_dicts( key_rd_dict, self.handled_key_rd_dicts ) @@ -483,6 +491,9 @@ clean_key_rd_dicts.append( new_key_rd_dict ) return clean_key_rd_dicts + def set_filter_dependencies_needed_for_compiling( self, value ): + self.filter_dependencies_needed_for_compiling = asbool( value ) + def update_circular_repository_dependencies( self, repository_key, repository_dependency, repository_dependencies ): repository_dependency_as_key = self.get_repository_dependency_as_key( repository_dependency ) repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py @@ -0,0 +1,218 @@ +import logging +import os +import tempfile + +from galaxy.util import asbool + +from tool_shed.util import basic_util +from tool_shed.util import hg_util +from tool_shed.util import tool_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import xml_util + +log = logging.getLogger( __name__ ) + + +class CustomDatatypeLoader( object ): + + def __init__( self, app ): + self.app = app + + def alter_config_and_load_prorietary_datatypes( self, datatypes_config, relative_install_dir, + deactivate=False, override=True ): + """ + Parse a custom datatypes config (a datatypes_conf.xml file included in an installed + tool shed repository) and add information to appropriate element attributes that will + enable custom datatype class modules, datatypes converters and display applications + to be discovered and properly imported by the datatypes registry. The value of override + will be False when a tool shed repository is being installed. Since installation is + occurring after the datatypes registry has been initialized, the registry's contents + cannot be overridden by conflicting data types. + """ + tree, error_message = xml_util.parse_xml( datatypes_config ) + if tree is None: + return None, None + datatypes_config_root = tree.getroot() + registration = datatypes_config_root.find( 'registration' ) + if registration is None: + # We have valid XML, but not a valid custom datatypes definition. + return None, None + sniffers = datatypes_config_root.find( 'sniffers' ) + converter_path, display_path = self.get_converter_and_display_paths( registration, + relative_install_dir ) + if converter_path: + # Path to datatype converters + registration.attrib[ 'proprietary_converter_path' ] = converter_path + if display_path: + # Path to datatype display applications + registration.attrib[ 'proprietary_display_path' ] = display_path + relative_path_to_datatype_file_name = None + datatype_files = datatypes_config_root.find( 'datatype_files' ) + datatype_class_modules = [] + if datatype_files is not None: + # The <datatype_files> tag set contains any number of <datatype_file> tags. + # <datatype_files> + # <datatype_file name="gmap.py"/> + # <datatype_file name="metagenomics.py"/> + # </datatype_files> + # We'll add attributes to the datatype tag sets so that the modules can be properly imported + # by the datatypes registry. + for elem in datatype_files.findall( 'datatype_file' ): + datatype_file_name = elem.get( 'name', None ) + if datatype_file_name: + # Find the file in the installed repository. + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == datatype_file_name: + datatype_class_modules.append( os.path.join( root, name ) ) + break + break + if datatype_class_modules: + for relative_path_to_datatype_file_name in datatype_class_modules: + datatype_file_name_path, datatype_file_name = os.path.split( relative_path_to_datatype_file_name ) + for elem in registration.findall( 'datatype' ): + # Handle 'type' attribute which should be something like one of the following: + # type="gmap:GmapDB" + # type="galaxy.datatypes.gmap:GmapDB" + dtype = elem.get( 'type', None ) + if dtype: + fields = dtype.split( ':' ) + proprietary_datatype_module = fields[ 0 ] + if proprietary_datatype_module.find( '.' ) >= 0: + # Handle the case where datatype_module is "galaxy.datatypes.gmap". + proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ] + # The value of proprietary_path must be an absolute path due to job_working_directory. + elem.attrib[ 'proprietary_path' ] = os.path.abspath( datatype_file_name_path ) + elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module + # Temporarily persist the custom datatypes configuration file so it can be loaded into the + # datatypes registry. + fd, proprietary_datatypes_config = tempfile.mkstemp( prefix="tmp-toolshed-acalpd" ) + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<datatypes>\n' ) + os.write( fd, '%s' % xml_util.xml_to_string( registration ) ) + if sniffers is not None: + os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) ) + os.write( fd, '</datatypes>\n' ) + os.close( fd ) + os.chmod( proprietary_datatypes_config, 0644 ) + # Load custom datatypes + self.app.datatypes_registry.load_datatypes( root_dir=self.app.config.root, + config=proprietary_datatypes_config, + deactivate=deactivate, + override=override ) + if deactivate: + # Reload the upload tool to eliminate deactivated datatype extensions from the file_type + # select list. + tool_util.reload_upload_tools( self.app ) + else: + self.append_to_datatypes_registry_upload_file_formats( registration ) + tool_util.reload_upload_tools( self.app ) + if datatype_files is not None: + try: + os.unlink( proprietary_datatypes_config ) + except: + pass + return converter_path, display_path + + def append_to_datatypes_registry_upload_file_formats( self, elem ): + # See if we have any datatypes that should be displayed in the upload tool's file_type select list. + for datatype_elem in elem.findall( 'datatype' ): + extension = datatype_elem.get( 'extension', None ) + display_in_upload = datatype_elem.get( 'display_in_upload', None ) + if extension is not None and display_in_upload is not None: + display_in_upload = asbool( str( display_in_upload ) ) + if display_in_upload and extension not in self.app.datatypes_registry.upload_file_formats: + self.app.datatypes_registry.upload_file_formats.append( extension ) + + def create_repository_dict_for_proprietary_datatypes( self, tool_shed, name, owner, installed_changeset_revision, + tool_dicts, converter_path=None, display_path=None ): + return dict( tool_shed=tool_shed, + repository_name=name, + repository_owner=owner, + installed_changeset_revision=installed_changeset_revision, + tool_dicts=tool_dicts, + converter_path=converter_path, + display_path=display_path ) + + def get_converter_and_display_paths( self, registration_elem, relative_install_dir ): + """ + Find the relative path to data type converters and display applications included + in installed tool shed repositories. + """ + converter_path = None + display_path = None + for elem in registration_elem.findall( 'datatype' ): + if not converter_path: + # If any of the <datatype> tag sets contain <converter> tags, set the converter_path + # if it is not already set. This requires developers to place all converters in the + # same subdirectory within the repository hierarchy. + for converter in elem.findall( 'converter' ): + converter_config = converter.get( 'file', None ) + if converter_config: + converter_config_file_name = basic_util.strip_path( converter_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == converter_config_file_name: + # The value of converter_path must be absolute due to job_working_directory. + converter_path = os.path.abspath( root ) + break + if converter_path: + break + if not display_path: + # If any of the <datatype> tag sets contain <display> tags, set the display_path + # if it is not already set. This requires developers to place all display acpplications + # in the same subdirectory within the repository hierarchy. + for display_app in elem.findall( 'display' ): + display_config = display_app.get( 'file', None ) + if display_config: + display_config_file_name = basic_util.strip_path( display_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == display_config_file_name: + # The value of display_path must be absolute due to job_working_directory. + display_path = os.path.abspath( root ) + break + if display_path: + break + if converter_path and display_path: + break + return converter_path, display_path + + def load_installed_datatype_converters( self, installed_repository_dict, deactivate=False ): + """Load or deactivate proprietary datatype converters.""" + self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, + installed_repository_dict=installed_repository_dict, + deactivate=deactivate ) + + def load_installed_datatypes( self, repository, relative_install_dir, deactivate=False ): + """ + Load proprietary datatypes and return information needed for loading custom + datatypes converters and display applications later. + """ + metadata = repository.metadata + repository_dict = None + datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir ) + if datatypes_config: + converter_path, display_path = \ + self.alter_config_and_load_prorietary_datatypes( datatypes_config, + relative_install_dir, + deactivate=deactivate ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = \ + self.create_repository_dict_for_proprietary_datatypes( tool_shed=repository.tool_shed, + name=repository.name, + owner=repository.owner, + installed_changeset_revision=repository.installed_changeset_revision, + tool_dicts=metadata.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + return repository_dict + + def load_installed_display_applications( self, installed_repository_dict, deactivate=False ): + """Load or deactivate custom datatype display applications.""" + self.app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, + deactivate=deactivate ) diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -18,7 +18,6 @@ from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc @@ -26,6 +25,7 @@ from tool_shed.util import tool_util from tool_shed.util import xml_util +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder @@ -594,18 +594,19 @@ files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) # Load data types required by tools. + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) converter_path, display_path = \ - datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, files_dir, override=False ) + cdl.alter_config_and_load_prorietary_datatypes( datatypes_config, files_dir, override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = \ - datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) + cdl.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, + name=tool_shed_repository.name, + owner=tool_shed_repository.owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) if converter_path: # Load proprietary datatype converters self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, installed_repository_dict=repository_dict ) diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -7,12 +7,12 @@ from galaxy import util from tool_shed.util import common_util from tool_shed.util import container_util -from tool_shed.util import datatype_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import xml_util from galaxy.model.orm import and_ +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tools import data_manager @@ -114,17 +114,17 @@ else: repository_install_dir = os.path.abspath( relative_install_dir ) # Activate proprietary datatypes. - installed_repository_dict = datatype_util.load_installed_datatypes( self.app, - repository, - repository_install_dir, - deactivate=False ) + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) + installed_repository_dict = cdl.load_installed_datatypes( repository, + repository_install_dir, + deactivate=False ) if installed_repository_dict: converter_path = installed_repository_dict.get( 'converter_path' ) if converter_path is not None: - datatype_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=False ) + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=False ) display_path = installed_repository_dict.get( 'display_path' ) if display_path is not None: - datatype_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=False ) + cdl.load_installed_display_applications( installed_repository_dict, deactivate=False ) def add_entry_to_installed_repository_dependencies_of_installed_repositories( self, repository ): """ @@ -732,22 +732,24 @@ self.add_entry_to_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies( tool_dependency ) def load_proprietary_datatypes( self ): + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) for tool_shed_repository in self.context.query( self.install_model.ToolShedRepository ) \ - .filter( and_( self.install_model.ToolShedRepository.table.c.includes_datatypes==True, - self.install_model.ToolShedRepository.table.c.deleted==False ) ) \ - .order_by( self.install_model.ToolShedRepository.table.c.id ): + .filter( and_( self.install_model.ToolShedRepository.table.c.includes_datatypes==True, + self.install_model.ToolShedRepository.table.c.deleted==False ) ) \ + .order_by( self.install_model.ToolShedRepository.table.c.id ): relative_install_dir = self.get_repository_install_dir( tool_shed_repository ) if relative_install_dir: - installed_repository_dict = datatype_util.load_installed_datatypes( self.app, tool_shed_repository, relative_install_dir ) + installed_repository_dict = cdl.load_installed_datatypes( tool_shed_repository, relative_install_dir ) if installed_repository_dict: self.installed_repository_dicts.append( installed_repository_dict ) def load_proprietary_converters_and_display_applications( self, deactivate=False ): + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) for installed_repository_dict in self.installed_repository_dicts: if installed_repository_dict[ 'converter_path' ]: - datatype_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=deactivate ) + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=deactivate ) if installed_repository_dict[ 'display_path' ]: - datatype_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=deactivate ) + cdl.load_installed_display_applications( installed_repository_dict, deactivate=deactivate ) def purge_repository( self, repository ): """Purge a repository with status New (a white ghost) from the database.""" diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -14,6 +14,7 @@ from galaxy.util.odict import odict from tool_shed.galaxy_install import install_manager +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.tools import tool_panel_manager @@ -22,7 +23,6 @@ from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import datatype_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util @@ -492,6 +492,7 @@ print '\nThe ToolMigrationManager returned the following error while installing tool dependency ', installed_tool_dependency.name, ':' print installed_tool_dependency.error_message, '\n\n' if 'datatypes' in metadata_dict: + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True @@ -499,21 +500,27 @@ self.app.install_model.context.flush() work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-hrc" ) datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir ) - # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started - # after this installation completes. - converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir + # Load proprietary data types required by tools. The value of override is not + # important here since the Galaxy server will be started after this installation + #completes. + converter_path, display_path = \ + cdl.alter_config_and_load_prorietary_datatypes( datatypes_config, + repo_install_dir, + override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. - repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed_url, - name=tool_shed_repository.name, - owner=self.repository_owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) + repository_dict = \ + cdl.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed_url, + name=tool_shed_repository.name, + owner=self.repository_owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) if converter_path: # Load proprietary datatype converters - self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict ) + self.app.datatypes_registry.load_datatype_converters( self.toolbox, + installed_repository_dict=repository_dict ) if display_path: # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 lib/tool_shed/util/datatype_util.py --- a/lib/tool_shed/util/datatype_util.py +++ /dev/null @@ -1,187 +0,0 @@ -import logging -import os -import tempfile -from galaxy import eggs -from galaxy.util import asbool -from tool_shed.util import basic_util -from tool_shed.util import hg_util -from tool_shed.util import tool_util -from tool_shed.util import xml_util -import tool_shed.util.shed_util_common as suc - -log = logging.getLogger( __name__ ) - -def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False, override=True ): - """ - Parse a proprietary datatypes config (a datatypes_conf.xml file included in an installed tool shed repository) and - add information to appropriate element attributes that will enable proprietary datatype class modules, datatypes converters - and display applications to be discovered and properly imported by the datatypes registry. The value of override will - be False when a tool shed repository is being installed. Since installation is occurring after the datatypes registry - has been initialized, the registry's contents cannot be overridden by conflicting data types. - """ - tree, error_message = xml_util.parse_xml( datatypes_config ) - if tree is None: - return None, None - datatypes_config_root = tree.getroot() - registration = datatypes_config_root.find( 'registration' ) - if registration is None: - # We have valid XML, but not a valid proprietary datatypes definition. - return None, None - sniffers = datatypes_config_root.find( 'sniffers' ) - converter_path, display_path = get_converter_and_display_paths( registration, relative_install_dir ) - if converter_path: - # Path to datatype converters - registration.attrib[ 'proprietary_converter_path' ] = converter_path - if display_path: - # Path to datatype display applications - registration.attrib[ 'proprietary_display_path' ] = display_path - relative_path_to_datatype_file_name = None - datatype_files = datatypes_config_root.find( 'datatype_files' ) - datatype_class_modules = [] - if datatype_files is not None: - # The <datatype_files> tag set contains any number of <datatype_file> tags. - # <datatype_files> - # <datatype_file name="gmap.py"/> - # <datatype_file name="metagenomics.py"/> - # </datatype_files> - # We'll add attributes to the datatype tag sets so that the modules can be properly imported by the datatypes registry. - for elem in datatype_files.findall( 'datatype_file' ): - datatype_file_name = elem.get( 'name', None ) - if datatype_file_name: - # Find the file in the installed repository. - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == datatype_file_name: - datatype_class_modules.append( os.path.join( root, name ) ) - break - break - if datatype_class_modules: - for relative_path_to_datatype_file_name in datatype_class_modules: - datatype_file_name_path, datatype_file_name = os.path.split( relative_path_to_datatype_file_name ) - for elem in registration.findall( 'datatype' ): - # Handle 'type' attribute which should be something like one of the following: - # type="gmap:GmapDB" - # type="galaxy.datatypes.gmap:GmapDB" - dtype = elem.get( 'type', None ) - if dtype: - fields = dtype.split( ':' ) - proprietary_datatype_module = fields[ 0 ] - if proprietary_datatype_module.find( '.' ) >= 0: - # Handle the case where datatype_module is "galaxy.datatypes.gmap". - proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ] - # The value of proprietary_path must be an absolute path due to job_working_directory. - elem.attrib[ 'proprietary_path' ] = os.path.abspath( datatype_file_name_path ) - elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module - # Temporarily persist the proprietary datatypes configuration file so it can be loaded into the datatypes registry. - fd, proprietary_datatypes_config = tempfile.mkstemp( prefix="tmp-toolshed-acalpd" ) - os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, '<datatypes>\n' ) - os.write( fd, '%s' % xml_util.xml_to_string( registration ) ) - if sniffers is not None: - os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) ) - os.write( fd, '</datatypes>\n' ) - os.close( fd ) - os.chmod( proprietary_datatypes_config, 0644 ) - # Load proprietary datatypes - app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=proprietary_datatypes_config, deactivate=deactivate, override=override ) - if deactivate: - # Reload the upload tool to eliminate deactivated datatype extensions from the file_type select list. - tool_util.reload_upload_tools( app ) - else: - append_to_datatypes_registry_upload_file_formats( app, registration ) - tool_util.reload_upload_tools( app ) - if datatype_files is not None: - try: - os.unlink( proprietary_datatypes_config ) - except: - pass - return converter_path, display_path - -def append_to_datatypes_registry_upload_file_formats( app, elem ): - # See if we have any datatypes that should be displayed in the upload tool's file_type select list. - for datatype_elem in elem.findall( 'datatype' ): - extension = datatype_elem.get( 'extension', None ) - display_in_upload = datatype_elem.get( 'display_in_upload', None ) - if extension is not None and display_in_upload is not None: - display_in_upload = asbool( str( display_in_upload ) ) - if display_in_upload and extension not in app.datatypes_registry.upload_file_formats: - app.datatypes_registry.upload_file_formats.append( extension ) - -def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): - return dict( tool_shed=tool_shed, - repository_name=name, - repository_owner=owner, - installed_changeset_revision=installed_changeset_revision, - tool_dicts=tool_dicts, - converter_path=converter_path, - display_path=display_path ) - -def get_converter_and_display_paths( registration_elem, relative_install_dir ): - """Find the relative path to data type converters and display applications included in installed tool shed repositories.""" - converter_path = None - display_path = None - for elem in registration_elem.findall( 'datatype' ): - if not converter_path: - # If any of the <datatype> tag sets contain <converter> tags, set the converter_path - # if it is not already set. This requires developers to place all converters in the - # same subdirectory within the repository hierarchy. - for converter in elem.findall( 'converter' ): - converter_config = converter.get( 'file', None ) - if converter_config: - converter_config_file_name = basic_util.strip_path( converter_config ) - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == converter_config_file_name: - # The value of converter_path must be absolute due to job_working_directory. - converter_path = os.path.abspath( root ) - break - if converter_path: - break - if not display_path: - # If any of the <datatype> tag sets contain <display> tags, set the display_path - # if it is not already set. This requires developers to place all display acpplications - # in the same subdirectory within the repository hierarchy. - for display_app in elem.findall( 'display' ): - display_config = display_app.get( 'file', None ) - if display_config: - display_config_file_name = basic_util.strip_path( display_config ) - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == display_config_file_name: - # The value of display_path must be absolute due to job_working_directory. - display_path = os.path.abspath( root ) - break - if display_path: - break - if converter_path and display_path: - break - return converter_path, display_path - -def load_installed_datatype_converters( app, installed_repository_dict, deactivate=False ): - # Load or deactivate proprietary datatype converters - app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=installed_repository_dict, deactivate=deactivate ) - -def load_installed_datatypes( app, repository, relative_install_dir, deactivate=False ): - # Load proprietary datatypes and return information needed for loading proprietary datatypes converters and display applications later. - metadata = repository.metadata - repository_dict = None - datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir ) - if datatypes_config: - converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate ) - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=repository.tool_shed, - name=repository.name, - owner=repository.owner, - installed_changeset_revision=repository.installed_changeset_revision, - tool_dicts=metadata.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) - return repository_dict - -def load_installed_display_applications( app, installed_repository_dict, deactivate=False ): - # Load or deactivate proprietary datatype display applications - app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, deactivate=deactivate ) diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 static/scripts/mvc/base-mvc.js --- a/static/scripts/mvc/base-mvc.js +++ b/static/scripts/mvc/base-mvc.js @@ -14,7 +14,7 @@ * @example * // Add to your models/views at the definition using chaining: * var MyModel = Backbone.Model.extend( LoggableMixin ).extend({ // ... }); - * + * * // or - more explicitly AFTER the definition: * var MyModel = Backbone.Model.extend({ * logger : console @@ -208,8 +208,23 @@ }; //============================================================================== -return { - LoggableMixin : LoggableMixin, - SessionStorageModel : SessionStorageModel, - HiddenUntilActivatedViewMixin : HiddenUntilActivatedViewMixin -};}); +function mixin( mixinHash1, /* mixinHash2, etc: ... variadic */ propsHash ){ + // usage: var NewModel = Something.extend( mixin( MyMixinA, MyMixinB, { ... }) ); + //NOTE: this does not combine any hashes (like events, etc.) and you're expected to handle that + + // simple reversal of param order on _.defaults() - to show mixins in top of definition + var args = Array.prototype.slice.call( arguments, 0 ), + lastArg = args.pop(); + args.unshift( lastArg ); + return _.defaults.apply( _, args ); +} + + +//============================================================================== + return { + LoggableMixin : LoggableMixin, + SessionStorageModel : SessionStorageModel, + HiddenUntilActivatedViewMixin : HiddenUntilActivatedViewMixin, + mixin : mixin + }; +}); diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 static/scripts/mvc/collection/collection-model.js --- /dev/null +++ b/static/scripts/mvc/collection/collection-model.js @@ -0,0 +1,250 @@ +define([ + "mvc/dataset/hda-model", + "mvc/base-mvc", + "utils/localization" +], function( HDA_MODEL, BASE_MVC, _l ){ +//============================================================================== +/** @class Backbone model for Dataset collection elements. + * DC Elements contain a sub-model named 'object'. This class moves that + * 'object' from the JSON in the attributes list to a full, instantiated + * sub-model found in this.object. This is done on intialization and + * everytime the 'change:object' event is fired. + * + * @borrows LoggableMixin#logger as #logger + * @borrows LoggableMixin#log as #log + * @constructs + */ +var DatasetCollectionElement = Backbone.Model.extend( BASE_MVC.LoggableMixin ).extend( +/** @lends DatasetCollectionElement.prototype */{ + //TODO:?? this model may be unneccessary - it reflects the api structure, but... + // if we munge the element with the element.object at parse, we can flatten the entire hierarchy + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, + + defaults : { + id : null, + model_class : 'DatasetCollectionElement', + element_identifier : null, + element_index : null, + element_type : null + }, + + /** Set up. + * @see Backbone.Collection#initialize + */ + initialize : function( model, options ){ + this.info( this + '.initialize:', model, options ); + options = options || {}; + //this._setUpListeners(); + + this.object = this._createObjectModel(); + this.on( 'change:object', function(){ + //console.log( 'change:object' ); +//TODO: prob. better to update the sub-model instead of re-creating it + this.object = this._createObjectModel(); + }); + }, + + _createObjectModel : function(){ + //console.log( '_createObjectModel', this.get( 'object' ), this.object ); + //TODO: same patterns as HDCA _createElementsModel - refactor to BASE_MVC.hasSubModel? + if( _.isUndefined( this.object ) ){ this.object = null; } + if( !this.get( 'object' ) ){ return this.object; } + + var object = this.get( 'object' ); + this.unset( 'object', { silent: true }); + + this.debug( 'DCE, element_type:', this.get( 'element_type' ) ); + switch( this.get( 'element_type' ) ){ + case 'dataset_collection': + this.object = new DatasetCollection( object ); + break; + case 'hda': + this.object = new HDA_MODEL.HistoryDatasetAssociation( object ); + break; + default: + throw new TypeError( 'Unknown element_type: ' + this.get( 'element_type' ) ); + } + return this.object; + }, + + /** String representation. */ + toString : function(){ + var objStr = ( this.object )?( '' + this.object ):( this.get( 'element_identifier' ) ); + return ([ 'DatasetCollectionElement(', objStr, ')' ].join( '' )); + } +}); + + +//============================================================================== +/** @class Backbone collection for DCEs. + * NOTE: used *only* in second level of list:paired collections (a + * collection that contains collections) + * + * @borrows LoggableMixin#logger as #logger + * @borrows LoggableMixin#log as #log + * @constructs + */ +var DatasetCollectionElementCollection = Backbone.Collection.extend( BASE_MVC.LoggableMixin ).extend( +/** @lends DatasetCollectionElementCollection.prototype */{ + model: DatasetCollectionElement, + + // comment this out to suppress log output + /** logger used to record this.log messages, commonly set to console */ + //logger : console, + + /** Set up. + * @see Backbone.Collection#initialize + */ + initialize : function( models, options ){ + options = options || {}; + this.info( this + '.initialize:', models, options ); + //this._setUpListeners(); + }, + + /** String representation. */ + toString : function(){ + return ([ 'DatasetCollectionElementCollection(', this.length, ')' ].join( '' )); + } +}); + + +//============================================================================== +/** @class Backbone model for Dataset Collections. + * DCs contain a bbone collection named 'elements' using the class found in + * this.collectionClass (gen. DatasetCollectionElementCollection). DCs move + * that 'object' from the JSON in the attributes list to a full, instantiated + * collection found in this.elements. This is done on intialization and + * everytime the 'change:elements' event is fired. + * + * @borrows LoggableMixin#logger as #logger + * @borrows LoggableMixin#log as #log + * @constructs + */ +var DatasetCollection = Backbone.Model.extend( BASE_MVC.LoggableMixin ).extend( +/** @lends ListDatasetCollection.prototype */{ + + //logger : console, + + /** default attributes for a model */ + defaults : { + collection_type : 'list' + }, + + collectionClass : DatasetCollectionElementCollection, + + /** */ + initialize : function( model, options ){ + this.info( 'DatasetCollection.initialize:', model, options ); + //historyContent.HistoryContent.prototype.initialize.call( this, attrs, options ); + this.elements = this._createElementsModel(); +//TODO:?? no way to use parse here? + this.on( 'change:elements', function(){ + this.log( 'change:elements' ); +//TODO: prob. better to update the collection instead of re-creating it + this.elements = this._createElementsModel(); + }); + }, + + /** move elements model attribute to full collection */ + _createElementsModel : function(){ + this.log( '_createElementsModel', this.get( 'elements' ), this.elements ); +//TODO: same patterns as DatasetCollectionElement _createObjectModel - refactor to BASE_MVC.hasSubModel? + var elements = this.get( 'elements' ) || []; + this.info( 'elements:', elements ); + this.unset( 'elements', { silent: true }); + this.elements = new this.collectionClass( elements ); + return this.elements; + }, + + hasDetails : function(){ +//TODO: this is incorrect for (accidentally) empty collections + return this.elements.length !== 0; + }, + + // ........................................................................ misc + /** String representation */ + toString : function(){ + var idAndName = [ this.get( 'id' ), this.get( 'name' ) || this.get( 'element_identifier' ) ]; + return 'DatasetCollection(' + ( idAndName.join(',') ) + ')'; + } +}); + + +//============================================================================== +/** @class Backbone collection for a collection of collection collections collecting correctly. */ +var DatasetCollectionCollection = Backbone.Collection.extend( BASE_MVC.LoggableMixin ).extend({ + + model: DatasetCollection, + + ///** logger used to record this.log messages, commonly set to console */ + //// comment this out to suppress log output + //logger : console, + + /** Set up. + * @see Backbone.Collection#initialize + */ + initialize : function( models, options ){ + options = options || {}; + this.info( 'DatasetCollectionCollection.initialize:', models, options ); + //this._setUpListeners(); + }, + + /** String representation. */ + toString : function(){ + return ([ 'DatasetCollectionCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//NOTE: the following prototypes may not be necessary - but I wanted to specifiy +// them (for now) and allow for the possibility of unique functionality +//============================================================================== +var ListDatasetCollection = DatasetCollection.extend( +/** @lends ListDatasetCollection.prototype */{ + + /** String representation. */ + toString : function(){ + return ([ 'ListDatasetCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//============================================================================== +var PairDatasetCollection = DatasetCollection.extend( +/** @lends ListDatasetCollection.prototype */{ + + /** String representation. */ + toString : function(){ + return ([ 'PairDatasetCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//============================================================================== +var ListPairedDatasetCollection = DatasetCollection.extend( +/** @lends ListDatasetCollection.prototype */{ + + // list:paired is the only collection that itself contains collections + //collectionClass : DatasetCollectionCollection, + + /** String representation. */ + toString : function(){ + return ([ 'ListPairedDatasetCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//============================================================================== + return { + DatasetCollectionElement : DatasetCollectionElement, + DatasetCollectionElementCollection : DatasetCollectionElementCollection, + DatasetCollection : DatasetCollection, + DatasetCollectionCollection : DatasetCollectionCollection, + ListDatasetCollection : ListDatasetCollection, + PairDatasetCollection : PairDatasetCollection, + ListPairedDatasetCollection : ListPairedDatasetCollection + }; +}); diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 static/scripts/mvc/collection/collection-panel.js --- /dev/null +++ b/static/scripts/mvc/collection/collection-panel.js @@ -0,0 +1,326 @@ +define([ + "mvc/collection/dataset-collection-base", + "mvc/dataset/hda-base", + "mvc/base-mvc", + "utils/localization" +], function( DC_BASE, HDA_BASE, BASE_MVC, _l ){ +/* ============================================================================= +TODO: + +============================================================================= */ +/** @class non-editable, read-only View/Controller for a dataset collection. + * @name CollectionPanel + * + * @augments Backbone.View + * @borrows LoggableMixin#logger as #logger + * @borrows LoggableMixin#log as #log + * @constructs + */ +var CollectionPanel = Backbone.View.extend( BASE_MVC.LoggableMixin ).extend( +/** @lends CollectionPanel.prototype */{ + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, + + tagName : 'div', + className : 'history-panel', + + /** (in ms) that jquery effects will use */ + fxSpeed : 'fast', + + // ......................................................................... SET UP + /** Set up the view, set up storage, bind listeners to HistoryContents events + * @param {Object} attributes optional settings for the panel + */ + initialize : function( attributes ){ + attributes = attributes || {}; + // set the logger if requested + if( attributes.logger ){ + this.logger = attributes.logger; + } + this.log( this + '.initialize:', attributes ); + + this.hasUser = attributes.hasUser; + this.HDAViewClass = attributes.HDAViewClass || HDA_BASE.HDABaseView; + }, + + /** create any event listeners for the panel + * @fires: rendered:initial on the first render + * @fires: empty-history when switching to a history with no HDAs or creating a new history + */ + _setUpListeners : function(){ + // debugging + //if( this.logger ){ + this.on( 'all', function( event ){ + this.log( this + '', arguments ); + }, this ); + //} + return this; + }, + + // ------------------------------------------------------------------------ history/hda event listening + /** listening for history and HDA events */ + _setUpModelEventHandlers : function(){ + return this; + }, + + // ------------------------------------------------------------------------ panel rendering + /** Render urls, historyPanel body, and hdas (if any are shown) + * @fires: rendered when the panel is attached and fully visible + * @see Backbone.View#render + */ + render : function( speed, callback ){ + this.log( 'render:', speed, callback ); + // send a speed of 0 to have no fade in/out performed + speed = ( speed === undefined )?( this.fxSpeed ):( speed ); + //console.debug( this + '.render, fxSpeed:', speed ); + var panel = this, + $newRender; + + // handle the possibility of no model (can occur if fetching the model returns an error) + if( !this.model ){ + return this; + } + $newRender = this.renderModel(); + + // fade out existing, swap with the new, fade in, set up behaviours + $( panel ).queue( 'fx', [ + function( next ){ + if( speed && panel.$el.is( ':visible' ) ){ + panel.$el.fadeOut( speed, next ); + } else { + next(); + } + }, + function( next ){ + // swap over from temp div newRender + panel.$el.empty(); + if( $newRender ){ + panel.$el.append( $newRender.children() ); + } + next(); + }, + function( next ){ + if( speed && !panel.$el.is( ':visible' ) ){ + panel.$el.fadeIn( speed, next ); + } else { + next(); + } + }, + function( next ){ + //TODO: ideally, these would be set up before the fade in (can't because of async save text) + if( callback ){ callback.call( this ); } + panel.trigger( 'rendered', this ); + next(); + } + ]); + return this; + }, + + /** render with history data + * @returns {jQuery} dom fragment as temporary container to be swapped out later + */ + renderModel : function( ){ + // tmp div for final swap in render + var $newRender = $( '<div/>' ).append( CollectionPanel.templates.panel( this.model.toJSON() ) ); + this._setUpBehaviours( $newRender ); + this.renderContents( $newRender ); + return $newRender; + }, + + /** Set up HistoryPanel js/widget behaviours */ + _setUpBehaviours : function( $where ){ + //TODO: these should be either sub-MVs, or handled by events + $where = $where || this.$el; + $where.find( '[title]' ).tooltip({ placement: 'bottom' }); + return this; + }, + + // ------------------------------------------------------------------------ sub-$element shortcuts + /** the scroll container for this panel - can be $el, $el.parent(), or grandparent depending on context */ + $container : function(){ + return ( this.findContainerFn )?( this.findContainerFn.call( this ) ):( this.$el.parent() ); + }, + /** where hdaViews are attached */ + $datasetsList : function( $where ){ + return ( $where || this.$el ).find( '.datasets-list' ); + }, + + // ------------------------------------------------------------------------ sub-views + /** Set up/render a view for each HDA to be shown, init with model and listeners. + * HDA views are cached to the map this.hdaViews (using the model.id as key). + * @param {jQuery} $whereTo what dom element to prepend the HDA views to + * @returns the number of visible hda views + */ + renderContents : function( $whereTo ){ + //console.debug( 'renderContents, elements:', this.model.elements ); + $whereTo = $whereTo || this.$el; + + var panel = this, + contentViews = {}, + visibleContents = this.model.elements || []; + //this.log( 'renderContents, visibleContents:', visibleContents, $whereTo ); + + this.$datasetsList( $whereTo ).empty(); + if( visibleContents && visibleContents.length ){ + visibleContents.each( function( content ){ + var contentId = content.id, + contentView = panel._createContentView( content ); + contentViews[ contentId ] = contentView; + panel.attachContentView( contentView.render(), $whereTo ); + }); + } + this.contentViews = contentViews; + return this.contentViews; + }, + + /** + * @param {HistoryDatasetAssociation} content + */ + _createContentView : function( content ){ + //console.debug( 'content json:', JSON.stringify( content, null, ' ' ) ); + var contentView = null, + ContentClass = this._getContentClass( content ); + //console.debug( 'content.object json:', JSON.stringify( content.object, null, ' ' ) ); + //console.debug( 'ContentClass:', ContentClass ); + //console.debug( 'content:', content ); + //console.debug( 'content.object:', content.object ); + contentView = new ContentClass({ + model : content.object, + linkTarget : this.linkTarget, + //draggable : true, + hasUser : this.hasUser, + logger : this.logger + }); + //this._setUpHdaListeners( contentView ); + return contentView; + }, + + _getContentClass : function( content ){ + switch( content.get( 'element_type' ) ){ + case 'hda': + return this.HDAViewClass; + case 'dataset_collection': + return DC_BASE.NestedDCEBaseView; + } + throw new TypeError( 'Unknown element type:', content.get( 'element_type' ) ); + }, + +// /** Set up HistoryPanel listeners for HDAView events. Currently binds: +// * HDAView#body-visible, HDAView#body-hidden to store expanded states +// * @param {HDAView} hdaView HDAView (base or edit) to listen to +// */ +// _setUpHdaListeners : function( hdaView ){ +// var panel = this; +// hdaView.on( 'error', function( model, xhr, options, msg ){ +// panel.errorHandler( model, xhr, options, msg ); +// }); +// // maintain a list of hdas whose bodies are expanded +// hdaView.on( 'body-expanded', function( model ){ +// panel.storage.addExpandedHda( model ); +// }); +// hdaView.on( 'body-collapsed', function( id ){ +// panel.storage.removeExpandedHda( id ); +// }); +// return this; +// }, + + /** attach an contentView to the panel */ + attachContentView : function( contentView, $whereTo ){ + $whereTo = $whereTo || this.$el; + var $datasetsList = this.$datasetsList( $whereTo ); + $datasetsList.append( contentView.$el ); + return this; + }, + + // ------------------------------------------------------------------------ panel events + /** event map */ + events : { + 'click .panel-navigation-back' : 'close' + }, + + /** */ + close : function( event ){ + this.$el.remove(); + this.trigger( 'collection-close' ); + }, + + // ........................................................................ misc + /** Return a string rep of the history */ + toString : function(){ + return 'CollectionPanel(' + (( this.model )?( this.model.get( 'name' )):( '' )) + ')'; + } +}); + + +//------------------------------------------------------------------------------ TEMPLATES +var _panelTemplate = [ + '<div class="history-controls">', + '<div class="panel-navigation">', + '<a class="panel-navigation-back" href="javascript:void(0)">', _l( 'Back' ), '</a>', + '</div>', + + '<div class="history-title">', + '<% if( collection.name ){ %>', + '<div class="history-name"><%= collection.hid %> : <%= collection.name %></div>', + '<% } %>', + '</div>', + + //'<div class="history-subtitle clear">', + // '<% if( history.nice_size ){ %>', + // '<div class="history-size"><%= history.nice_size %></div>', + // '<% } %>', + // '<div class="history-secondary-actions"></div>', + //'</div>', + // + //'<% if( history.deleted ){ %>', + // '<div class="warningmessagesmall"><strong>', + // _l( 'You are currently viewing a deleted history!' ), + // '</strong></div>', + //'<% } %>', + // + //'<div class="message-container">', + // '<% if( history.message ){ %>', + // // should already be localized + // '<div class="<%= history.status %>message"><%= history.message %></div>', + // '<% } %>', + //'</div>', + // + //'<div class="quota-message errormessage">', + // _l( 'You are over your disk quota' ), '. ', + // _l( 'Tool execution is on hold until your disk usage drops below your allocated quota' ), '.', + //'</div>', + // + //'<div class="tags-display"></div>', + //'<div class="annotation-display"></div>', + //'<div class="history-dataset-actions">', + // '<div class="btn-group">', + // '<button class="history-select-all-datasets-btn btn btn-default"', + // 'data-mode="select">', _l( 'All' ), '</button>', + // '<button class="history-deselect-all-datasets-btn btn btn-default"', + // 'data-mode="select">', _l( 'None' ), '</button>', + // '</div>', + // '<button class="history-dataset-action-popup-btn btn btn-default">', + // _l( 'For all selected' ), '...</button>', + //'</div>', + '</div>', + // end history controls + + // where the datasets/hdas are added + '<div class="datasets-list"></div>' + +].join( '' ); + +CollectionPanel.templates = { + panel : function( JSON ){ + return _.template( _panelTemplate, JSON, { variable: 'collection' }); + } +}; + + +//============================================================================== + return { + CollectionPanel: CollectionPanel + }; +}); diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 static/scripts/mvc/collection/dataset-collection-base.js --- a/static/scripts/mvc/collection/dataset-collection-base.js +++ b/static/scripts/mvc/collection/dataset-collection-base.js @@ -1,328 +1,151 @@ define([ - "mvc/history/history-content-base-view", + "mvc/base-mvc", "utils/localization" -], function( historyContentBaseView, _l ){ +], function( BASE_MVC, _l ){ /* global Backbone, LoggableMixin */ //============================================================================== -/** @class Read only view for HistoryDatasetCollectionAssociation. - * @name HDABaseView +/** @class Read only view for DatasetCollection. + * @name DCBaseView * * @augments Backbone.View * @borrows LoggableMixin#logger as #logger * @borrows LoggableMixin#log as #log * @constructs */ -var DatasetCollectionBaseView = historyContentBaseView.HistoryContentBaseView.extend({ - className : "dataset hda history-panel-hda", - id : function(){ return 'hdca-' + this.model.get( 'id' ); }, +var DCBaseView = Backbone.View.extend( BASE_MVC.LoggableMixin ).extend({ + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, + + /** */ + className : "dataset-collection", + /** */ + fxSpeed : 'fast', /** */ initialize : function( attributes ){ if( attributes.logger ){ this.logger = this.model.logger = attributes.logger; } - this.log( this + '.initialize:', attributes ); - /** is the view currently in selection mode? */ - this.selectable = attributes.selectable || false; - //this.log( '\t selectable:', this.selectable ); - /** is the view currently selected? */ - this.selected = attributes.selected || false; - /** is the body of this collection view expanded/not? */ - this.expanded = attributes.expanded || false; + this.log( 'DCBaseView.initialize:', attributes ); }, - /** */ - render : function( fade ){ - var $newRender = this._buildNewRender(); - - this._queueNewRender( $newRender, fade ); - return this; - }, - - /** */ - _buildNewRender : function(){ - var $newRender = $( DatasetCollectionBaseView.templates.skeleton( this.model.toJSON() ) ); - $newRender.find( '.dataset-primary-actions' ).append( this._render_titleButtons() ); - $newRender.children( '.dataset-body' ).replaceWith( this._render_body() ); - this._setUpBehaviors( $newRender ); - return $newRender; - }, - - /** */ - _queueNewRender : function( $newRender, fade ) { - fade = ( fade === undefined )?( true ):( fade ); - var view = this; - - // fade the old render out (if desired) - if( fade ){ - $( view ).queue( function( next ){ this.$el.fadeOut( view.fxSpeed, next ); }); - } - // empty the old render, update to any new HDA state, swap in the new render contents, handle multi-select - $( view ).queue( function( next ){ - this.$el.empty() - .attr( 'class', view.className ).addClass( 'state-' + view.model.get( 'state' ) ) - .append( $newRender.children() ); - if( this.selectable ){ this.showSelector( 0 ); } - next(); - }); - // fade the new in - if( fade ){ - $( view ).queue( function( next ){ this.$el.fadeIn( view.fxSpeed, next ); }); - } - // trigger an event to know we're ready - $( view ).queue( function( next ){ - this.trigger( 'rendered', view ); - if( this.model.inReadyState() ){ - this.trigger( 'rendered:ready', view ); - } - if( this.draggable ){ this.draggableOn(); } - next(); - }); - }, - - // ................................................................................ titlebar buttons - /** Render icon-button group for the common, most easily accessed actions. - * @returns {jQuery} rendered DOM - */ - _render_titleButtons : function(){ - // render just the display for read-only - return [ ]; - }, - - // ......................................................................... state body renderers - /** Render the enclosing div of the collection body and, if expanded, the html in the body - * @returns {jQuery} rendered DOM - */ - _render_body : function(){ - var $body = $( '<div>Error: unknown state "' + this.model.get( 'state' ) + '".</div>' ), - // cheesy: get function by assumed matching name - renderFn = this[ '_render_body_' + this.model.get( 'state' ) ]; - if( _.isFunction( renderFn ) ){ - $body = renderFn.call( this ); - } - this._setUpBehaviors( $body ); - - // only render the body html if it's being shown - if( this.expanded ){ - $body.show(); - } - return $body; - }, - - /** set up js behaviors, event handlers for elements within the given container - * @param {jQuery} $container jq object that contains the elements to process (defaults to this.$el) - */ - _setUpBehaviors : function( $container ){ - $container = $container || this.$el; - // set up canned behavior on children (bootstrap, popupmenus, editable_text, etc.) - make_popup_menus( $container ); - $container.find( '[title]' ).tooltip({ placement : 'bottom' }); - }, - - // TODO: Eliminate duplication between following event map and one for HDAs. - - // ......................................................................... events - /** event map */ - events : { - // expand the body when the title is clicked or when in focus and space or enter is pressed - 'click .dataset-title-bar' : 'toggleBodyVisibility', - 'keydown .dataset-title-bar' : 'toggleBodyVisibility', - - // toggle selected state - 'click .dataset-selector' : 'toggleSelect' - }, - - /** Show or hide the body/details of history content. - * note: if the model does not have detailed data, fetch that data before showing the body - * @param {Event} event the event that triggered this (@link HDABaseView#events) - * @param {Boolean} expanded if true, expand; if false, collapse - * @fires body-expanded when a body has been expanded - * @fires body-collapsed when a body has been collapsed - */ - toggleBodyVisibility : function( event, expand ){ - // bail (with propagation) if keydown and not space or enter - var KEYCODE_SPACE = 32, KEYCODE_RETURN = 13; - if( event && ( event.type === 'keydown' ) - && !( event.keyCode === KEYCODE_SPACE || event.keyCode === KEYCODE_RETURN ) ){ - return true; - } - - var $body = this.$el.find( '.dataset-body' ); - expand = ( expand === undefined )?( !$body.is( ':visible' ) ):( expand ); - if( expand ){ - this.expandBody(); - } else { - this.collapseBody(); - } - return false; - }, - - /** Render and show the full, detailed body of this view including extra data and controls. - * @fires body-expanded when a body has been expanded - */ - expandBody : function(){ - var contentView = this; - - function _renderBodyAndExpand(){ - contentView.$el.children( '.dataset-body' ).replaceWith( contentView._render_body() ); - contentView.$el.children( '.dataset-body' ).slideDown( contentView.fxSpeed, function(){ - contentView.expanded = true; - contentView.trigger( 'body-expanded', contentView.model ); - }); - } - // TODO: Fetch more details like HDA view... - _renderBodyAndExpand(); - }, - - /** Hide the body/details of an HDA. - * @fires body-collapsed when a body has been collapsed - */ - collapseBody : function(){ - var hdaView = this; - this.$el.children( '.dataset-body' ).slideUp( hdaView.fxSpeed, function(){ - hdaView.expanded = false; - hdaView.trigger( 'body-collapsed', hdaView.model.id ); - }); - }, - - /** Render an 'ok' collection. - * @param {jQuery} parent DOM to which to append this body - */ - _render_body_ok : function(){ - // most common state renderer and the most complicated - var $body = $( DatasetCollectionBaseView.templates.body( this.model.toJSON() ) ); - - // return shortened form if del'd (no display apps or peek?) - if( this.model.get( 'deleted' ) ){ - return $body; - } - - return $body; - }, - - // ......................................................................... selection - /** display a (fa-icon) checkbox on the left of the hda that fires events when checked - * Note: this also hides the primary actions - */ - showSelector : function(){ - // make sure selected state is represented properly - if( this.selected ){ - this.select( null, true ); - } - - this.selectable = true; - this.trigger( 'selectable', true, this ); - - this.$( '.dataset-primary-actions' ).hide(); - this.$( '.dataset-selector' ).show(); - }, - - /** remove the selection checkbox */ - hideSelector : function(){ - // reverse the process from showSelect - this.selectable = false; - this.trigger( 'selectable', false, this ); - - this.$( '.dataset-selector' ).hide(); - this.$( '.dataset-primary-actions' ).show(); - }, - - toggleSelector : function(){ - if( !this.$el.find( '.dataset-selector' ).is( ':visible' ) ){ - this.showSelector(); - } else { - this.hideSelector(); - } - }, - - /** event handler for selection (also programmatic selection) - */ - select : function( event ){ - // switch icon, set selected, and trigger event - this.$el.find( '.dataset-selector span' ) - .removeClass( 'fa-square-o' ).addClass( 'fa-check-square-o' ); - if( !this.selected ){ - this.trigger( 'selected', this, event ); - this.selected = true; - } - return false; - }, - - /** event handler for clearing selection (also programmatic deselection) - */ - deselect : function( event ){ - // switch icon, set selected, and trigger event - this.$el.find( '.dataset-selector span' ) - .removeClass( 'fa-check-square-o' ).addClass( 'fa-square-o' ); - if( this.selected ){ - this.trigger( 'de-selected', this, event ); - this.selected = false; - } - return false; - }, - - toggleSelect : function( event ){ - if( this.selected ){ - this.deselect( event ); - } else { - this.select( event ); - } - }, +//TODO: render has been removed from the inheritance chain here, so this won't work when called as is // ......................................................................... misc /** String representation */ toString : function(){ var modelString = ( this.model )?( this.model + '' ):( '(no model)' ); - return 'HDCABaseView(' + modelString + ')'; + return 'DCBaseView(' + modelString + ')'; } }); -//------------------------------------------------------------------------------ TEMPLATES -//TODO: possibly break these out into a sep. module -var skeletonTemplate = _.template([ - '<div class="dataset hda">', - '<div class="dataset-warnings">', - '<% if ( collection.deleted ) { %>', - '<div class="dataset-deleted-msg warningmessagesmall"><strong>', - _l( 'This collection has been deleted.' ), +/** templates for DCBaseViews (skeleton and body) */ +DCBaseView.templates = (function(){ +// use closure to run underscore template fn only once at module load + var skeletonTemplate = _.template([ + '<div class="dataset hda">', + '<div class="dataset-warnings">', + '<% if ( collection.deleted ) { %>', + '<div class="dataset-deleted-msg warningmessagesmall"><strong>', + _l( 'This collection has been deleted.' ), + '</div>', + '<% } %>', + '<% if ( !collection.visible ) { %>', + '<div class="dataset-hidden-msg warningmessagesmall"><strong>', + _l( 'This collection has been hidden.' ), + '</div>', + '<% } %>', + '</div>', + '<div class="dataset-primary-actions"></div>', + '<div class="dataset-title-bar clear" tabindex="0">', + '<span class="dataset-state-icon state-icon"></span>', + '<div class="dataset-title">', + '<span class="dataset-name"><%- collection.name %></span>', '</div>', - '<% } %>', - '<% if ( !collection.visible ) { %>', - '<div class="dataset-hidden-msg warningmessagesmall"><strong>', - _l( 'This collection has been hidden.' ), - '</div>', - '<% } %>', - '</div>', - '<div class="dataset-selector"><span class="fa fa-2x fa-square-o"></span></div>', - '<div class="dataset-primary-actions"></div>', - '<div class="dataset-title-bar clear" tabindex="0">', - '<span class="dataset-state-icon state-icon"></span>', - '<div class="dataset-title">', - '<span class="hda-hid"><%= collection.hid %></span> ', - '<span class="dataset-name"><%= collection.name %></span>', '</div>', - '</div>', - '<div class="dataset-body"></div>', - '</div>' -].join( '' )); + '<div class="dataset-body"></div>', + '</div>' + ].join( '' )); -var bodyTemplate = _.template([ - '<div class="dataset-body">', - '<div class="dataset-summary">', - _l( 'A dataset collection.' ), - '</div>' -].join( '' )); + var bodyTemplate = _.template([ + '<div class="dataset-body">', + '<div class="dataset-summary">', + _l( 'A dataset collection.' ), + '</div>' + ].join( '' )); -DatasetCollectionBaseView.templates = { // we override here in order to pass the localizer (_L) into the template scope - since we use it as a fn within - skeleton : function( collectionJSON ){ - return skeletonTemplate({ _l: _l, collection: collectionJSON }); - }, - body : function( collectionJSON ){ - return bodyTemplate({ _l: _l, collection: collectionJSON }); - } -}; + return { + skeleton : function( collectionJSON ){ + return skeletonTemplate({ _l: _l, collection: collectionJSON }); + }, + body : function( collectionJSON ){ + return bodyTemplate({ _l: _l, collection: collectionJSON }); + } + }; +}()); + + +//TODO: unused +////============================================================================== +///** @class Read only view for DatasetCollectionElement. +// * @name DCEBaseView +// * +// * @augments Backbone.View +// * @borrows LoggableMixin#logger as #logger +// * @borrows LoggableMixin#log as #log +// * @constructs +// */ +//var NestedDCBaseView = DCBaseView.extend({ +// +// logger : console, +// +// /** */ +// initialize : function( attributes ){ +// if( attributes.logger ){ this.logger = this.model.logger = attributes.logger; } +// this.warn( this + '.initialize:', attributes ); +// DCBaseView.prototype.initialize.call( this, attributes ); +// }, +// +// _template : function(){ +// this.debug( this.model ); +// this.debug( this.model.toJSON() ); +// return NestedDCBaseView.templates.skeleton( this.model.toJSON() ); +// }, +// +// // ......................................................................... misc +// /** String representation */ +// toString : function(){ +// var modelString = ( this.model )?( this.model + '' ):( '(no model)' ); +// return 'NestedDCBaseView(' + modelString + ')'; +// } +//}); +// +////------------------------------------------------------------------------------ TEMPLATES +////TODO: possibly break these out into a sep. module +//NestedDCBaseView.templates = (function(){ +// var skeleton = _.template([ +// '<div class="dataset hda history-panel-hda state-ok">', +// '<div class="dataset-primary-actions"></div>', +// '<div class="dataset-title-bar clear" tabindex="0">', +// '<div class="dataset-title">', +// '<span class="dataset-name"><%= collection.name %></span>', +// '</div>', +// '</div>', +// '</div>' +// ].join( '' )); +// // we override here in order to pass the localizer (_L) into the template scope - since we use it as a fn within +// return { +// skeleton : function( json ){ +// return skeleton({ _l: _l, collection: json }); +// } +// }; +//}()); + //============================================================================== return { - DatasetCollectionBaseView : DatasetCollectionBaseView + DCBaseView : DCBaseView, + //NestedDCBaseView : NestedDCBaseView, }; }); diff -r 4974ffd8b2cb5ee994a0d517c2332261898d4d46 -r 7aade8875ea4dde276b33922bbc7fdbc0630a9f5 static/scripts/mvc/collection/dataset-collection-edit.js --- a/static/scripts/mvc/collection/dataset-collection-edit.js +++ b/static/scripts/mvc/collection/dataset-collection-edit.js @@ -1,75 +1,36 @@ define([ - "mvc/dataset/hda-model", + "mvc/dataset/states", "mvc/collection/dataset-collection-base", "utils/localization" -], function( hdaModel, datasetCollectionBase, _l ){ +], function( STATES, DC_BASE_VIEW, _l ){ //============================================================================== -/** @class Editing view for HistoryDatasetCollectionAssociation. +var _super = DC_BASE_VIEW.DCBaseView; +/** @class Editing view for DatasetCollection. * @name DatasetCollectionEditView * - * @augments DatasetCollectionBaseView + * @augments DCBaseView * @constructs */ -var DatasetCollectionEditView = datasetCollectionBase.DatasetCollectionBaseView.extend( { +var DCEditView = _super.extend({ + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, initialize : function( attributes ){ - datasetCollectionBase.DatasetCollectionBaseView.prototype.initialize.call( this, attributes ); - }, - - // ......................................................................... edit attr, delete - /** Render icon-button group for the common, most easily accessed actions. - * Overrides _render_titleButtons to include editing related buttons. - * @see DatasetCollectionBaseView#_render_titleButtons - * @returns {jQuery} rendered DOM - */ - _render_titleButtons : function(){ - // render the display, edit attr and delete icon-buttons - return datasetCollectionBase.DatasetCollectionBaseView.prototype._render_titleButtons.call( this ).concat([ - this._render_deleteButton() - ]); - }, - - /** Render icon-button to delete this hda. - * @returns {jQuery} rendered DOM - */ - _render_deleteButton : function(){ - // don't show delete if... - if( ( this.model.get( 'state' ) === hdaModel.HistoryDatasetAssociation.STATES.NEW ) - || ( this.model.get( 'state' ) === hdaModel.HistoryDatasetAssociation.STATES.NOT_VIEWABLE ) - || ( !this.model.get( 'accessible' ) ) ){ - return null; - } - - var self = this, - deleteBtnData = { - title : _l( 'Delete' ), - classes : 'dataset-delete', - onclick : function() { - // ...bler... tooltips being left behind in DOM (hover out never called on deletion) - self.$el.find( '.icon-btn.dataset-delete' ).trigger( 'mouseout' ); - self.model[ 'delete' ](); - } - }; - if( this.model.get( 'deleted' ) ){ - deleteBtnData = { - title : _l( 'Dataset collection is already deleted' ), - disabled : true - }; - } - deleteBtnData.faIcon = 'fa-times'; - return faIconButton( deleteBtnData ); + _super.prototype.initialize.call( this, attributes ); }, // ......................................................................... misc /** string rep */ toString : function(){ var modelString = ( this.model )?( this.model + '' ):( '(no model)' ); - return 'HDCAEditView(' + modelString + ')'; + return 'DCEditView(' + modelString + ')'; } }); //============================================================================== return { - DatasetCollectionEditView : DatasetCollectionEditView + DCEditView : DCEditView }; }); This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for exporting repositories that contain package recipes that define additional packages needed only for compiling the dependent package.
by commits-noreply＠bitbucket.org 23 Jul '14

23 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e45ed2368920/ Changeset: e45ed2368920 User: greg Date: 2014-07-23 22:19:39 Summary: Fix for exporting repositories that contain package recipes that define additional packages needed only for compiling the dependent package. Affected #: 3 files diff -r b08a47eb8c3c15022b9e3f7d9990d40fd0fa70c0 -r e45ed2368920315ab1f9cfe28d81e07cefc5c955 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1262,7 +1262,11 @@ repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) metadata = repository_metadata.metadata toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + # Initialize the repository dependency RelationBuilder. rb = relation_builder.RelationBuilder( trans.app, repository, repository_metadata, toolshed_base_url ) + # Work-around to ensure repositories that contain packages needed only for compiling + # a dependent package are included in the capsule. + rb.set_filter_dependencies_needed_for_compiling( False ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() if repository_dependencies: diff -r b08a47eb8c3c15022b9e3f7d9990d40fd0fa70c0 -r e45ed2368920315ab1f9cfe28d81e07cefc5c955 lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -245,9 +245,13 @@ repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, self.repository_id, self.changeset_revision ) - # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. + # Get a dictionary of all repositories upon which the contents of the current + # repository_metadata record depend. toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) rb = RelationBuilder( self.app, repository, repository_metadata, toolshed_base_url ) + # Work-around to ensure repositories that contain packages needed only for compiling + # a dependent package are included in the capsule. + rb.set_filter_dependencies_needed_for_compiling( False ) repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() repo = hg_util.get_repo_for_repository( self.app, repository=self.repository, diff -r b08a47eb8c3c15022b9e3f7d9990d40fd0fa70c0 -r e45ed2368920315ab1f9cfe28d81e07cefc5c955 lib/tool_shed/dependencies/repository/relation_builder.py --- a/lib/tool_shed/dependencies/repository/relation_builder.py +++ b/lib/tool_shed/dependencies/repository/relation_builder.py @@ -23,6 +23,11 @@ self.handled_key_rd_dicts = [] self.key_rd_dicts_to_be_processed = [] self.tool_shed_url = tool_shed_url + # This is a temporary work-around for handling repository dependencies that are needed + # only if compiling a dependent package. This value should be True unless exporting + # a repository capsule, in which case the set_filter_dependencies_needed_for_compiling() + # function is called. + self.filter_dependencies_needed_for_compiling = True def can_add_to_key_rd_dicts( self, key_rd_dict, key_rd_dicts ): """Handle the case where an update to the changeset revision was done.""" @@ -389,9 +394,12 @@ current_repository_key_rd_dicts = \ self.get_updated_changeset_revisions_for_repository_dependencies( current_repository_key_rd_dicts ) for key_rd_dict in current_repository_key_rd_dicts: - # Filter out repository dependencies that are required only if compiling the dependent - # repository's tool dependency. - key_rd_dict = self.filter_only_if_compiling_contained_td( key_rd_dict ) + if self.filter_dependencies_needed_for_compiling: + # Filter out repository dependencies that are required only if compiling the dependent + # repository's tool dependency. + # TODO: this temporary work-around should be removed when the underlying framework + # support for handling only_if_compiling_contained_td-flagged repositories is completed. + key_rd_dict = self.filter_only_if_compiling_contained_td( key_rd_dict ) if key_rd_dict: is_circular = False in_handled_key_rd_dicts = self.in_key_rd_dicts( key_rd_dict, self.handled_key_rd_dicts ) @@ -483,6 +491,9 @@ clean_key_rd_dicts.append( new_key_rd_dict ) return clean_key_rd_dicts + def set_filter_dependencies_needed_for_compiling( self, value ): + self.filter_dependencies_needed_for_compiling = asbool( value ) + def update_circular_repository_dependencies( self, repository_key, repository_dependency, repository_dependencies ): repository_dependency_as_key = self.get_repository_dependency_as_key( repository_dependency ) repository_key_as_repository_dependency = repository_key.split( container_util.STRSEP ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Strip repository names when creating in the Tool Shed.
by commits-noreply＠bitbucket.org 23 Jul '14

23 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b08a47eb8c3c/ Changeset: b08a47eb8c3c User: greg Date: 2014-07-23 19:40:23 Summary: Strip repository names when creating in the Tool Shed. Affected #: 1 file diff -r 598a4ec015f66237d41ed2176c673fd52b744f15 -r b08a47eb8c3c15022b9e3f7d9990d40fd0fa70c0 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1062,7 +1062,7 @@ action='browse_repositories', message=message, status=status ) ) - name = kwd.get( 'name', '' ) + name = kwd.get( 'name', '' ).strip() description = kwd.get( 'description', '' ) long_description = kwd.get( 'long_description', '' ) category_ids = util.listify( kwd.get( 'category_id', '' ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: History/Collection/HDA MVC: refactoring; Base MVC: create mixin shortcut to extend, use with HistoryContentMixin; Pages: allow display of collections; HDAs: fix annotation default overwriting existing when only partial model is saved; Style: begin separation of collections styles and prep for DOM class renaming; pack scripts
by commits-noreply＠bitbucket.org 23 Jul '14

23 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/598a4ec015f6/ Changeset: 598a4ec015f6 User: carlfeberhard Date: 2014-07-23 18:59:04 Summary: History/Collection/HDA MVC: refactoring; Base MVC: create mixin shortcut to extend, use with HistoryContentMixin; Pages: allow display of collections; HDAs: fix annotation default overwriting existing when only partial model is saved; Style: begin separation of collections styles and prep for DOM class renaming; pack scripts Affected #: 50 files diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -195,6 +195,7 @@ class HistoryController( BaseUIController, SharableMixin, UsesAnnotations, UsesItemRatings, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, ExportsHistoryMixin, ImportsHistoryMixin ): + def __init__( self, app ): super( HistoryController, self ).__init__( app ) self.mgrs = util.bunch.Bunch( diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 lib/galaxy/webapps/galaxy/controllers/page.py --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -1,5 +1,6 @@ from sqlalchemy import desc, and_ from galaxy import model, web +from galaxy import managers from galaxy.web import error, url_for from galaxy.model.item_attrs import UsesItemRatings from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesHistoryMixin, UsesStoredWorkflowMixin, UsesVisualizationMixin @@ -285,6 +286,12 @@ _page_selection_grid = PageSelectionGrid() _visualization_selection_grid = VisualizationSelectionGrid() + def __init__( self, app ): + super( PageController, self ).__init__( app ) + self.mgrs = util.bunch.Bunch( + histories=managers.histories.HistoryManager() + ) + @web.expose @web.require_login() def list( self, trans, *args, **kwargs ): @@ -718,6 +725,7 @@ """ Returns html suitable for embedding in another page. """ + #TODO: should be moved to history controller and/or called via ajax from the template history = self.get_history( trans, id, False, True ) if not history: return None @@ -729,15 +737,22 @@ hda_dicts = [] datasets = self.get_history_datasets( trans, history ) - for hda in datasets: - hda_dict = self.get_hda_dict( trans, hda ) - hda_dict[ 'annotation' ] = self.get_item_annotation_str( trans.sa_session, history.user, hda ) - hda_dicts.append( hda_dict ) - history_dict = self.get_history_dict( trans, history, hda_dictionaries=hda_dicts ) - history_dict[ 'annotation' ] = history.annotation + #for hda in datasets: + # hda_dict = self.get_hda_dict( trans, hda ) + # hda_dict[ 'annotation' ] = self.get_item_annotation_str( trans.sa_session, history.user, hda ) + # hda_dicts.append( hda_dict ) + #history_dict = self.get_history_dict( trans, history, hda_dictionaries=hda_dicts ) + #history_dict[ 'annotation' ] = history.annotation + + # include all datasets: hidden, deleted, and purged + #TODO!: doubled query (hda_dictionaries + datasets) + history_data = self.mgrs.histories._get_history_data( trans, history ) + history_dictionary = history_data[ 'history' ] + hda_dictionaries = history_data[ 'contents' ] + history_dictionary[ 'annotation' ] = history.annotation filled = trans.fill_template( "history/embed.mako", item=history, item_data=datasets, - user_is_owner=user_is_owner, history_dict=history_dict, hda_dicts=hda_dicts ) + user_is_owner=user_is_owner, history_dict=history_dictionary, hda_dicts=hda_dictionaries ) return filled def _get_embed_html( self, trans, item_class, item_id ): diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 static/scripts/mvc/base-mvc.js --- a/static/scripts/mvc/base-mvc.js +++ b/static/scripts/mvc/base-mvc.js @@ -14,7 +14,7 @@ * @example * // Add to your models/views at the definition using chaining: * var MyModel = Backbone.Model.extend( LoggableMixin ).extend({ // ... }); - * + * * // or - more explicitly AFTER the definition: * var MyModel = Backbone.Model.extend({ * logger : console @@ -208,8 +208,23 @@ }; //============================================================================== -return { - LoggableMixin : LoggableMixin, - SessionStorageModel : SessionStorageModel, - HiddenUntilActivatedViewMixin : HiddenUntilActivatedViewMixin -};}); +function mixin( mixinHash1, /* mixinHash2, etc: ... variadic */ propsHash ){ + // usage: var NewModel = Something.extend( mixin( MyMixinA, MyMixinB, { ... }) ); + //NOTE: this does not combine any hashes (like events, etc.) and you're expected to handle that + + // simple reversal of param order on _.defaults() - to show mixins in top of definition + var args = Array.prototype.slice.call( arguments, 0 ), + lastArg = args.pop(); + args.unshift( lastArg ); + return _.defaults.apply( _, args ); +} + + +//============================================================================== + return { + LoggableMixin : LoggableMixin, + SessionStorageModel : SessionStorageModel, + HiddenUntilActivatedViewMixin : HiddenUntilActivatedViewMixin, + mixin : mixin + }; +}); diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 static/scripts/mvc/collection/collection-model.js --- /dev/null +++ b/static/scripts/mvc/collection/collection-model.js @@ -0,0 +1,250 @@ +define([ + "mvc/dataset/hda-model", + "mvc/base-mvc", + "utils/localization" +], function( HDA_MODEL, BASE_MVC, _l ){ +//============================================================================== +/** @class Backbone model for Dataset collection elements. + * DC Elements contain a sub-model named 'object'. This class moves that + * 'object' from the JSON in the attributes list to a full, instantiated + * sub-model found in this.object. This is done on intialization and + * everytime the 'change:object' event is fired. + * + * @borrows LoggableMixin#logger as #logger + * @borrows LoggableMixin#log as #log + * @constructs + */ +var DatasetCollectionElement = Backbone.Model.extend( BASE_MVC.LoggableMixin ).extend( +/** @lends DatasetCollectionElement.prototype */{ + //TODO:?? this model may be unneccessary - it reflects the api structure, but... + // if we munge the element with the element.object at parse, we can flatten the entire hierarchy + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, + + defaults : { + id : null, + model_class : 'DatasetCollectionElement', + element_identifier : null, + element_index : null, + element_type : null + }, + + /** Set up. + * @see Backbone.Collection#initialize + */ + initialize : function( model, options ){ + this.info( this + '.initialize:', model, options ); + options = options || {}; + //this._setUpListeners(); + + this.object = this._createObjectModel(); + this.on( 'change:object', function(){ + //console.log( 'change:object' ); +//TODO: prob. better to update the sub-model instead of re-creating it + this.object = this._createObjectModel(); + }); + }, + + _createObjectModel : function(){ + //console.log( '_createObjectModel', this.get( 'object' ), this.object ); + //TODO: same patterns as HDCA _createElementsModel - refactor to BASE_MVC.hasSubModel? + if( _.isUndefined( this.object ) ){ this.object = null; } + if( !this.get( 'object' ) ){ return this.object; } + + var object = this.get( 'object' ); + this.unset( 'object', { silent: true }); + + this.debug( 'DCE, element_type:', this.get( 'element_type' ) ); + switch( this.get( 'element_type' ) ){ + case 'dataset_collection': + this.object = new DatasetCollection( object ); + break; + case 'hda': + this.object = new HDA_MODEL.HistoryDatasetAssociation( object ); + break; + default: + throw new TypeError( 'Unknown element_type: ' + this.get( 'element_type' ) ); + } + return this.object; + }, + + /** String representation. */ + toString : function(){ + var objStr = ( this.object )?( '' + this.object ):( this.get( 'element_identifier' ) ); + return ([ 'DatasetCollectionElement(', objStr, ')' ].join( '' )); + } +}); + + +//============================================================================== +/** @class Backbone collection for DCEs. + * NOTE: used *only* in second level of list:paired collections (a + * collection that contains collections) + * + * @borrows LoggableMixin#logger as #logger + * @borrows LoggableMixin#log as #log + * @constructs + */ +var DatasetCollectionElementCollection = Backbone.Collection.extend( BASE_MVC.LoggableMixin ).extend( +/** @lends DatasetCollectionElementCollection.prototype */{ + model: DatasetCollectionElement, + + // comment this out to suppress log output + /** logger used to record this.log messages, commonly set to console */ + //logger : console, + + /** Set up. + * @see Backbone.Collection#initialize + */ + initialize : function( models, options ){ + options = options || {}; + this.info( this + '.initialize:', models, options ); + //this._setUpListeners(); + }, + + /** String representation. */ + toString : function(){ + return ([ 'DatasetCollectionElementCollection(', this.length, ')' ].join( '' )); + } +}); + + +//============================================================================== +/** @class Backbone model for Dataset Collections. + * DCs contain a bbone collection named 'elements' using the class found in + * this.collectionClass (gen. DatasetCollectionElementCollection). DCs move + * that 'object' from the JSON in the attributes list to a full, instantiated + * collection found in this.elements. This is done on intialization and + * everytime the 'change:elements' event is fired. + * + * @borrows LoggableMixin#logger as #logger + * @borrows LoggableMixin#log as #log + * @constructs + */ +var DatasetCollection = Backbone.Model.extend( BASE_MVC.LoggableMixin ).extend( +/** @lends ListDatasetCollection.prototype */{ + + //logger : console, + + /** default attributes for a model */ + defaults : { + collection_type : 'list' + }, + + collectionClass : DatasetCollectionElementCollection, + + /** */ + initialize : function( model, options ){ + this.info( 'DatasetCollection.initialize:', model, options ); + //historyContent.HistoryContent.prototype.initialize.call( this, attrs, options ); + this.elements = this._createElementsModel(); +//TODO:?? no way to use parse here? + this.on( 'change:elements', function(){ + this.log( 'change:elements' ); +//TODO: prob. better to update the collection instead of re-creating it + this.elements = this._createElementsModel(); + }); + }, + + /** move elements model attribute to full collection */ + _createElementsModel : function(){ + this.log( '_createElementsModel', this.get( 'elements' ), this.elements ); +//TODO: same patterns as DatasetCollectionElement _createObjectModel - refactor to BASE_MVC.hasSubModel? + var elements = this.get( 'elements' ) || []; + this.info( 'elements:', elements ); + this.unset( 'elements', { silent: true }); + this.elements = new this.collectionClass( elements ); + return this.elements; + }, + + hasDetails : function(){ +//TODO: this is incorrect for (accidentally) empty collections + return this.elements.length !== 0; + }, + + // ........................................................................ misc + /** String representation */ + toString : function(){ + var idAndName = [ this.get( 'id' ), this.get( 'name' ) || this.get( 'element_identifier' ) ]; + return 'DatasetCollection(' + ( idAndName.join(',') ) + ')'; + } +}); + + +//============================================================================== +/** @class Backbone collection for a collection of collection collections collecting correctly. */ +var DatasetCollectionCollection = Backbone.Collection.extend( BASE_MVC.LoggableMixin ).extend({ + + model: DatasetCollection, + + ///** logger used to record this.log messages, commonly set to console */ + //// comment this out to suppress log output + //logger : console, + + /** Set up. + * @see Backbone.Collection#initialize + */ + initialize : function( models, options ){ + options = options || {}; + this.info( 'DatasetCollectionCollection.initialize:', models, options ); + //this._setUpListeners(); + }, + + /** String representation. */ + toString : function(){ + return ([ 'DatasetCollectionCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//NOTE: the following prototypes may not be necessary - but I wanted to specifiy +// them (for now) and allow for the possibility of unique functionality +//============================================================================== +var ListDatasetCollection = DatasetCollection.extend( +/** @lends ListDatasetCollection.prototype */{ + + /** String representation. */ + toString : function(){ + return ([ 'ListDatasetCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//============================================================================== +var PairDatasetCollection = DatasetCollection.extend( +/** @lends ListDatasetCollection.prototype */{ + + /** String representation. */ + toString : function(){ + return ([ 'PairDatasetCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//============================================================================== +var ListPairedDatasetCollection = DatasetCollection.extend( +/** @lends ListDatasetCollection.prototype */{ + + // list:paired is the only collection that itself contains collections + //collectionClass : DatasetCollectionCollection, + + /** String representation. */ + toString : function(){ + return ([ 'ListPairedDatasetCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//============================================================================== + return { + DatasetCollectionElement : DatasetCollectionElement, + DatasetCollectionElementCollection : DatasetCollectionElementCollection, + DatasetCollection : DatasetCollection, + DatasetCollectionCollection : DatasetCollectionCollection, + ListDatasetCollection : ListDatasetCollection, + PairDatasetCollection : PairDatasetCollection, + ListPairedDatasetCollection : ListPairedDatasetCollection + }; +}); diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 static/scripts/mvc/collection/collection-panel.js --- /dev/null +++ b/static/scripts/mvc/collection/collection-panel.js @@ -0,0 +1,326 @@ +define([ + "mvc/collection/dataset-collection-base", + "mvc/dataset/hda-base", + "mvc/base-mvc", + "utils/localization" +], function( DC_BASE, HDA_BASE, BASE_MVC, _l ){ +/* ============================================================================= +TODO: + +============================================================================= */ +/** @class non-editable, read-only View/Controller for a dataset collection. + * @name CollectionPanel + * + * @augments Backbone.View + * @borrows LoggableMixin#logger as #logger + * @borrows LoggableMixin#log as #log + * @constructs + */ +var CollectionPanel = Backbone.View.extend( BASE_MVC.LoggableMixin ).extend( +/** @lends CollectionPanel.prototype */{ + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, + + tagName : 'div', + className : 'history-panel', + + /** (in ms) that jquery effects will use */ + fxSpeed : 'fast', + + // ......................................................................... SET UP + /** Set up the view, set up storage, bind listeners to HistoryContents events + * @param {Object} attributes optional settings for the panel + */ + initialize : function( attributes ){ + attributes = attributes || {}; + // set the logger if requested + if( attributes.logger ){ + this.logger = attributes.logger; + } + this.log( this + '.initialize:', attributes ); + + this.hasUser = attributes.hasUser; + this.HDAViewClass = attributes.HDAViewClass || HDA_BASE.HDABaseView; + }, + + /** create any event listeners for the panel + * @fires: rendered:initial on the first render + * @fires: empty-history when switching to a history with no HDAs or creating a new history + */ + _setUpListeners : function(){ + // debugging + //if( this.logger ){ + this.on( 'all', function( event ){ + this.log( this + '', arguments ); + }, this ); + //} + return this; + }, + + // ------------------------------------------------------------------------ history/hda event listening + /** listening for history and HDA events */ + _setUpModelEventHandlers : function(){ + return this; + }, + + // ------------------------------------------------------------------------ panel rendering + /** Render urls, historyPanel body, and hdas (if any are shown) + * @fires: rendered when the panel is attached and fully visible + * @see Backbone.View#render + */ + render : function( speed, callback ){ + this.log( 'render:', speed, callback ); + // send a speed of 0 to have no fade in/out performed + speed = ( speed === undefined )?( this.fxSpeed ):( speed ); + //console.debug( this + '.render, fxSpeed:', speed ); + var panel = this, + $newRender; + + // handle the possibility of no model (can occur if fetching the model returns an error) + if( !this.model ){ + return this; + } + $newRender = this.renderModel(); + + // fade out existing, swap with the new, fade in, set up behaviours + $( panel ).queue( 'fx', [ + function( next ){ + if( speed && panel.$el.is( ':visible' ) ){ + panel.$el.fadeOut( speed, next ); + } else { + next(); + } + }, + function( next ){ + // swap over from temp div newRender + panel.$el.empty(); + if( $newRender ){ + panel.$el.append( $newRender.children() ); + } + next(); + }, + function( next ){ + if( speed && !panel.$el.is( ':visible' ) ){ + panel.$el.fadeIn( speed, next ); + } else { + next(); + } + }, + function( next ){ + //TODO: ideally, these would be set up before the fade in (can't because of async save text) + if( callback ){ callback.call( this ); } + panel.trigger( 'rendered', this ); + next(); + } + ]); + return this; + }, + + /** render with history data + * @returns {jQuery} dom fragment as temporary container to be swapped out later + */ + renderModel : function( ){ + // tmp div for final swap in render + var $newRender = $( '<div/>' ).append( CollectionPanel.templates.panel( this.model.toJSON() ) ); + this._setUpBehaviours( $newRender ); + this.renderContents( $newRender ); + return $newRender; + }, + + /** Set up HistoryPanel js/widget behaviours */ + _setUpBehaviours : function( $where ){ + //TODO: these should be either sub-MVs, or handled by events + $where = $where || this.$el; + $where.find( '[title]' ).tooltip({ placement: 'bottom' }); + return this; + }, + + // ------------------------------------------------------------------------ sub-$element shortcuts + /** the scroll container for this panel - can be $el, $el.parent(), or grandparent depending on context */ + $container : function(){ + return ( this.findContainerFn )?( this.findContainerFn.call( this ) ):( this.$el.parent() ); + }, + /** where hdaViews are attached */ + $datasetsList : function( $where ){ + return ( $where || this.$el ).find( '.datasets-list' ); + }, + + // ------------------------------------------------------------------------ sub-views + /** Set up/render a view for each HDA to be shown, init with model and listeners. + * HDA views are cached to the map this.hdaViews (using the model.id as key). + * @param {jQuery} $whereTo what dom element to prepend the HDA views to + * @returns the number of visible hda views + */ + renderContents : function( $whereTo ){ + //console.debug( 'renderContents, elements:', this.model.elements ); + $whereTo = $whereTo || this.$el; + + var panel = this, + contentViews = {}, + visibleContents = this.model.elements || []; + //this.log( 'renderContents, visibleContents:', visibleContents, $whereTo ); + + this.$datasetsList( $whereTo ).empty(); + if( visibleContents && visibleContents.length ){ + visibleContents.each( function( content ){ + var contentId = content.id, + contentView = panel._createContentView( content ); + contentViews[ contentId ] = contentView; + panel.attachContentView( contentView.render(), $whereTo ); + }); + } + this.contentViews = contentViews; + return this.contentViews; + }, + + /** + * @param {HistoryDatasetAssociation} content + */ + _createContentView : function( content ){ + //console.debug( 'content json:', JSON.stringify( content, null, ' ' ) ); + var contentView = null, + ContentClass = this._getContentClass( content ); + //console.debug( 'content.object json:', JSON.stringify( content.object, null, ' ' ) ); + //console.debug( 'ContentClass:', ContentClass ); + //console.debug( 'content:', content ); + //console.debug( 'content.object:', content.object ); + contentView = new ContentClass({ + model : content.object, + linkTarget : this.linkTarget, + //draggable : true, + hasUser : this.hasUser, + logger : this.logger + }); + //this._setUpHdaListeners( contentView ); + return contentView; + }, + + _getContentClass : function( content ){ + switch( content.get( 'element_type' ) ){ + case 'hda': + return this.HDAViewClass; + case 'dataset_collection': + return DC_BASE.NestedDCEBaseView; + } + throw new TypeError( 'Unknown element type:', content.get( 'element_type' ) ); + }, + +// /** Set up HistoryPanel listeners for HDAView events. Currently binds: +// * HDAView#body-visible, HDAView#body-hidden to store expanded states +// * @param {HDAView} hdaView HDAView (base or edit) to listen to +// */ +// _setUpHdaListeners : function( hdaView ){ +// var panel = this; +// hdaView.on( 'error', function( model, xhr, options, msg ){ +// panel.errorHandler( model, xhr, options, msg ); +// }); +// // maintain a list of hdas whose bodies are expanded +// hdaView.on( 'body-expanded', function( model ){ +// panel.storage.addExpandedHda( model ); +// }); +// hdaView.on( 'body-collapsed', function( id ){ +// panel.storage.removeExpandedHda( id ); +// }); +// return this; +// }, + + /** attach an contentView to the panel */ + attachContentView : function( contentView, $whereTo ){ + $whereTo = $whereTo || this.$el; + var $datasetsList = this.$datasetsList( $whereTo ); + $datasetsList.append( contentView.$el ); + return this; + }, + + // ------------------------------------------------------------------------ panel events + /** event map */ + events : { + 'click .panel-navigation-back' : 'close' + }, + + /** */ + close : function( event ){ + this.$el.remove(); + this.trigger( 'collection-close' ); + }, + + // ........................................................................ misc + /** Return a string rep of the history */ + toString : function(){ + return 'CollectionPanel(' + (( this.model )?( this.model.get( 'name' )):( '' )) + ')'; + } +}); + + +//------------------------------------------------------------------------------ TEMPLATES +var _panelTemplate = [ + '<div class="history-controls">', + '<div class="panel-navigation">', + '<a class="panel-navigation-back" href="javascript:void(0)">', _l( 'Back' ), '</a>', + '</div>', + + '<div class="history-title">', + '<% if( collection.name ){ %>', + '<div class="history-name"><%= collection.hid %> : <%= collection.name %></div>', + '<% } %>', + '</div>', + + //'<div class="history-subtitle clear">', + // '<% if( history.nice_size ){ %>', + // '<div class="history-size"><%= history.nice_size %></div>', + // '<% } %>', + // '<div class="history-secondary-actions"></div>', + //'</div>', + // + //'<% if( history.deleted ){ %>', + // '<div class="warningmessagesmall"><strong>', + // _l( 'You are currently viewing a deleted history!' ), + // '</strong></div>', + //'<% } %>', + // + //'<div class="message-container">', + // '<% if( history.message ){ %>', + // // should already be localized + // '<div class="<%= history.status %>message"><%= history.message %></div>', + // '<% } %>', + //'</div>', + // + //'<div class="quota-message errormessage">', + // _l( 'You are over your disk quota' ), '. ', + // _l( 'Tool execution is on hold until your disk usage drops below your allocated quota' ), '.', + //'</div>', + // + //'<div class="tags-display"></div>', + //'<div class="annotation-display"></div>', + //'<div class="history-dataset-actions">', + // '<div class="btn-group">', + // '<button class="history-select-all-datasets-btn btn btn-default"', + // 'data-mode="select">', _l( 'All' ), '</button>', + // '<button class="history-deselect-all-datasets-btn btn btn-default"', + // 'data-mode="select">', _l( 'None' ), '</button>', + // '</div>', + // '<button class="history-dataset-action-popup-btn btn btn-default">', + // _l( 'For all selected' ), '...</button>', + //'</div>', + '</div>', + // end history controls + + // where the datasets/hdas are added + '<div class="datasets-list"></div>' + +].join( '' ); + +CollectionPanel.templates = { + panel : function( JSON ){ + return _.template( _panelTemplate, JSON, { variable: 'collection' }); + } +}; + + +//============================================================================== + return { + CollectionPanel: CollectionPanel + }; +}); diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 static/scripts/mvc/collection/dataset-collection-base.js --- a/static/scripts/mvc/collection/dataset-collection-base.js +++ b/static/scripts/mvc/collection/dataset-collection-base.js @@ -1,328 +1,151 @@ define([ - "mvc/history/history-content-base-view", + "mvc/base-mvc", "utils/localization" -], function( historyContentBaseView, _l ){ +], function( BASE_MVC, _l ){ /* global Backbone, LoggableMixin */ //============================================================================== -/** @class Read only view for HistoryDatasetCollectionAssociation. - * @name HDABaseView +/** @class Read only view for DatasetCollection. + * @name DCBaseView * * @augments Backbone.View * @borrows LoggableMixin#logger as #logger * @borrows LoggableMixin#log as #log * @constructs */ -var DatasetCollectionBaseView = historyContentBaseView.HistoryContentBaseView.extend({ - className : "dataset hda history-panel-hda", - id : function(){ return 'hdca-' + this.model.get( 'id' ); }, +var DCBaseView = Backbone.View.extend( BASE_MVC.LoggableMixin ).extend({ + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, + + /** */ + className : "dataset-collection", + /** */ + fxSpeed : 'fast', /** */ initialize : function( attributes ){ if( attributes.logger ){ this.logger = this.model.logger = attributes.logger; } - this.log( this + '.initialize:', attributes ); - /** is the view currently in selection mode? */ - this.selectable = attributes.selectable || false; - //this.log( '\t selectable:', this.selectable ); - /** is the view currently selected? */ - this.selected = attributes.selected || false; - /** is the body of this collection view expanded/not? */ - this.expanded = attributes.expanded || false; + this.log( 'DCBaseView.initialize:', attributes ); }, - /** */ - render : function( fade ){ - var $newRender = this._buildNewRender(); - - this._queueNewRender( $newRender, fade ); - return this; - }, - - /** */ - _buildNewRender : function(){ - var $newRender = $( DatasetCollectionBaseView.templates.skeleton( this.model.toJSON() ) ); - $newRender.find( '.dataset-primary-actions' ).append( this._render_titleButtons() ); - $newRender.children( '.dataset-body' ).replaceWith( this._render_body() ); - this._setUpBehaviors( $newRender ); - return $newRender; - }, - - /** */ - _queueNewRender : function( $newRender, fade ) { - fade = ( fade === undefined )?( true ):( fade ); - var view = this; - - // fade the old render out (if desired) - if( fade ){ - $( view ).queue( function( next ){ this.$el.fadeOut( view.fxSpeed, next ); }); - } - // empty the old render, update to any new HDA state, swap in the new render contents, handle multi-select - $( view ).queue( function( next ){ - this.$el.empty() - .attr( 'class', view.className ).addClass( 'state-' + view.model.get( 'state' ) ) - .append( $newRender.children() ); - if( this.selectable ){ this.showSelector( 0 ); } - next(); - }); - // fade the new in - if( fade ){ - $( view ).queue( function( next ){ this.$el.fadeIn( view.fxSpeed, next ); }); - } - // trigger an event to know we're ready - $( view ).queue( function( next ){ - this.trigger( 'rendered', view ); - if( this.model.inReadyState() ){ - this.trigger( 'rendered:ready', view ); - } - if( this.draggable ){ this.draggableOn(); } - next(); - }); - }, - - // ................................................................................ titlebar buttons - /** Render icon-button group for the common, most easily accessed actions. - * @returns {jQuery} rendered DOM - */ - _render_titleButtons : function(){ - // render just the display for read-only - return [ ]; - }, - - // ......................................................................... state body renderers - /** Render the enclosing div of the collection body and, if expanded, the html in the body - * @returns {jQuery} rendered DOM - */ - _render_body : function(){ - var $body = $( '<div>Error: unknown state "' + this.model.get( 'state' ) + '".</div>' ), - // cheesy: get function by assumed matching name - renderFn = this[ '_render_body_' + this.model.get( 'state' ) ]; - if( _.isFunction( renderFn ) ){ - $body = renderFn.call( this ); - } - this._setUpBehaviors( $body ); - - // only render the body html if it's being shown - if( this.expanded ){ - $body.show(); - } - return $body; - }, - - /** set up js behaviors, event handlers for elements within the given container - * @param {jQuery} $container jq object that contains the elements to process (defaults to this.$el) - */ - _setUpBehaviors : function( $container ){ - $container = $container || this.$el; - // set up canned behavior on children (bootstrap, popupmenus, editable_text, etc.) - make_popup_menus( $container ); - $container.find( '[title]' ).tooltip({ placement : 'bottom' }); - }, - - // TODO: Eliminate duplication between following event map and one for HDAs. - - // ......................................................................... events - /** event map */ - events : { - // expand the body when the title is clicked or when in focus and space or enter is pressed - 'click .dataset-title-bar' : 'toggleBodyVisibility', - 'keydown .dataset-title-bar' : 'toggleBodyVisibility', - - // toggle selected state - 'click .dataset-selector' : 'toggleSelect' - }, - - /** Show or hide the body/details of history content. - * note: if the model does not have detailed data, fetch that data before showing the body - * @param {Event} event the event that triggered this (@link HDABaseView#events) - * @param {Boolean} expanded if true, expand; if false, collapse - * @fires body-expanded when a body has been expanded - * @fires body-collapsed when a body has been collapsed - */ - toggleBodyVisibility : function( event, expand ){ - // bail (with propagation) if keydown and not space or enter - var KEYCODE_SPACE = 32, KEYCODE_RETURN = 13; - if( event && ( event.type === 'keydown' ) - && !( event.keyCode === KEYCODE_SPACE || event.keyCode === KEYCODE_RETURN ) ){ - return true; - } - - var $body = this.$el.find( '.dataset-body' ); - expand = ( expand === undefined )?( !$body.is( ':visible' ) ):( expand ); - if( expand ){ - this.expandBody(); - } else { - this.collapseBody(); - } - return false; - }, - - /** Render and show the full, detailed body of this view including extra data and controls. - * @fires body-expanded when a body has been expanded - */ - expandBody : function(){ - var contentView = this; - - function _renderBodyAndExpand(){ - contentView.$el.children( '.dataset-body' ).replaceWith( contentView._render_body() ); - contentView.$el.children( '.dataset-body' ).slideDown( contentView.fxSpeed, function(){ - contentView.expanded = true; - contentView.trigger( 'body-expanded', contentView.model ); - }); - } - // TODO: Fetch more details like HDA view... - _renderBodyAndExpand(); - }, - - /** Hide the body/details of an HDA. - * @fires body-collapsed when a body has been collapsed - */ - collapseBody : function(){ - var hdaView = this; - this.$el.children( '.dataset-body' ).slideUp( hdaView.fxSpeed, function(){ - hdaView.expanded = false; - hdaView.trigger( 'body-collapsed', hdaView.model.id ); - }); - }, - - /** Render an 'ok' collection. - * @param {jQuery} parent DOM to which to append this body - */ - _render_body_ok : function(){ - // most common state renderer and the most complicated - var $body = $( DatasetCollectionBaseView.templates.body( this.model.toJSON() ) ); - - // return shortened form if del'd (no display apps or peek?) - if( this.model.get( 'deleted' ) ){ - return $body; - } - - return $body; - }, - - // ......................................................................... selection - /** display a (fa-icon) checkbox on the left of the hda that fires events when checked - * Note: this also hides the primary actions - */ - showSelector : function(){ - // make sure selected state is represented properly - if( this.selected ){ - this.select( null, true ); - } - - this.selectable = true; - this.trigger( 'selectable', true, this ); - - this.$( '.dataset-primary-actions' ).hide(); - this.$( '.dataset-selector' ).show(); - }, - - /** remove the selection checkbox */ - hideSelector : function(){ - // reverse the process from showSelect - this.selectable = false; - this.trigger( 'selectable', false, this ); - - this.$( '.dataset-selector' ).hide(); - this.$( '.dataset-primary-actions' ).show(); - }, - - toggleSelector : function(){ - if( !this.$el.find( '.dataset-selector' ).is( ':visible' ) ){ - this.showSelector(); - } else { - this.hideSelector(); - } - }, - - /** event handler for selection (also programmatic selection) - */ - select : function( event ){ - // switch icon, set selected, and trigger event - this.$el.find( '.dataset-selector span' ) - .removeClass( 'fa-square-o' ).addClass( 'fa-check-square-o' ); - if( !this.selected ){ - this.trigger( 'selected', this, event ); - this.selected = true; - } - return false; - }, - - /** event handler for clearing selection (also programmatic deselection) - */ - deselect : function( event ){ - // switch icon, set selected, and trigger event - this.$el.find( '.dataset-selector span' ) - .removeClass( 'fa-check-square-o' ).addClass( 'fa-square-o' ); - if( this.selected ){ - this.trigger( 'de-selected', this, event ); - this.selected = false; - } - return false; - }, - - toggleSelect : function( event ){ - if( this.selected ){ - this.deselect( event ); - } else { - this.select( event ); - } - }, +//TODO: render has been removed from the inheritance chain here, so this won't work when called as is // ......................................................................... misc /** String representation */ toString : function(){ var modelString = ( this.model )?( this.model + '' ):( '(no model)' ); - return 'HDCABaseView(' + modelString + ')'; + return 'DCBaseView(' + modelString + ')'; } }); -//------------------------------------------------------------------------------ TEMPLATES -//TODO: possibly break these out into a sep. module -var skeletonTemplate = _.template([ - '<div class="dataset hda">', - '<div class="dataset-warnings">', - '<% if ( collection.deleted ) { %>', - '<div class="dataset-deleted-msg warningmessagesmall"><strong>', - _l( 'This collection has been deleted.' ), +/** templates for DCBaseViews (skeleton and body) */ +DCBaseView.templates = (function(){ +// use closure to run underscore template fn only once at module load + var skeletonTemplate = _.template([ + '<div class="dataset hda">', + '<div class="dataset-warnings">', + '<% if ( collection.deleted ) { %>', + '<div class="dataset-deleted-msg warningmessagesmall"><strong>', + _l( 'This collection has been deleted.' ), + '</div>', + '<% } %>', + '<% if ( !collection.visible ) { %>', + '<div class="dataset-hidden-msg warningmessagesmall"><strong>', + _l( 'This collection has been hidden.' ), + '</div>', + '<% } %>', + '</div>', + '<div class="dataset-primary-actions"></div>', + '<div class="dataset-title-bar clear" tabindex="0">', + '<span class="dataset-state-icon state-icon"></span>', + '<div class="dataset-title">', + '<span class="dataset-name"><%- collection.name %></span>', '</div>', - '<% } %>', - '<% if ( !collection.visible ) { %>', - '<div class="dataset-hidden-msg warningmessagesmall"><strong>', - _l( 'This collection has been hidden.' ), - '</div>', - '<% } %>', - '</div>', - '<div class="dataset-selector"><span class="fa fa-2x fa-square-o"></span></div>', - '<div class="dataset-primary-actions"></div>', - '<div class="dataset-title-bar clear" tabindex="0">', - '<span class="dataset-state-icon state-icon"></span>', - '<div class="dataset-title">', - '<span class="hda-hid"><%= collection.hid %></span> ', - '<span class="dataset-name"><%= collection.name %></span>', '</div>', - '</div>', - '<div class="dataset-body"></div>', - '</div>' -].join( '' )); + '<div class="dataset-body"></div>', + '</div>' + ].join( '' )); -var bodyTemplate = _.template([ - '<div class="dataset-body">', - '<div class="dataset-summary">', - _l( 'A dataset collection.' ), - '</div>' -].join( '' )); + var bodyTemplate = _.template([ + '<div class="dataset-body">', + '<div class="dataset-summary">', + _l( 'A dataset collection.' ), + '</div>' + ].join( '' )); -DatasetCollectionBaseView.templates = { // we override here in order to pass the localizer (_L) into the template scope - since we use it as a fn within - skeleton : function( collectionJSON ){ - return skeletonTemplate({ _l: _l, collection: collectionJSON }); - }, - body : function( collectionJSON ){ - return bodyTemplate({ _l: _l, collection: collectionJSON }); - } -}; + return { + skeleton : function( collectionJSON ){ + return skeletonTemplate({ _l: _l, collection: collectionJSON }); + }, + body : function( collectionJSON ){ + return bodyTemplate({ _l: _l, collection: collectionJSON }); + } + }; +}()); + + +//TODO: unused +////============================================================================== +///** @class Read only view for DatasetCollectionElement. +// * @name DCEBaseView +// * +// * @augments Backbone.View +// * @borrows LoggableMixin#logger as #logger +// * @borrows LoggableMixin#log as #log +// * @constructs +// */ +//var NestedDCBaseView = DCBaseView.extend({ +// +// logger : console, +// +// /** */ +// initialize : function( attributes ){ +// if( attributes.logger ){ this.logger = this.model.logger = attributes.logger; } +// this.warn( this + '.initialize:', attributes ); +// DCBaseView.prototype.initialize.call( this, attributes ); +// }, +// +// _template : function(){ +// this.debug( this.model ); +// this.debug( this.model.toJSON() ); +// return NestedDCBaseView.templates.skeleton( this.model.toJSON() ); +// }, +// +// // ......................................................................... misc +// /** String representation */ +// toString : function(){ +// var modelString = ( this.model )?( this.model + '' ):( '(no model)' ); +// return 'NestedDCBaseView(' + modelString + ')'; +// } +//}); +// +////------------------------------------------------------------------------------ TEMPLATES +////TODO: possibly break these out into a sep. module +//NestedDCBaseView.templates = (function(){ +// var skeleton = _.template([ +// '<div class="dataset hda history-panel-hda state-ok">', +// '<div class="dataset-primary-actions"></div>', +// '<div class="dataset-title-bar clear" tabindex="0">', +// '<div class="dataset-title">', +// '<span class="dataset-name"><%= collection.name %></span>', +// '</div>', +// '</div>', +// '</div>' +// ].join( '' )); +// // we override here in order to pass the localizer (_L) into the template scope - since we use it as a fn within +// return { +// skeleton : function( json ){ +// return skeleton({ _l: _l, collection: json }); +// } +// }; +//}()); + //============================================================================== return { - DatasetCollectionBaseView : DatasetCollectionBaseView + DCBaseView : DCBaseView, + //NestedDCBaseView : NestedDCBaseView, }; }); diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 static/scripts/mvc/collection/dataset-collection-edit.js --- a/static/scripts/mvc/collection/dataset-collection-edit.js +++ b/static/scripts/mvc/collection/dataset-collection-edit.js @@ -1,75 +1,36 @@ define([ - "mvc/dataset/hda-model", + "mvc/dataset/states", "mvc/collection/dataset-collection-base", "utils/localization" -], function( hdaModel, datasetCollectionBase, _l ){ +], function( STATES, DC_BASE_VIEW, _l ){ //============================================================================== -/** @class Editing view for HistoryDatasetCollectionAssociation. +var _super = DC_BASE_VIEW.DCBaseView; +/** @class Editing view for DatasetCollection. * @name DatasetCollectionEditView * - * @augments DatasetCollectionBaseView + * @augments DCBaseView * @constructs */ -var DatasetCollectionEditView = datasetCollectionBase.DatasetCollectionBaseView.extend( { +var DCEditView = _super.extend({ + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, initialize : function( attributes ){ - datasetCollectionBase.DatasetCollectionBaseView.prototype.initialize.call( this, attributes ); - }, - - // ......................................................................... edit attr, delete - /** Render icon-button group for the common, most easily accessed actions. - * Overrides _render_titleButtons to include editing related buttons. - * @see DatasetCollectionBaseView#_render_titleButtons - * @returns {jQuery} rendered DOM - */ - _render_titleButtons : function(){ - // render the display, edit attr and delete icon-buttons - return datasetCollectionBase.DatasetCollectionBaseView.prototype._render_titleButtons.call( this ).concat([ - this._render_deleteButton() - ]); - }, - - /** Render icon-button to delete this hda. - * @returns {jQuery} rendered DOM - */ - _render_deleteButton : function(){ - // don't show delete if... - if( ( this.model.get( 'state' ) === hdaModel.HistoryDatasetAssociation.STATES.NEW ) - || ( this.model.get( 'state' ) === hdaModel.HistoryDatasetAssociation.STATES.NOT_VIEWABLE ) - || ( !this.model.get( 'accessible' ) ) ){ - return null; - } - - var self = this, - deleteBtnData = { - title : _l( 'Delete' ), - classes : 'dataset-delete', - onclick : function() { - // ...bler... tooltips being left behind in DOM (hover out never called on deletion) - self.$el.find( '.icon-btn.dataset-delete' ).trigger( 'mouseout' ); - self.model[ 'delete' ](); - } - }; - if( this.model.get( 'deleted' ) ){ - deleteBtnData = { - title : _l( 'Dataset collection is already deleted' ), - disabled : true - }; - } - deleteBtnData.faIcon = 'fa-times'; - return faIconButton( deleteBtnData ); + _super.prototype.initialize.call( this, attributes ); }, // ......................................................................... misc /** string rep */ toString : function(){ var modelString = ( this.model )?( this.model + '' ):( '(no model)' ); - return 'HDCAEditView(' + modelString + ')'; + return 'DCEditView(' + modelString + ')'; } }); //============================================================================== return { - DatasetCollectionEditView : DatasetCollectionEditView + DCEditView : DCEditView }; }); diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 static/scripts/mvc/collection/hdca-base.js --- /dev/null +++ b/static/scripts/mvc/collection/hdca-base.js @@ -0,0 +1,116 @@ +define([ + "mvc/base-mvc", + "mvc/collection/dataset-collection-base", + "mvc/history/history-content-base-view", + "utils/localization" +], function( BASE_MVC, DC_BASE, HISTORY_CONTENT_BASE_VIEW, _l ){ +/* global Backbone, LoggableMixin */ +//============================================================================== +var HCVMixin = HISTORY_CONTENT_BASE_VIEW.HistoryContentViewMixin, + _super = DC_BASE.DCBaseView; +/** @class Read only view for HistoryDatasetCollectionAssociation. + * @name + * + * @augments Backbone.View + * @borrows LoggableMixin#logger as #logger + * @borrows LoggableMixin#log as #log + * @constructs + */ +var HDCABaseView = _super.extend( BASE_MVC.mixin( HCVMixin, { + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, + + /** */ + className : "dataset hda history-panel-hda", + + /** */ + initialize : function( attributes ){ + if( attributes.logger ){ this.logger = this.model.logger = attributes.logger; } + this.log( this + '.initialize:', attributes ); + _super.prototype.initialize.call( this, attributes ); + HCVMixin.initialize.call( this, attributes ); + }, + + /** */ + _template : function(){ + return HDCABaseView.templates.skeleton( this.model.toJSON() ); + }, + + /** */ + events : _.extend( _.clone( HCVMixin.events ), { + }), + + /** */ + _renderBody : function(){ + // override this + var $body = $( _super.templates.body( this.model.toJSON() ) ); + if( this.expanded ){ + $body.show(); + } + return $body; + }, + + // ......................................................................... misc + /** String representation */ + toString : function(){ + var modelString = ( this.model )?( this.model + '' ):( '(no model)' ); + return 'HDCABaseView(' + modelString + ')'; + } +})); + +/** templates for HDCAs (skeleton and body) */ +HDCABaseView.templates = HDCABaseView.prototype.templates = (function(){ +// use closure to run underscore template fn only once at module load + var skeletonTemplate = _.template([ + '<div class="dataset hda">', + '<div class="dataset-warnings">', + '<% if ( collection.deleted ) { %>', + '<div class="dataset-deleted-msg warningmessagesmall"><strong>', + _l( 'This collection has been deleted.' ), + '</div>', + '<% } %>', + '<% if ( !collection.visible ) { %>', + '<div class="dataset-hidden-msg warningmessagesmall"><strong>', + _l( 'This collection has been hidden.' ), + '</div>', + '<% } %>', + '</div>', + '<div class="dataset-selector"><span class="fa fa-2x fa-square-o"></span></div>', + '<div class="dataset-primary-actions"></div>', + '<div class="dataset-title-bar clear" tabindex="0">', + '<span class="dataset-state-icon state-icon"></span>', + '<div class="dataset-title">', + '<span class="hda-hid"><%- collection.hid %></span> ', + '<span class="dataset-name"><%- collection.name %></span>', + '</div>', + '</div>', + '<div class="dataset-body"></div>', + '</div>' + ].join( '' )); + + var bodyTemplate = _.template([ + '<div class="dataset-body">', + '<div class="dataset-summary">', + _l( 'A dataset collection.' ), + '</div>' + ].join( '' )); + + // we override here in order to pass the localizer (_L) into the template scope - since we use it as a fn within + return { + skeleton : function( collectionJSON ){ + return skeletonTemplate({ _l: _l, collection: collectionJSON }); + }, + body : function( collectionJSON ){ + return bodyTemplate({ _l: _l, collection: collectionJSON }); + } + }; +}()); + + +//============================================================================== + return { + HDCABaseView : HDCABaseView + }; +}); diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 static/scripts/mvc/collection/hdca-edit.js --- /dev/null +++ b/static/scripts/mvc/collection/hdca-edit.js @@ -0,0 +1,74 @@ +define([ + "mvc/dataset/states", + "mvc/collection/hdca-base", + "utils/localization" +], function( STATES, HDCA_BASE, _l ){ +//============================================================================== +var _super = HDCA_BASE.HDCABaseView; +/** @class Editing view for HistoryDatasetCollectionAssociation. + * @name DatasetCollectionEditView + * + * @augments HDCABaseView + * @constructs + */ +var HDCAEditView = _super.extend({ + + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output + //logger : console, + + initialize : function( attributes ){ + _super.prototype.initialize.call( this, attributes ); + }, + + // ......................................................................... edit attr, delete + /** Render icon-button group for the common, most easily accessed actions. + * Overrides _render_titleButtons to include editing related buttons. + * @returns {jQuery} rendered DOM + */ + _render_titleButtons : function(){ + this.log( this + '._render_titleButtons' ); + // render the display, edit attr and delete icon-buttons + return _super.prototype._render_titleButtons.call( this ) + .concat([ + this._render_deleteButton() + ]); + }, + + /** Render icon-button to delete this hda. + * @returns {jQuery} rendered DOM + */ + _render_deleteButton : function(){ + var self = this, + deleteBtnData = { + title : _l( 'Delete' ), + classes : 'dataset-delete', + onclick : function() { + // ...bler... tooltips being left behind in DOM (hover out never called on deletion) + self.$el.find( '.icon-btn.dataset-delete' ).trigger( 'mouseout' ); + self.model[ 'delete' ](); + } + }; + if( self.model.get( 'deleted' ) ){ + deleteBtnData = { + title : _l( 'Dataset collection is already deleted' ), + disabled : true + }; + } + deleteBtnData.faIcon = 'fa-times'; + return faIconButton( deleteBtnData ); + }, + + // ......................................................................... misc + /** string rep */ + toString : function(){ + var modelString = ( this.model )?( this.model + '' ):( '(no model)' ); + return 'HDCAEditView(' + modelString + ')'; + } +}); + +//============================================================================== + return { + HDCAEditView : HDCAEditView + }; +}); diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 static/scripts/mvc/collection/hdca-model.js --- a/static/scripts/mvc/collection/hdca-model.js +++ b/static/scripts/mvc/collection/hdca-model.js @@ -1,69 +1,89 @@ define([ "mvc/history/history-content-base", + "mvc/collection/collection-model", "utils/localization" -], function( historyContent, _l ){ +], function( HISTORY_CONTENT, DC_MODEL, _l ){ //============================================================================== -var HistoryDatasetCollectionAssociation = historyContent.HistoryContent.extend( -/** @lends HistoryDatasetCollectionAssociation.prototype */{ - /** default attributes for a model */ - defaults : { - // parent (containing) history - history_id : null, - // often used with tagging - model_class : 'HistoryDatasetCollectionAssociation', - history_content_type : 'dataset_collection', - hid : 0, +var hcontentMixin = HISTORY_CONTENT.HistoryContentMixin, +/** @class Backbone model for (generic) Dataset Collection within a History. + * @constructs + */ + HistoryDatasetCollection = DC_MODEL.DatasetCollection.extend( hcontentMixin ); - id : null, - name : '(unnamed dataset collection)', - // one of HistoryDatasetAssociation.STATES, calling them all 'ok' for now. - state : 'ok', - accessible : true, - deleted : false, - visible : true, +//NOTE: the following prototypes may not be necessary - but I wanted to specifiy +// them (for now) and allow for the possibility of unique functionality +//============================================================================== +var ListDC = DC_MODEL.ListDatasetCollection, +/** @class Backbone model for List Dataset Collection within a History. + * @constructs + */ + HistoryListDatasetCollection = ListDC.extend( hcontentMixin ).extend( +/** @lends HistoryListDatasetCollection.prototype */{ - purged : false, // Purged doesn't make sense for collections - at least right now. - - tags : [], - annotation : '' - }, - urls : function(){ + initialize : function( model, options ){ + ListDC.prototype.initialize.call( this, model, options ); + hcontentMixin.initialize.call( this, model, options ); + //TODO: in lieu of any state info for collections, show as 'ok' + this.set( 'state', 'ok', { silent: true }); }, - inReadyState : function(){ - return true; // TODO - }, - - // ........................................................................ search - /** what attributes of an collection will be used in a text search */ - searchAttributes : [ - 'name' - ], - - /** our attr keys don't often match the labels we display to the user - so, when using - * attribute specifiers ('name="bler"') in a term, allow passing in aliases for the - * following attr keys. - */ - searchAliases : { - title : 'name' - // TODO: Add tag... - }, - - // ........................................................................ misc - /** String representation */ + /** String representation. */ toString : function(){ - var nameAndId = this.get( 'id' ) || ''; - if( this.get( 'name' ) ){ - nameAndId = this.get( 'hid' ) + ' :"' + this.get( 'name' ) + '",' + nameAndId; - } - return 'HDCA-' + this.get( 'collection_type' ) + '(' + nameAndId + ')'; + return ([ 'HistoryListDatasetCollection(', this.get( 'name' ), ')' ].join( '' )); } }); //============================================================================== +var PairDC = DC_MODEL.PairDatasetCollection, +/** @class Backbone model for Pair Dataset Collection within a History. + * @constructs + */ + HistoryPairDatasetCollection = PairDC.extend( hcontentMixin ).extend( +/** @lends HistoryPairDatasetCollection.prototype */{ + + initialize : function( model, options ){ + PairDC.prototype.initialize.call( this, model, options ); + hcontentMixin.initialize.call( this, model, options ); + //TODO: in lieu of any state info for collections, show as 'ok' + this.set( 'state', 'ok', { silent: true }); + }, + + /** String representation. */ + toString : function(){ + return ([ 'HistoryPairDatasetCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//============================================================================== +var ListPairedDC = DC_MODEL.ListPairedDatasetCollection, +/** @class Backbone model for List of Pairs Dataset Collection within a History. + * @constructs + */ + HistoryListPairedDatasetCollection = ListPairedDC.extend( hcontentMixin ).extend( +/** @lends HistoryListPairedDatasetCollection.prototype */{ + + initialize : function( model, options ){ + ListPairedDC.prototype.initialize.call( this, model, options ); + hcontentMixin.initialize.call( this, model, options ); + //TODO: in lieu of any state info for collections, show as 'ok' + this.set( 'state', 'ok', { silent: true }); + }, + + /** String representation. */ + toString : function(){ + return ([ 'HistoryListPairedDatasetCollection(', this.get( 'name' ), ')' ].join( '' )); + } +}); + + +//============================================================================== return { - HistoryDatasetCollectionAssociation : HistoryDatasetCollectionAssociation + HistoryDatasetCollection : HistoryDatasetCollection, + HistoryListDatasetCollection : HistoryListDatasetCollection, + HistoryPairDatasetCollection : HistoryPairDatasetCollection, + HistoryListPairedDatasetCollection : HistoryListPairedDatasetCollection }; }); diff -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 -r 598a4ec015f66237d41ed2176c673fd52b744f15 static/scripts/mvc/dataset/hda-base.js --- a/static/scripts/mvc/dataset/hda-base.js +++ b/static/scripts/mvc/dataset/hda-base.js @@ -1,11 +1,12 @@ define([ - "mvc/dataset/hda-model", + "mvc/dataset/states", "mvc/history/history-content-base-view", "mvc/data", "utils/localization" -], function( hdaModel, historyContentBaseView, dataset, _l ){ +], function( STATES, HCONTENT_BASE_VIEW, DATA, _l ){ /* global Backbone */ //============================================================================== +var _super = HCONTENT_BASE_VIEW.HistoryContentBaseView; /** @class Read only view for HistoryDatasetAssociation. * @name HDABaseView * @@ -14,11 +15,11 @@ * @borrows LoggableMixin#log as #log * @constructs */ -var HDABaseView = historyContentBaseView.HistoryContentBaseView.extend( +var HDABaseView = _super.extend( /** @lends HDABaseView.prototype */{ - ///** logger used to record this.log messages, commonly set to console */ - //// comment this out to suppress log output + /** logger used to record this.log messages, commonly set to console */ + // comment this out to suppress log output //logger : console, tagName : "div", @@ -38,6 +39,8 @@ if( attributes.logger ){ this.logger = this.model.logger = attributes.logger; } this.log( this + '.initialize:', attributes ); + _super.prototype.initialize.call( this, attributes ); + /** list of rendering functions for the default, primary icon-buttons. */ this.defaultPrimaryActionButtonRenderers = [ this._render_showParamsButton @@ -46,15 +49,6 @@ /** where should pages from links be displayed? (default to new tab/window) */ this.linkTarget = attributes.linkTarget || '_blank'; - /** is the view currently in selection mode? */ - this.selectable = attributes.selectable || false; - //this.log( '\t selectable:', this.selectable ); - /** is the view currently selected? */ - this.selected = attributes.selected || false; - //this.log( '\t selected:', this.selected ); - /** is the body of this hda view expanded/not? */ - this.expanded = attributes.expanded || false; - //this.log( '\t expanded:', this.expanded ); /** is the body of this hda view expanded/not? */ this.draggable = attributes.draggable || false; //this.log( '\t draggable:', this.draggable ); @@ -86,77 +80,20 @@ // ......................................................................... render main /** Render this HDA, set up ui. * @param {Boolean} fade whether or not to fade out/in when re-rendering - * @fires rendered when rendered - * @fires rendered:ready when first rendered and NO running HDAs * @returns {Object} this HDABaseView */ render : function( fade ){ //HACK: hover exit doesn't seem to be called on prev. tooltips when RE-rendering - so: no tooltip hide // handle that here by removing previous view's tooltips this.$el.find("[title]").tooltip( "destroy" ); - // re-get web controller urls for functions relating to this hda. (new model data may have changed this) this.urls = this.model.urls(); - var $newRender = this._buildNewRender(); - this._queueNewRender( $newRender, fade ); - return this; - + return _super.prototype.render.call( this, fade ); }, - _buildNewRender : function(){ - // create a new render using a skeleton template, render title buttons, render body, and set up events, etc. - var $newRender = $( HDABaseView.templates.skeleton( this.model.toJSON() ) ); - $newRender.find( '.dataset-primary-actions' ).append( this._render_titleButtons() ); - $newRender.children( '.dataset-body' ).replaceWith( this._render_body() ); - this._setUpBehaviors( $newRender ); - //this._renderSelectable( $newRender ); - return $newRender; - }, - - _queueNewRender : function( $newRender, fade ) { - fade = ( fade === undefined )?( true ):( fade ); - var view = this; - - // fade the old render out (if desired) - if( fade ){ - $( view ).queue( function( next ){ this.$el.fadeOut( view.fxSpeed, next ); }); - } - // empty the old render, update to any new HDA state, swap in the new render contents, handle multi-select - $( view ).queue( function( next ){ - this.$el.empty() - .attr( 'class', view.className ).addClass( 'state-' + view.model.get( 'state' ) ) - .append( $newRender.children() ); - if( this.selectable ){ this.showSelector( 0 ); } - next(); - }); - // fade the new in - if( fade ){ - $( view ).queue( function( next ){ this.$el.fadeIn( view.fxSpeed, next ); }); - } - // trigger an event to know we're ready - $( view ).queue( function( next ){ - this.trigger( 'rendered', view ); - if( this.model.inReadyState() ){ - this.trigger( 'rendered:ready', view ); - } - if( this.draggable ){ this.draggableOn(); } - next(); - }); - }, - - /** set up js behaviors, event handlers for elements within the given container - * @param {jQuery} $container jq object that contains the elements to process (defaults to this.$el) - */ - _setUpBehaviors : function( $container ){ - $container = $container || this.$el; - // set up canned behavior on children (bootstrap, popupmenus, editable_text, etc.) - make_popup_menus( $container ); - $container.find( '[title]' ).tooltip({ placement : 'bottom' }); - }, - // ................................................................................ titlebar buttons - /** Render icon-button group for the common, most easily accessed actions. + /** In this override, render the dataset display button * @returns {jQuery} rendered DOM */ _render_titleButtons : function(){ @@ -170,8 +107,8 @@ _render_displayButton : function(){ // don't show display if not viewable or not accessible // (do show if in error, running) - if( ( this.model.get( 'state' ) === hdaModel.HistoryDatasetAssociation.STATES.NOT_VIEWABLE ) - || ( this.model.get( 'state' ) === hdaModel.HistoryDatasetAssociation.STATES.DISCARDED ) + if( ( this.model.get( 'state' ) === STATES.NOT_VIEWABLE ) + || ( this.model.get( 'state' ) === STATES.DISCARDED ) || ( !this.model.get( 'accessible' ) ) ){ return null; } @@ -187,12 +124,12 @@ displayBtnData.title = _l( 'Cannot display datasets removed from disk' ); // disable if still uploading - } else if( this.model.get( 'state' ) === hdaModel.HistoryDatasetAssociation.STATES.UPLOAD ){ + } else if( this.model.get( 'state' ) === STATES.UPLOAD ){ displayBtnData.disabled = true; displayBtnData.title = _l( 'This dataset must finish uploading before it can be viewed' ); // disable if still new - } else if( this.model.get( 'state' ) === hdaModel.HistoryDatasetAssociation.STATES.NEW ){ + } else if( this.model.get( 'state' ) === STATES.NEW ){ displayBtnData.disabled = true; displayBtnData.title = _l( 'This dataset is not yet viewable' ); @@ -211,9 +148,9 @@ title : "Data Viewer: " + self.model.get( 'name' ), type : "other", content : function( parent_elt ){ - var new_dataset = new dataset.TabularDataset({ id: self.model.get( 'id' ) }); + var new_dataset = new DATA.TabularDataset({ id: self.model.get( 'id' ) }); $.when( new_dataset.fetch() ).then( function(){ - dataset.createTabularDatasetChunkedView({ + DATA.createTabularDatasetChunkedView({ model: new_dataset, parent_elt: parent_elt, embedded: true, @@ -298,7 +235,7 @@ /** Render the enclosing div of the hda body and, if expanded, the html in the body * @returns {jQuery} rendered DOM */ - _render_body : function(){ + _renderBody : function(){ var $body = $( '<div>Error: unknown dataset state "' + this.model.get( 'state' ) + '".</div>' ), // cheesy: get function by assumed matching name renderFn = this[ '_render_body_' + this.model.get( 'state' ) ]; @@ -437,61 +374,23 @@ // ......................................................................... events /** event map */ - events : { - // expand the body when the title is clicked or when in focus and space or enter is pressed - 'click .dataset-title-bar' : 'toggleBodyVisibility', - 'keydown .dataset-title-bar' : 'toggleBodyVisibility', + events : _.extend( _.clone( _super.prototype.events ), { + }), - // dragging - don't work, originalEvent === null - //'dragstart .dataset-title-bar' : 'dragStartHandler', - //'dragend .dataset-title-bar' : 'dragEndHandler' - - // toggle selected state - 'click .dataset-selector' : 'toggleSelect' - }, - - /** Show or hide the body/details of history content. - * note: if the model does not have detailed data, fetch that data before showing the body - * @param {Event} event the event that triggered this (@link HDABaseView#events) - * @param {Boolean} expanded if true, expand; if false, collapse - * @fires body-expanded when a body has been expanded - * @fires body-collapsed when a body has been collapsed + /** Override for expanding hda details (within the panel) + * note: in this override, the fetch for details does *not* fire a change event (silent == true) + * @fires expanded when a body has been expanded */ - toggleBodyVisibility : function( event, expand ){ - // bail (with propagation) if keydown and not space or enter - var KEYCODE_SPACE = 32, KEYCODE_RETURN = 13; - if( event && ( event.type === 'keydown' ) - && !( event.keyCode === KEYCODE_SPACE || event.keyCode === KEYCODE_RETURN ) ){ - return true; - } - - var $body = this.$el.find( '.dataset-body' ); - expand = ( expand === undefined )?( !$body.is( ':visible' ) ):( expand ); - if( expand ){ - this.expandBody(); - } else { - this.collapseBody(); - } - return false; - }, - - /** Render and show the full, detailed body of this view including extra data and controls. - * @fires body-expanded when a body has been expanded - */ - expandBody : function(){ + expand : function(){ var hdaView = this; function _renderBodyAndExpand(){ - hdaView.$el.children( '.dataset-body' ).replaceWith( hdaView._render_body() ); + hdaView.$el.children( '.dataset-body' ).replaceWith( hdaView._renderBody() ); + //NOTE: needs to be set after the above or the slide will not show + hdaView.expanded = true; hdaView.$el.children( '.dataset-body' ).slideDown( hdaView.fxSpeed, function(){ - hdaView.expanded = true; - hdaView.trigger( 'body-expanded', hdaView.model ); + hdaView.trigger( 'expanded', hdaView.model ); }); - - //hdaView.render( false ).$el.children( '.dataset-body' ).slideDown( hdaView.fxSpeed, function(){ - // hdaView.expanded = true; - // hdaView.trigger( 'body-expanded', hdaView.model.get( 'id' ) ); - //}); } // fetch first if no details in the model if( this.model.inReadyState() && !this.model.hasDetails() ){ @@ -505,128 +404,6 @@ } }, - /** Hide the body/details of an HDA. - * @fires body-collapsed when a body has been collapsed - */ - collapseBody : function(){ - var hdaView = this; - this.$el.children( '.dataset-body' ).slideUp( hdaView.fxSpeed, function(){ - hdaView.expanded = false; - hdaView.trigger( 'body-collapsed', hdaView.model.id ); - }); - }, - - // ......................................................................... selection - /** display a (fa-icon) checkbox on the left of the hda that fires events when checked - * Note: this also hides the primary actions - */ - showSelector : function(){ - // make sure selected state is represented properly - if( this.selected ){ - this.select( null, true ); - } - - this.selectable = true; - this.trigger( 'selectable', true, this ); - - this.$( '.dataset-primary-actions' ).hide(); - this.$( '.dataset-selector' ).show(); - }, - - /** remove the selection checkbox */ - hideSelector : function(){ - // reverse the process from showSelect - this.selectable = false; - this.trigger( 'selectable', false, this ); - - this.$( '.dataset-selector' ).hide(); - this.$( '.dataset-primary-actions' ).show(); - }, - - toggleSelector : function(){ - if( !this.$el.find( '.dataset-selector' ).is( ':visible' ) ){ - this.showSelector(); - } else { - this.hideSelector(); - } - }, - - /** event handler for selection (also programmatic selection) - */ - select : function( event ){ - // switch icon, set selected, and trigger event - this.$el.find( '.dataset-selector span' ) - .removeClass( 'fa-square-o' ).addClass( 'fa-check-square-o' ); - if( !this.selected ){ - this.trigger( 'selected', this, event ); - this.selected = true; - } - return false; - }, - - /** event handler for clearing selection (also programmatic deselection) - */ - deselect : function( event ){ - // switch icon, set selected, and trigger event - this.$el.find( '.dataset-selector span' ) - .removeClass( 'fa-check-square-o' ).addClass( 'fa-square-o' ); - if( this.selected ){ - this.trigger( 'de-selected', this, event ); - this.selected = false; - } - return false; - }, - - toggleSelect : function( event ){ - if( this.selected ){ - this.deselect( event ); - } else { - this.select( event ); - } - }, - - // ......................................................................... drag/drop - draggableOn : function(){ - this.draggable = true; - //TODO: I have no idea why this doesn't work with the events hash or jq.on()... - //this.$el.find( '.dataset-title-bar' ) - // .attr( 'draggable', true ) - // .bind( 'dragstart', this.dragStartHandler, false ) - // .bind( 'dragend', this.dragEndHandler, false ); - this.dragStartHandler = _.bind( this._dragStartHandler, this ); - this.dragEndHandler = _.bind( this._dragEndHandler, this ); - - var titleBar = this.$el.find( '.dataset-title-bar' ).attr( 'draggable', true ).get(0); - titleBar.addEventListener( 'dragstart', this.dragStartHandler, false ); - titleBar.addEventListener( 'dragend', this.dragEndHandler, false ); - }, - draggableOff : function(){ - this.draggable = false; - var titleBar = this.$el.find( '.dataset-title-bar' ).attr( 'draggable', false ).get(0); - titleBar.removeEventListener( 'dragstart', this.dragStartHandler, false ); - titleBar.removeEventListener( 'dragend', this.dragEndHandler, false ); - }, - toggleDraggable : function(){ - if( this.draggable ){ - this.draggableOff(); - } else { - this.draggableOn(); - } - }, - _dragStartHandler : function( event ){ - //console.debug( 'dragStartHandler:', this, event, arguments ) - this.trigger( 'dragstart', this ); - event.dataTransfer.effectAllowed = 'move'; - //TODO: all except IE: should be 'application/json', IE: must be 'text' - event.dataTransfer.setData( 'text', JSON.stringify( this.model.toJSON() ) ); - return false; - }, - _dragEndHandler : function( event ){ - this.trigger( 'dragend', this ); - //console.debug( 'dragEndHandler:', event ) - return false; - }, - // ......................................................................... removal /** Remove this view's html from the DOM and remove all event listeners. * @param {Function} callback an optional function called when removal is done @@ -799,7 +576,7 @@ '</div>' ].join( '' )); -HDABaseView.templates = { +HDABaseView.templates = HDABaseView.prototype.templates = { // we override here in order to pass the localizer (_L) into the template scope - since we use it as a fn within skeleton : function( hdaJSON ){ return skeletonTemplate({ _l: _l, hda: hdaJSON }); This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a CustomDatatypeLoader for Galaxy installs, eliminate the use of the tool shed's datatyps_util module.
by commits-noreply＠bitbucket.org 23 Jul '14

23 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/74d7ff952ec9/ Changeset: 74d7ff952ec9 User: greg Date: 2014-07-23 17:26:49 Summary: Add a CustomDatatypeLoader for Galaxy installs, eliminate the use of the tool shed's datatyps_util module. Affected #: 7 files diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -12,7 +12,6 @@ import tool_shed.repository_types.util as rt_util from tool_shed.util import common_util -from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import readme_util @@ -25,6 +24,7 @@ from tool_shed.galaxy_install import dependency_display from tool_shed.galaxy_install import install_manager +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.grids import admin_toolshed_grids from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager @@ -256,17 +256,17 @@ dmh.remove_from_data_manager( tool_shed_repository ) if tool_shed_repository.includes_datatypes: # Deactivate proprietary datatypes. - installed_repository_dict = datatype_util.load_installed_datatypes( trans.app, - tool_shed_repository, - repository_install_dir, - deactivate=True ) + cdl = custom_datatype_manager.CustomDatatypeLoader( trans.app ) + installed_repository_dict = cdl.load_installed_datatypes( tool_shed_repository, + repository_install_dir, + deactivate=True ) if installed_repository_dict: converter_path = installed_repository_dict.get( 'converter_path' ) if converter_path is not None: - datatype_util.load_installed_datatype_converters( trans.app, installed_repository_dict, deactivate=True ) + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=True ) display_path = installed_repository_dict.get( 'display_path' ) if display_path is not None: - datatype_util.load_installed_display_applications( trans.app, installed_repository_dict, deactivate=True ) + cdl.load_installed_display_applications( installed_repository_dict, deactivate=True ) if remove_from_disk_checked: try: # Remove the repository from disk. diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py @@ -0,0 +1,218 @@ +import logging +import os +import tempfile + +from galaxy.util import asbool + +from tool_shed.util import basic_util +from tool_shed.util import hg_util +from tool_shed.util import tool_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import xml_util + +log = logging.getLogger( __name__ ) + + +class CustomDatatypeLoader( object ): + + def __init__( self, app ): + self.app = app + + def alter_config_and_load_prorietary_datatypes( self, datatypes_config, relative_install_dir, + deactivate=False, override=True ): + """ + Parse a custom datatypes config (a datatypes_conf.xml file included in an installed + tool shed repository) and add information to appropriate element attributes that will + enable custom datatype class modules, datatypes converters and display applications + to be discovered and properly imported by the datatypes registry. The value of override + will be False when a tool shed repository is being installed. Since installation is + occurring after the datatypes registry has been initialized, the registry's contents + cannot be overridden by conflicting data types. + """ + tree, error_message = xml_util.parse_xml( datatypes_config ) + if tree is None: + return None, None + datatypes_config_root = tree.getroot() + registration = datatypes_config_root.find( 'registration' ) + if registration is None: + # We have valid XML, but not a valid custom datatypes definition. + return None, None + sniffers = datatypes_config_root.find( 'sniffers' ) + converter_path, display_path = self.get_converter_and_display_paths( registration, + relative_install_dir ) + if converter_path: + # Path to datatype converters + registration.attrib[ 'proprietary_converter_path' ] = converter_path + if display_path: + # Path to datatype display applications + registration.attrib[ 'proprietary_display_path' ] = display_path + relative_path_to_datatype_file_name = None + datatype_files = datatypes_config_root.find( 'datatype_files' ) + datatype_class_modules = [] + if datatype_files is not None: + # The <datatype_files> tag set contains any number of <datatype_file> tags. + # <datatype_files> + # <datatype_file name="gmap.py"/> + # <datatype_file name="metagenomics.py"/> + # </datatype_files> + # We'll add attributes to the datatype tag sets so that the modules can be properly imported + # by the datatypes registry. + for elem in datatype_files.findall( 'datatype_file' ): + datatype_file_name = elem.get( 'name', None ) + if datatype_file_name: + # Find the file in the installed repository. + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == datatype_file_name: + datatype_class_modules.append( os.path.join( root, name ) ) + break + break + if datatype_class_modules: + for relative_path_to_datatype_file_name in datatype_class_modules: + datatype_file_name_path, datatype_file_name = os.path.split( relative_path_to_datatype_file_name ) + for elem in registration.findall( 'datatype' ): + # Handle 'type' attribute which should be something like one of the following: + # type="gmap:GmapDB" + # type="galaxy.datatypes.gmap:GmapDB" + dtype = elem.get( 'type', None ) + if dtype: + fields = dtype.split( ':' ) + proprietary_datatype_module = fields[ 0 ] + if proprietary_datatype_module.find( '.' ) >= 0: + # Handle the case where datatype_module is "galaxy.datatypes.gmap". + proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ] + # The value of proprietary_path must be an absolute path due to job_working_directory. + elem.attrib[ 'proprietary_path' ] = os.path.abspath( datatype_file_name_path ) + elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module + # Temporarily persist the custom datatypes configuration file so it can be loaded into the + # datatypes registry. + fd, proprietary_datatypes_config = tempfile.mkstemp( prefix="tmp-toolshed-acalpd" ) + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<datatypes>\n' ) + os.write( fd, '%s' % xml_util.xml_to_string( registration ) ) + if sniffers is not None: + os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) ) + os.write( fd, '</datatypes>\n' ) + os.close( fd ) + os.chmod( proprietary_datatypes_config, 0644 ) + # Load custom datatypes + self.app.datatypes_registry.load_datatypes( root_dir=self.app.config.root, + config=proprietary_datatypes_config, + deactivate=deactivate, + override=override ) + if deactivate: + # Reload the upload tool to eliminate deactivated datatype extensions from the file_type + # select list. + tool_util.reload_upload_tools( self.app ) + else: + self.append_to_datatypes_registry_upload_file_formats( registration ) + tool_util.reload_upload_tools( self.app ) + if datatype_files is not None: + try: + os.unlink( proprietary_datatypes_config ) + except: + pass + return converter_path, display_path + + def append_to_datatypes_registry_upload_file_formats( self, elem ): + # See if we have any datatypes that should be displayed in the upload tool's file_type select list. + for datatype_elem in elem.findall( 'datatype' ): + extension = datatype_elem.get( 'extension', None ) + display_in_upload = datatype_elem.get( 'display_in_upload', None ) + if extension is not None and display_in_upload is not None: + display_in_upload = asbool( str( display_in_upload ) ) + if display_in_upload and extension not in self.app.datatypes_registry.upload_file_formats: + self.app.datatypes_registry.upload_file_formats.append( extension ) + + def create_repository_dict_for_proprietary_datatypes( self, tool_shed, name, owner, installed_changeset_revision, + tool_dicts, converter_path=None, display_path=None ): + return dict( tool_shed=tool_shed, + repository_name=name, + repository_owner=owner, + installed_changeset_revision=installed_changeset_revision, + tool_dicts=tool_dicts, + converter_path=converter_path, + display_path=display_path ) + + def get_converter_and_display_paths( self, registration_elem, relative_install_dir ): + """ + Find the relative path to data type converters and display applications included + in installed tool shed repositories. + """ + converter_path = None + display_path = None + for elem in registration_elem.findall( 'datatype' ): + if not converter_path: + # If any of the <datatype> tag sets contain <converter> tags, set the converter_path + # if it is not already set. This requires developers to place all converters in the + # same subdirectory within the repository hierarchy. + for converter in elem.findall( 'converter' ): + converter_config = converter.get( 'file', None ) + if converter_config: + converter_config_file_name = basic_util.strip_path( converter_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == converter_config_file_name: + # The value of converter_path must be absolute due to job_working_directory. + converter_path = os.path.abspath( root ) + break + if converter_path: + break + if not display_path: + # If any of the <datatype> tag sets contain <display> tags, set the display_path + # if it is not already set. This requires developers to place all display acpplications + # in the same subdirectory within the repository hierarchy. + for display_app in elem.findall( 'display' ): + display_config = display_app.get( 'file', None ) + if display_config: + display_config_file_name = basic_util.strip_path( display_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == display_config_file_name: + # The value of display_path must be absolute due to job_working_directory. + display_path = os.path.abspath( root ) + break + if display_path: + break + if converter_path and display_path: + break + return converter_path, display_path + + def load_installed_datatype_converters( self, installed_repository_dict, deactivate=False ): + """Load or deactivate proprietary datatype converters.""" + self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, + installed_repository_dict=installed_repository_dict, + deactivate=deactivate ) + + def load_installed_datatypes( self, repository, relative_install_dir, deactivate=False ): + """ + Load proprietary datatypes and return information needed for loading custom + datatypes converters and display applications later. + """ + metadata = repository.metadata + repository_dict = None + datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir ) + if datatypes_config: + converter_path, display_path = \ + self.alter_config_and_load_prorietary_datatypes( datatypes_config, + relative_install_dir, + deactivate=deactivate ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = \ + self.create_repository_dict_for_proprietary_datatypes( tool_shed=repository.tool_shed, + name=repository.name, + owner=repository.owner, + installed_changeset_revision=repository.installed_changeset_revision, + tool_dicts=metadata.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + return repository_dict + + def load_installed_display_applications( self, installed_repository_dict, deactivate=False ): + """Load or deactivate custom datatype display applications.""" + self.app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, + deactivate=deactivate ) diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -18,7 +18,6 @@ from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc @@ -26,6 +25,7 @@ from tool_shed.util import tool_util from tool_shed.util import xml_util +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder @@ -594,18 +594,19 @@ files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) # Load data types required by tools. + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) converter_path, display_path = \ - datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, files_dir, override=False ) + cdl.alter_config_and_load_prorietary_datatypes( datatypes_config, files_dir, override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = \ - datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) + cdl.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, + name=tool_shed_repository.name, + owner=tool_shed_repository.owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) if converter_path: # Load proprietary datatype converters self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, installed_repository_dict=repository_dict ) diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -7,12 +7,12 @@ from galaxy import util from tool_shed.util import common_util from tool_shed.util import container_util -from tool_shed.util import datatype_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import xml_util from galaxy.model.orm import and_ +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tools import data_manager @@ -114,17 +114,17 @@ else: repository_install_dir = os.path.abspath( relative_install_dir ) # Activate proprietary datatypes. - installed_repository_dict = datatype_util.load_installed_datatypes( self.app, - repository, - repository_install_dir, - deactivate=False ) + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) + installed_repository_dict = cdl.load_installed_datatypes( repository, + repository_install_dir, + deactivate=False ) if installed_repository_dict: converter_path = installed_repository_dict.get( 'converter_path' ) if converter_path is not None: - datatype_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=False ) + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=False ) display_path = installed_repository_dict.get( 'display_path' ) if display_path is not None: - datatype_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=False ) + cdl.load_installed_display_applications( installed_repository_dict, deactivate=False ) def add_entry_to_installed_repository_dependencies_of_installed_repositories( self, repository ): """ @@ -732,22 +732,24 @@ self.add_entry_to_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies( tool_dependency ) def load_proprietary_datatypes( self ): + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) for tool_shed_repository in self.context.query( self.install_model.ToolShedRepository ) \ - .filter( and_( self.install_model.ToolShedRepository.table.c.includes_datatypes==True, - self.install_model.ToolShedRepository.table.c.deleted==False ) ) \ - .order_by( self.install_model.ToolShedRepository.table.c.id ): + .filter( and_( self.install_model.ToolShedRepository.table.c.includes_datatypes==True, + self.install_model.ToolShedRepository.table.c.deleted==False ) ) \ + .order_by( self.install_model.ToolShedRepository.table.c.id ): relative_install_dir = self.get_repository_install_dir( tool_shed_repository ) if relative_install_dir: - installed_repository_dict = datatype_util.load_installed_datatypes( self.app, tool_shed_repository, relative_install_dir ) + installed_repository_dict = cdl.load_installed_datatypes( tool_shed_repository, relative_install_dir ) if installed_repository_dict: self.installed_repository_dicts.append( installed_repository_dict ) def load_proprietary_converters_and_display_applications( self, deactivate=False ): + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) for installed_repository_dict in self.installed_repository_dicts: if installed_repository_dict[ 'converter_path' ]: - datatype_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=deactivate ) + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=deactivate ) if installed_repository_dict[ 'display_path' ]: - datatype_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=deactivate ) + cdl.load_installed_display_applications( installed_repository_dict, deactivate=deactivate ) def purge_repository( self, repository ): """Purge a repository with status New (a white ghost) from the database.""" diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -14,6 +14,7 @@ from galaxy.util.odict import odict from tool_shed.galaxy_install import install_manager +from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.tools import tool_panel_manager @@ -22,7 +23,6 @@ from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import datatype_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util @@ -492,6 +492,7 @@ print '\nThe ToolMigrationManager returned the following error while installing tool dependency ', installed_tool_dependency.name, ':' print installed_tool_dependency.error_message, '\n\n' if 'datatypes' in metadata_dict: + cdl = custom_datatype_manager.CustomDatatypeLoader( self.app ) tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True @@ -499,21 +500,27 @@ self.app.install_model.context.flush() work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-hrc" ) datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir ) - # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started - # after this installation completes. - converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir + # Load proprietary data types required by tools. The value of override is not + # important here since the Galaxy server will be started after this installation + #completes. + converter_path, display_path = \ + cdl.alter_config_and_load_prorietary_datatypes( datatypes_config, + repo_install_dir, + override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. - repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed_url, - name=tool_shed_repository.name, - owner=self.repository_owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) + repository_dict = \ + cdl.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed_url, + name=tool_shed_repository.name, + owner=self.repository_owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) if converter_path: # Load proprietary datatype converters - self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict ) + self.app.datatypes_registry.load_datatype_converters( self.toolbox, + installed_repository_dict=repository_dict ) if display_path: # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) diff -r 053943b668af334c480b464fe7351b50284b7099 -r 74d7ff952ec902e68cc99745fdcd3c14e86208b2 lib/tool_shed/util/datatype_util.py --- a/lib/tool_shed/util/datatype_util.py +++ /dev/null @@ -1,187 +0,0 @@ -import logging -import os -import tempfile -from galaxy import eggs -from galaxy.util import asbool -from tool_shed.util import basic_util -from tool_shed.util import hg_util -from tool_shed.util import tool_util -from tool_shed.util import xml_util -import tool_shed.util.shed_util_common as suc - -log = logging.getLogger( __name__ ) - -def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False, override=True ): - """ - Parse a proprietary datatypes config (a datatypes_conf.xml file included in an installed tool shed repository) and - add information to appropriate element attributes that will enable proprietary datatype class modules, datatypes converters - and display applications to be discovered and properly imported by the datatypes registry. The value of override will - be False when a tool shed repository is being installed. Since installation is occurring after the datatypes registry - has been initialized, the registry's contents cannot be overridden by conflicting data types. - """ - tree, error_message = xml_util.parse_xml( datatypes_config ) - if tree is None: - return None, None - datatypes_config_root = tree.getroot() - registration = datatypes_config_root.find( 'registration' ) - if registration is None: - # We have valid XML, but not a valid proprietary datatypes definition. - return None, None - sniffers = datatypes_config_root.find( 'sniffers' ) - converter_path, display_path = get_converter_and_display_paths( registration, relative_install_dir ) - if converter_path: - # Path to datatype converters - registration.attrib[ 'proprietary_converter_path' ] = converter_path - if display_path: - # Path to datatype display applications - registration.attrib[ 'proprietary_display_path' ] = display_path - relative_path_to_datatype_file_name = None - datatype_files = datatypes_config_root.find( 'datatype_files' ) - datatype_class_modules = [] - if datatype_files is not None: - # The <datatype_files> tag set contains any number of <datatype_file> tags. - # <datatype_files> - # <datatype_file name="gmap.py"/> - # <datatype_file name="metagenomics.py"/> - # </datatype_files> - # We'll add attributes to the datatype tag sets so that the modules can be properly imported by the datatypes registry. - for elem in datatype_files.findall( 'datatype_file' ): - datatype_file_name = elem.get( 'name', None ) - if datatype_file_name: - # Find the file in the installed repository. - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == datatype_file_name: - datatype_class_modules.append( os.path.join( root, name ) ) - break - break - if datatype_class_modules: - for relative_path_to_datatype_file_name in datatype_class_modules: - datatype_file_name_path, datatype_file_name = os.path.split( relative_path_to_datatype_file_name ) - for elem in registration.findall( 'datatype' ): - # Handle 'type' attribute which should be something like one of the following: - # type="gmap:GmapDB" - # type="galaxy.datatypes.gmap:GmapDB" - dtype = elem.get( 'type', None ) - if dtype: - fields = dtype.split( ':' ) - proprietary_datatype_module = fields[ 0 ] - if proprietary_datatype_module.find( '.' ) >= 0: - # Handle the case where datatype_module is "galaxy.datatypes.gmap". - proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ] - # The value of proprietary_path must be an absolute path due to job_working_directory. - elem.attrib[ 'proprietary_path' ] = os.path.abspath( datatype_file_name_path ) - elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module - # Temporarily persist the proprietary datatypes configuration file so it can be loaded into the datatypes registry. - fd, proprietary_datatypes_config = tempfile.mkstemp( prefix="tmp-toolshed-acalpd" ) - os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, '<datatypes>\n' ) - os.write( fd, '%s' % xml_util.xml_to_string( registration ) ) - if sniffers is not None: - os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) ) - os.write( fd, '</datatypes>\n' ) - os.close( fd ) - os.chmod( proprietary_datatypes_config, 0644 ) - # Load proprietary datatypes - app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=proprietary_datatypes_config, deactivate=deactivate, override=override ) - if deactivate: - # Reload the upload tool to eliminate deactivated datatype extensions from the file_type select list. - tool_util.reload_upload_tools( app ) - else: - append_to_datatypes_registry_upload_file_formats( app, registration ) - tool_util.reload_upload_tools( app ) - if datatype_files is not None: - try: - os.unlink( proprietary_datatypes_config ) - except: - pass - return converter_path, display_path - -def append_to_datatypes_registry_upload_file_formats( app, elem ): - # See if we have any datatypes that should be displayed in the upload tool's file_type select list. - for datatype_elem in elem.findall( 'datatype' ): - extension = datatype_elem.get( 'extension', None ) - display_in_upload = datatype_elem.get( 'display_in_upload', None ) - if extension is not None and display_in_upload is not None: - display_in_upload = asbool( str( display_in_upload ) ) - if display_in_upload and extension not in app.datatypes_registry.upload_file_formats: - app.datatypes_registry.upload_file_formats.append( extension ) - -def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): - return dict( tool_shed=tool_shed, - repository_name=name, - repository_owner=owner, - installed_changeset_revision=installed_changeset_revision, - tool_dicts=tool_dicts, - converter_path=converter_path, - display_path=display_path ) - -def get_converter_and_display_paths( registration_elem, relative_install_dir ): - """Find the relative path to data type converters and display applications included in installed tool shed repositories.""" - converter_path = None - display_path = None - for elem in registration_elem.findall( 'datatype' ): - if not converter_path: - # If any of the <datatype> tag sets contain <converter> tags, set the converter_path - # if it is not already set. This requires developers to place all converters in the - # same subdirectory within the repository hierarchy. - for converter in elem.findall( 'converter' ): - converter_config = converter.get( 'file', None ) - if converter_config: - converter_config_file_name = basic_util.strip_path( converter_config ) - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == converter_config_file_name: - # The value of converter_path must be absolute due to job_working_directory. - converter_path = os.path.abspath( root ) - break - if converter_path: - break - if not display_path: - # If any of the <datatype> tag sets contain <display> tags, set the display_path - # if it is not already set. This requires developers to place all display acpplications - # in the same subdirectory within the repository hierarchy. - for display_app in elem.findall( 'display' ): - display_config = display_app.get( 'file', None ) - if display_config: - display_config_file_name = basic_util.strip_path( display_config ) - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == display_config_file_name: - # The value of display_path must be absolute due to job_working_directory. - display_path = os.path.abspath( root ) - break - if display_path: - break - if converter_path and display_path: - break - return converter_path, display_path - -def load_installed_datatype_converters( app, installed_repository_dict, deactivate=False ): - # Load or deactivate proprietary datatype converters - app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=installed_repository_dict, deactivate=deactivate ) - -def load_installed_datatypes( app, repository, relative_install_dir, deactivate=False ): - # Load proprietary datatypes and return information needed for loading proprietary datatypes converters and display applications later. - metadata = repository.metadata - repository_dict = None - datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir ) - if datatypes_config: - converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate ) - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=repository.tool_shed, - name=repository.name, - owner=repository.owner, - installed_changeset_revision=repository.installed_changeset_revision, - tool_dicts=metadata.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) - return repository_dict - -def load_installed_display_applications( app, installed_repository_dict, deactivate=False ): - # Load or deactivate proprietary datatype display applications - app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, deactivate=deactivate ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: More standardized (if still terse) exception handling in AuthenticationController.
by commits-noreply＠bitbucket.org 23 Jul '14

23 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/053943b668af/ Changeset: 053943b668af User: jmchilton Date: 2014-07-23 16:16:08 Summary: More standardized (if still terse) exception handling in AuthenticationController. Affected #: 3 files diff -r f24d7ac8ee9549fc19435aa1448854e9024982ff -r 053943b668af334c480b464fe7351b50284b7099 lib/galaxy/exceptions/__init__.py --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -86,6 +86,11 @@ err_code = error_codes.USER_REQUEST_INVALID_PARAMETER +class AuthenticationFailed( MessageException ): + status_code = 401 + err_code = error_codes.USER_AUTHENTICATION_FAILED + + class AuthenticationRequired( MessageException ): status_code = 403 #TODO: as 401 and send WWW-Authenticate: ??? diff -r f24d7ac8ee9549fc19435aa1448854e9024982ff -r 053943b668af334c480b464fe7351b50284b7099 lib/galaxy/exceptions/error_codes.json --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -60,6 +60,11 @@ "message": "Supplied incorrect or incompatible tool meta parameters." }, { + "name": "USER_AUTHENTICATION_FAILED", + "code": 401001, + "message": "Authentication failed, invalid credentials supplied." + }, + { "name": "USER_NO_API_KEY", "code": 403001, "message": "API authentication required for this request" diff -r f24d7ac8ee9549fc19435aa1448854e9024982ff -r 053943b668af334c480b464fe7351b50284b7099 lib/galaxy/webapps/galaxy/api/authenticate.py --- a/lib/galaxy/webapps/galaxy/api/authenticate.py +++ b/lib/galaxy/webapps/galaxy/api/authenticate.py @@ -12,11 +12,10 @@ """ from base64 import b64decode -from paste.httpexceptions import HTTPBadRequest from urllib import unquote -from galaxy import web -from galaxy.exceptions import ObjectNotFound +from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous +from galaxy import exceptions from galaxy.web.base.controller import BaseAPIController, CreatesApiKeysMixin import logging @@ -25,7 +24,7 @@ class AuthenticationController( BaseAPIController, CreatesApiKeysMixin ): - @web.expose_api_anonymous + @expose_api_anonymous def get_api_key( self, trans, **kwd ): """ def get_api_key( self, trans, **kwd ) @@ -43,7 +42,7 @@ if ( len( user ) is not 1 ): # DB is inconsistent and we have more users with same email - raise ObjectNotFound + raise exceptions.ObjectNotFound() else: user = user[0] is_valid_user = user.check_password( password ) @@ -54,8 +53,7 @@ key = self.create_api_key( trans, user ) return dict( api_key=key ) else: - trans.response.status = 500 - return "invalid password" + raise exceptions.AuthenticationFailed() def _decode_baseauth( self, encoded_str ): """ @@ -81,7 +79,7 @@ try: email, password = b64decode( split[ 0 ] ).split( ':' ) except: - raise HTTPBadRequest + raise exceptions.ActionInputError() # If there are only two elements, check the first and ensure it says # 'basic' so that we know we're about to decode the right thing. If not, @@ -91,13 +89,13 @@ try: email, password = b64decode( split[ 1 ] ).split( ':' ) except: - raise HTTPBadRequest + raise exceptions.ActionInputError() else: - raise HTTPBadRequest + raise exceptions.ActionInputError() # If there are more than 2 elements, something crazy must be happening. # Bail. else: - raise HTTPBadRequest + raise exceptions.ActionInputError() return unquote( email ), unquote( password ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fixes in exception handlers in the Tool Shed's create_categories and create_users API modules.
by commits-noreply＠bitbucket.org 23 Jul '14

23 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f24d7ac8ee95/ Changeset: f24d7ac8ee95 User: greg Date: 2014-07-23 15:47:21 Summary: Fixes in exception handlers in the Tool Shed's create_categories and create_users API modules. Affected #: 2 files diff -r 0c62ccbc86246c9aebf8ff78c550234dfa6d5bd5 -r f24d7ac8ee9549fc19435aa1448854e9024982ff lib/tool_shed/scripts/api/create_categories.py --- a/lib/tool_shed/scripts/api/create_categories.py +++ b/lib/tool_shed/scripts/api/create_categories.py @@ -44,7 +44,7 @@ response = submit( url, data, api_key ) except Exception, e: response = str( e ) - log.exception( str( e ) ) + print "Error attempting to create category using URL: ", url, " exception: ", str( e ) create_response_dict = dict( response=response ) create_response_dicts.append( create_response_dict ) diff -r 0c62ccbc86246c9aebf8ff78c550234dfa6d5bd5 -r f24d7ac8ee9549fc19435aa1448854e9024982ff lib/tool_shed/scripts/api/create_users.py --- a/lib/tool_shed/scripts/api/create_users.py +++ b/lib/tool_shed/scripts/api/create_users.py @@ -47,7 +47,7 @@ response = submit( url, data, api_key ) except Exception, e: response = str( e ) - log.exception( str( e ) ) + print "Error attempting to create user using URL: ", url, " exception: ", str( e ) create_response_dict = dict( response=response ) create_response_dicts.append( create_response_dict ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Move some functions out of the Tool Shed's shed_util_common module.
by commits-noreply＠bitbucket.org 22 Jul '14

22 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0c62ccbc8624/ Changeset: 0c62ccbc8624 User: greg Date: 2014-07-22 22:46:30 Summary: Move some functions out of the Tool Shed's shed_util_common module. Affected #: 7 files diff -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a -r 0c62ccbc86246c9aebf8ff78c550234dfa6d5bd5 lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -9,6 +9,7 @@ from time import strftime from galaxy import web +from galaxy.model.orm import and_ from galaxy.util import asbool from galaxy.util import CHUNK_SIZE from galaxy.util.odict import odict @@ -408,7 +409,7 @@ flush = True # Do not allow dependent repository revisions to be automatically installed if population # resulted in errors. - dependent_downloadable_revisions = suc.get_dependent_downloadable_revisions( self.app, repository_metadata ) + dependent_downloadable_revisions = self.get_dependent_downloadable_revisions( repository_metadata ) for dependent_downloadable_revision in dependent_downloadable_revisions: if dependent_downloadable_revision.downloadable: dependent_downloadable_revision.downloadable = False @@ -541,6 +542,66 @@ archives.append( archive_file_name ) return archives, error_message + def get_dependent_downloadable_revisions( self, repository_metadata ): + """ + Return all repository_metadata records that are downloadable and that depend upon the received + repository_metadata record. + """ + # This method is called only from the tool shed. + sa_session = self.app.model.context.current + rm_changeset_revision = repository_metadata.changeset_revision + rm_repository = repository_metadata.repository + rm_repository_name = str( rm_repository.name ) + rm_repository_owner = str( rm_repository.user.username ) + dependent_downloadable_revisions = [] + for repository in sa_session.query( self.app.model.Repository ) \ + .filter( and_( self.app.model.Repository.table.c.id != rm_repository.id, + self.app.model.Repository.table.c.deleted == False, + self.app.model.Repository.table.c.deprecated == False ) ): + downloadable_revisions = repository.downloadable_revisions + if downloadable_revisions: + for downloadable_revision in downloadable_revisions: + if downloadable_revision.has_repository_dependencies: + metadata = downloadable_revision.metadata + if metadata: + repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) + repository_dependencies_tups = repository_dependencies_dict.get( 'repository_dependencies', [] ) + for repository_dependencies_tup in repository_dependencies_tups: + tool_shed, \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependencies_tup ) + if name == rm_repository_name and owner == rm_repository_owner: + # We've discovered a repository revision that depends upon the repository associated + # with the received repository_metadata record, but we need to make sure it depends + # upon the revision. + if changeset_revision == rm_changeset_revision: + dependent_downloadable_revisions.append( downloadable_revision ) + else: + # Make sure the defined changeset_revision is current. + defined_repository_metadata = \ + sa_session.query( self.app.model.RepositoryMetadata ) \ + .filter( self.app.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) \ + .first() + if defined_repository_metadata is None: + # The defined changeset_revision is not associated with a repository_metadata + # record, so updates must be necessary. + defined_repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + defined_repo = hg_util.get_repo_for_repository( self.app, + repository=defined_repository, + repo_path=None, + create=False ) + updated_changeset_revision = \ + suc.get_next_downloadable_changeset_revision( defined_repository, + defined_repo, + changeset_revision ) + if updated_changeset_revision == rm_changeset_revision: + dependent_downloadable_revisions.append( downloadable_revision ) + return dependent_downloadable_revisions + def get_export_info_dict( self, export_info_file_path ): """ Parse the export_info.xml file contained within the capsule and return a dictionary diff -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a -r 0c62ccbc86246c9aebf8ff78c550234dfa6d5bd5 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -342,8 +342,8 @@ installed_rd_tups = [] missing_rd_tups = [] for tsr in repository.repository_dependencies: - prior_installation_required = suc.set_prior_installation_required( self.app, repository, tsr ) - only_if_compiling_contained_td = suc.set_only_if_compiling_contained_td( repository, tsr ) + prior_installation_required = self.set_prior_installation_required( repository, tsr ) + only_if_compiling_contained_td = self.set_only_if_compiling_contained_td( repository, tsr ) rd_tup = [ tsr.tool_shed, tsr.name, tsr.owner, @@ -957,6 +957,47 @@ return True return False + def set_only_if_compiling_contained_td( self, repository, required_repository ): + """ + Return True if the received required_repository is only needed to compile a tool + dependency defined for the received repository. + """ + # This method is called only from Galaxy when rendering repository dependencies + # for an installed tool shed repository. + # TODO: Do we need to check more than changeset_revision here? + required_repository_tup = [ required_repository.tool_shed, \ + required_repository.name, \ + required_repository.owner, \ + required_repository.changeset_revision ] + for tup in repository.tuples_of_repository_dependencies_needed_for_compiling_td: + partial_tup = tup[ 0:4 ] + if partial_tup == required_repository_tup: + return 'True' + return 'False' + + def set_prior_installation_required( self, repository, required_repository ): + """ + Return True if the received required_repository must be installed before the + received repository. + """ + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, + str( required_repository.tool_shed ) ) + required_repository_tup = [ tool_shed_url, + str( required_repository.name ), + str( required_repository.owner ), + str( required_repository.changeset_revision ) ] + # Get the list of repository dependency tuples associated with the received repository + # where prior_installation_required is True. + required_rd_tups_that_must_be_installed = repository.requires_prior_installation_of + for required_rd_tup in required_rd_tups_that_must_be_installed: + # Repository dependency tuples in metadata include a prior_installation_required value, + # so strip it for comparision. + partial_required_rd_tup = required_rd_tup[ 0:4 ] + if partial_required_rd_tup == required_repository_tup: + # Return the string value of prior_installation_required, which defaults to 'False'. + return str( required_rd_tup[ 4 ] ) + return 'False' + def update_existing_tool_dependency( self, repository, original_dependency_dict, new_dependencies_dict ): """ Update an exsiting tool dependency whose definition was updated in a change set diff -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a -r 0c62ccbc86246c9aebf8ff78c550234dfa6d5bd5 lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -2,9 +2,9 @@ Class encapsulating the management of repository dependencies installed or being installed into Galaxy from the Tool Shed. """ - import json import logging +import os import urllib import urllib2 @@ -63,11 +63,10 @@ break if d_repository is None: # The dependent repository is not in the received list so look in the database. - d_repository = suc.get_or_create_tool_shed_repository( self.app, - d_toolshed, - d_name, - d_owner, - d_changeset_revision ) + d_repository = self.get_or_create_tool_shed_repository( d_toolshed, + d_name, + d_owner, + d_changeset_revision ) # Process each repository_dependency defined for the current dependent repository. for repository_dependency_components_list in val: required_repository = None @@ -87,11 +86,10 @@ break if required_repository is None: # The required repository is not in the received list so look in the database. - required_repository = suc.get_or_create_tool_shed_repository( self.app, - rd_toolshed, - rd_name, - rd_owner, - rd_changeset_revision ) + required_repository = self.get_or_create_tool_shed_repository( rd_toolshed, + rd_name, + rd_owner, + rd_changeset_revision ) # Ensure there is a repository_dependency relationship between d_repository and required_repository. rrda = None for rd in d_repository.repository_dependencies: @@ -257,6 +255,35 @@ self.build_repository_dependency_relationships( all_repo_info_dicts, all_created_or_updated_tool_shed_repositories ) return created_or_updated_tool_shed_repositories, tool_panel_section_keys, all_repo_info_dicts, filtered_repo_info_dicts + def get_or_create_tool_shed_repository( self, tool_shed, name, owner, changeset_revision ): + """ + Return a tool shed repository database record defined by the combination of + tool shed, repository name, repository owner and changeset_revision or + installed_changeset_revision. A new tool shed repository record will be + created if one is not located. + """ + install_model = self.app.install_model + # We store the port in the database. + tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) + # This method is used only in Galaxy, not the tool shed. + repository = suc.get_repository_for_dependency_relationship( self.app, tool_shed, name, owner, changeset_revision ) + if not repository: + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed ) + repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) + ctx_rev = suc.get_ctx_rev( self.app, tool_shed_url, name, owner, changeset_revision ) + repository = suc.create_or_update_tool_shed_repository( app=self.app, + name=name, + description=None, + installed_changeset_revision=changeset_revision, + ctx_rev=ctx_rev, + repository_clone_url=repository_clone_url, + metadata_dict={}, + status=install_model.ToolShedRepository.installation_status.NEW, + current_changeset_revision=None, + owner=owner, + dist_to_shed=False ) + return repository + def get_repository_dependencies_for_installed_tool_shed_repository( self, app, repository ): """ Send a request to the appropriate tool shed to retrieve the dictionary of repository dependencies defined diff -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a -r 0c62ccbc86246c9aebf8ff78c550234dfa6d5bd5 lib/tool_shed/metadata/repository_metadata_manager.py --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -410,7 +410,7 @@ repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) for changeset in repo.changelog: changeset_hash = str( repo.changectx( changeset ) ) - skip_tool_test = suc.get_skip_tool_test_by_changeset_revision( self.app, changeset_hash ) + skip_tool_test = self.get_skip_tool_test_by_changeset_revision( changeset_hash ) if skip_tool_test: # We found a skip_tool_test record associated with the changeset_revision, # so see if it has a valid repository_revision. @@ -521,6 +521,16 @@ return self.sa_session.query( self.app.model.Repository ) \ .filter( self.app.model.Repository.table.c.deleted == False ) + def get_skip_tool_test_by_changeset_revision( self, changeset_revision ): + """ + Return a skip_tool_test record whose initial_changeset_revision is the received + changeset_revision. + """ + # There should only be one, but we'll use first() so callers won't have to handle exceptions. + return self.sa_session.query( self.app.model.SkipToolTest ) \ + .filter( self.app.model.SkipToolTest.table.c.initial_changeset_revision == changeset_revision ) \ + .first() + def new_datatypes_metadata_required( self, repository_metadata, metadata_dict ): """ Compare the last saved metadata for each datatype in the repository with the new metadata diff -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a -r 0c62ccbc86246c9aebf8ff78c550234dfa6d5bd5 lib/tool_shed/tools/data_table_manager.py --- a/lib/tool_shed/tools/data_table_manager.py +++ b/lib/tool_shed/tools/data_table_manager.py @@ -2,8 +2,9 @@ import os import shutil +from xml.etree import ElementTree as XmlET + from tool_shed.util import hg_util -from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util log = logging.getLogger( __name__ ) @@ -14,6 +15,36 @@ def __init__( self, app ): self.app = app + def generate_repository_info_elem( self, tool_shed, repository_name, changeset_revision, owner, + parent_elem=None, **kwd ): + """Create and return an ElementTree repository info Element.""" + if parent_elem is None: + elem = XmlET.Element( 'tool_shed_repository' ) + else: + elem = XmlET.SubElement( parent_elem, 'tool_shed_repository' ) + tool_shed_elem = XmlET.SubElement( elem, 'tool_shed' ) + tool_shed_elem.text = tool_shed + repository_name_elem = XmlET.SubElement( elem, 'repository_name' ) + repository_name_elem.text = repository_name + repository_owner_elem = XmlET.SubElement( elem, 'repository_owner' ) + repository_owner_elem.text = owner + changeset_revision_elem = XmlET.SubElement( elem, 'installed_changeset_revision' ) + changeset_revision_elem.text = changeset_revision + #add additional values + #TODO: enhance additional values to allow e.g. use of dict values that will recurse + for key, value in kwd.iteritems(): + new_elem = XmlET.SubElement( elem, key ) + new_elem.text = value + return elem + + def generate_repository_info_elem_from_repository( self, tool_shed_repository, parent_elem=None, **kwd ): + return self.generate_repository_info_elem( tool_shed_repository.tool_shed, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision, + tool_shed_repository.owner, + parent_elem=parent_elem, + **kwd ) + def get_tool_index_sample_files( self, sample_files ): """ Try to return the list of all appropriate tool data sample files included @@ -129,7 +160,7 @@ if path: file_elem.set( 'path', os.path.normpath( os.path.join( target_dir, os.path.split( path )[1] ) ) ) # Store repository info in the table tag set for trace-ability. - repo_elem = suc.generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=elem ) + repo_elem = self.generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=elem ) if elems: # Remove old data_table os.unlink( tool_data_table_conf_filename ) diff -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a -r 0c62ccbc86246c9aebf8ff78c550234dfa6d5bd5 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -7,6 +7,18 @@ log = logging.getLogger( __name__ ) +def get_latest_changeset_revision( app, repository, repo ): + repository_tip = repository.tip( app ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, + app.security.encode_id( repository.id ), + repository_tip ) + if repository_metadata and repository_metadata.downloadable: + return repository_tip + changeset_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=False ) + if changeset_revisions: + return changeset_revisions[ -1 ] + return hg_util.INITIAL_CHANGELOG_HASH + def get_latest_repository_metadata( app, decoded_repository_id, downloadable=False ): """Get last metadata defined for a specified repository from the database.""" sa_session = app.model.context.current @@ -15,7 +27,7 @@ if downloadable: changeset_revision = suc.get_latest_downloadable_changeset_revision( app, repository, repo ) else: - changeset_revision = suc.get_latest_changeset_revision( app, repository, repo ) + changeset_revision = get_latest_changeset_revision( app, repository, repo ) return suc.get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), changeset_revision ) diff -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a -r 0c62ccbc86246c9aebf8ff78c550234dfa6d5bd5 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -17,7 +17,6 @@ from tool_shed.util import encoding_util from tool_shed.util import hg_util -from xml.etree import ElementTree as XmlET from urllib2 import HTTPError log = logging.getLogger( __name__ ) @@ -165,35 +164,6 @@ components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ] return components_list -def generate_repository_info_elem( tool_shed, repository_name, changeset_revision, owner, parent_elem=None, **kwd ): - """Create and return an ElementTree repository info Element.""" - if parent_elem is None: - elem = XmlET.Element( 'tool_shed_repository' ) - else: - elem = XmlET.SubElement( parent_elem, 'tool_shed_repository' ) - tool_shed_elem = XmlET.SubElement( elem, 'tool_shed' ) - tool_shed_elem.text = tool_shed - repository_name_elem = XmlET.SubElement( elem, 'repository_name' ) - repository_name_elem.text = repository_name - repository_owner_elem = XmlET.SubElement( elem, 'repository_owner' ) - repository_owner_elem.text = owner - changeset_revision_elem = XmlET.SubElement( elem, 'installed_changeset_revision' ) - changeset_revision_elem.text = changeset_revision - #add additional values - #TODO: enhance additional values to allow e.g. use of dict values that will recurse - for key, value in kwd.iteritems(): - new_elem = XmlET.SubElement( elem, key ) - new_elem.text = value - return elem - -def generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=None, **kwd ): - return generate_repository_info_elem( tool_shed_repository.tool_shed, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision, - tool_shed_repository.owner, - parent_elem=parent_elem, - **kwd ) - def generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) @@ -291,61 +261,6 @@ return repository_metadata return None -def get_dependent_downloadable_revisions( app, repository_metadata ): - """ - Return all repository_metadata records that are downloadable and that depend upon the received - repository_metadata record. - """ - # This method is called only from the tool shed. - sa_session = app.model.context.current - rm_changeset_revision = repository_metadata.changeset_revision - rm_repository = repository_metadata.repository - rm_repository_name = str( rm_repository.name ) - rm_repository_owner = str( rm_repository.user.username ) - dependent_downloadable_revisions = [] - for repository in sa_session.query( app.model.Repository ) \ - .filter( and_( app.model.Repository.table.c.id != rm_repository.id, - app.model.Repository.table.c.deleted == False, - app.model.Repository.table.c.deprecated == False ) ): - downloadable_revisions = repository.downloadable_revisions - if downloadable_revisions: - for downloadable_revision in downloadable_revisions: - if downloadable_revision.has_repository_dependencies: - metadata = downloadable_revision.metadata - if metadata: - repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) - repository_dependencies_tups = repository_dependencies_dict.get( 'repository_dependencies', [] ) - for repository_dependencies_tup in repository_dependencies_tups: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependencies_tup ) - if name == rm_repository_name and owner == rm_repository_owner: - # We've discovered a repository revision that depends upon the repository associated - # with the received repository_metadata record, but we need to make sure it depends - # upon the revision. - if changeset_revision == rm_changeset_revision: - dependent_downloadable_revisions.append( downloadable_revision ) - else: - # Make sure the defined changeset_revision is current. - defined_repository_metadata = \ - sa_session.query( app.model.RepositoryMetadata ) \ - .filter( app.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) \ - .first() - if defined_repository_metadata is None: - # The defined changeset_revision is not associated with a repository_metadata - # record, so updates must be necessary. - defined_repository = get_repository_by_name_and_owner( app, name, owner ) - defined_repo = hg_util.get_repo_for_repository( app, - repository=defined_repository, - repo_path=None, - create=False ) - updated_changeset_revision = \ - get_next_downloadable_changeset_revision( defined_repository, - defined_repo, - changeset_revision ) - if updated_changeset_revision == rm_changeset_revision: - dependent_downloadable_revisions.append( downloadable_revision ) - return dependent_downloadable_revisions - def get_ids_of_tool_shed_repositories_being_installed( app, as_string=False ): installing_repository_ids = [] new_status = app.install_model.ToolShedRepository.installation_status.NEW @@ -365,18 +280,6 @@ return ','.join( installing_repository_ids ) return installing_repository_ids -def get_latest_changeset_revision( app, repository, repo ): - repository_tip = repository.tip( app ) - repository_metadata = get_repository_metadata_by_changeset_revision( app, - app.security.encode_id( repository.id ), - repository_tip ) - if repository_metadata and repository_metadata.downloadable: - return repository_tip - changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=False ) - if changeset_revisions: - return changeset_revisions[ -1 ] - return hg_util.INITIAL_CHANGELOG_HASH - def get_latest_downloadable_changeset_revision( app, repository, repo ): repository_tip = repository.tip( app ) repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) @@ -441,37 +344,11 @@ continue return key -def get_or_create_tool_shed_repository( app, tool_shed, name, owner, changeset_revision ): +def get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ): """ - Return a tool shed repository database record defined by the combination of - tool shed, repository name, repository owner and changeset_revision or - installed_changeset_revision. A new tool shed repository record will be - created if one is not located. + Return an ordered list of changeset_revisions that are associated with metadata + where order is defined by the repository changelog. """ - install_model = app.install_model - # We store the port in the database. - tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) - # This method is used only in Galaxy, not the tool shed. - repository = get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ) - if not repository: - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed ) - repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) - ctx_rev = get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ) - repository = create_or_update_tool_shed_repository( app=app, - name=name, - description=None, - installed_changeset_revision=changeset_revision, - ctx_rev=ctx_rev, - repository_clone_url=repository_clone_url, - metadata_dict={}, - status=install_model.ToolShedRepository.installation_status.NEW, - current_changeset_revision=None, - owner=owner, - dist_to_shed=False ) - return repository - -def get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ): - """Return an ordered list of changeset_revisions that are associated with metadata where order is defined by the repository changelog.""" if downloadable: metadata_revisions = repository.downloadable_revisions else: @@ -491,9 +368,11 @@ def get_prior_import_or_install_required_dict( app, tsr_ids, repo_info_dicts ): """ - This method is used in the Tool Shed when exporting a repository and its dependencies, and in Galaxy when a repository and its dependencies - are being installed. Return a dictionary whose keys are the received tsr_ids and whose values are a list of tsr_ids, each of which is contained - in the received list of tsr_ids and whose associated repository must be imported or installed prior to the repository associated with the tsr_id key. + This method is used in the Tool Shed when exporting a repository and its dependencies, + and in Galaxy when a repository and its dependencies are being installed. Return a + dictionary whose keys are the received tsr_ids and whose values are a list of tsr_ids, + each of which is contained in the received list of tsr_ids and whose associated repository + must be imported or installed prior to the repository associated with the tsr_id key. """ # Initialize the dictionary. prior_import_or_install_required_dict = {} @@ -698,11 +577,14 @@ def get_repository_ids_requiring_prior_import_or_install( app, tsr_ids, repository_dependencies ): """ - This method is used in the Tool Shed when exporting a repository and its dependencies, and in Galaxy when a repository and its dependencies - are being installed. Inspect the received repository_dependencies and determine if the encoded id of each required repository is in the received - tsr_ids. If so, then determine whether that required repository should be imported / installed prior to its dependent repository. Return a list - of encoded repository ids, each of which is contained in the received list of tsr_ids, and whose associated repositories must be imported / installed - prior to the dependent repository associated with the received repository_dependencies. + This method is used in the Tool Shed when exporting a repository and its dependencies, + and in Galaxy when a repository and its dependencies are being installed. Inspect the + received repository_dependencies and determine if the encoded id of each required + repository is in the received tsr_ids. If so, then determine whether that required + repository should be imported / installed prior to its dependent repository. Return a + list of encoded repository ids, each of which is contained in the received list of tsr_ids, + and whose associated repositories must be imported / installed prior to the dependent + repository associated with the received repository_dependencies. """ prior_tsr_ids = [] if repository_dependencies: @@ -710,20 +592,32 @@ if key in [ 'description', 'root_key' ]: continue for rd_tup in rd_tups: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + tool_shed, \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) - # If only_if_compiling_contained_td is False, then the repository dependency is not required to be installed prior to the dependent - # repository even if prior_installation_required is True. This is because the only meaningful content of the repository dependency - # is its contained tool dependency, which is required in order to compile the dependent repository's tool dependency. In the scenario - # where the repository dependency is not installed prior to the dependent repository's tool dependency compilation process, the tool - # dependency compilation framework will install the repository dependency prior to compilation of the dependent repository's tool - # dependency. + # If only_if_compiling_contained_td is False, then the repository dependency + # is not required to be installed prior to the dependent repository even if + # prior_installation_required is True. This is because the only meaningful + # content of the repository dependency is its contained tool dependency, which + # is required in order to compile the dependent repository's tool dependency. + # In the scenario where the repository dependency is not installed prior to the + # dependent repository's tool dependency compilation process, the tool dependency + # compilation framework will install the repository dependency prior to compilation + # of the dependent repository's tool dependency. if not util.asbool( only_if_compiling_contained_td ): if util.asbool( prior_installation_required ): if is_tool_shed_client( app ): # We store the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) - repository = get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ) + repository = get_repository_for_dependency_relationship( app, + tool_shed, + name, + owner, + changeset_revision ) else: repository = get_repository_by_name_and_owner( app, name, owner ) if repository: @@ -779,14 +673,6 @@ query = app.model.context.query( app.model.Repository ) return query -def get_skip_tool_test_by_changeset_revision( app, changeset_revision ): - """Return a skip_tool_test record whose initial_changeset_revision is the received changeset_revision.""" - # There should only be one, but we'll use first() so callers won't have to handle exceptions. - sa_session = app.model.context.current - return sa_session.query( app.model.SkipToolTest ) \ - .filter( app.model.SkipToolTest.table.c.initial_changeset_revision == changeset_revision ) \ - .first() - def get_tool_panel_config_tool_path_install_dir( app, repository ): """ Return shed-related tool panel config, the tool_path configured in it, and the relative path to @@ -1220,42 +1106,6 @@ text = re.sub( r'\.\. image:: (?!https?://)/?(.+)', r'.. image:: %s/\1' % route_to_images, text ) return text -def set_only_if_compiling_contained_td( repository, required_repository ): - """ - Return True if the received required_repository is only needed to compile a tool - dependency defined for the received repository. - """ - # This method is called only from Galaxy when rendering repository dependencies - # for an installed tool shed repository. - # TODO: Do we need to check more than changeset_revision here? - required_repository_tup = [ required_repository.tool_shed, \ - required_repository.name, \ - required_repository.owner, \ - required_repository.changeset_revision ] - for tup in repository.tuples_of_repository_dependencies_needed_for_compiling_td: - partial_tup = tup[ 0:4 ] - if partial_tup == required_repository_tup: - return 'True' - return 'False' - -def set_prior_installation_required( app, repository, required_repository ): - """Return True if the received required_repository must be installed before the received repository.""" - # This method is called only from Galaxy when rendering repository dependencies for an installed Tool Shed repository. - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, str( required_repository.tool_shed ) ) - required_repository_tup = [ tool_shed_url, - str( required_repository.name ), - str( required_repository.owner ), - str( required_repository.changeset_revision ) ] - # Get the list of repository dependency tuples associated with the received repository where prior_installation_required is True. - required_rd_tups_that_must_be_installed = repository.requires_prior_installation_of - for required_rd_tup in required_rd_tups_that_must_be_installed: - # Repository dependency tuples in metadata include a prior_installation_required value, so strip it for comparision. - partial_required_rd_tup = required_rd_tup[ 0:4 ] - if partial_required_rd_tup == required_repository_tup: - # Return the string value of prior_installation_required, which defaults to 'False'. - return str( required_rd_tup[ 4 ] ) - return 'False' - def set_repository_attributes( app, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): if remove_from_disk: relative_install_dir = repository.repo_path( app ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Rename the Tool Shed's repository_maintenance_util module to be repository_util.
by commits-noreply＠bitbucket.org 22 Jul '14

22 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4ee9e584dbc0/ Changeset: 4ee9e584dbc0 User: greg Date: 2014-07-22 21:38:59 Summary: Rename the Tool Shed's repository_maintenance_util module to be repository_util. Affected #: 9 files diff -r 9cce4ead10795278446af4ab84eecfc4e2b26dce -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -16,7 +16,7 @@ from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import readme_util -from tool_shed.util import repository_maintenance_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -48,7 +48,7 @@ def activate_repository( self, trans, **kwd ): """Activate a repository that was deactivated but not uninstalled.""" repository_id = kwd[ 'id' ] - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) try: trans.app.installed_repository_manager.activate_repository( repository ) except Exception, e: @@ -74,7 +74,7 @@ def browse_repository( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) return trans.fill_template( '/admin/tool_shed_repository/browse_repository.mako', repository=repository, message=message, @@ -107,7 +107,7 @@ action='purge_repository', **kwd ) ) if operation == "activate or reinstall": - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if repository.uninstalled: # Since we're reinstalling the repository we need to find the latest changeset revision to which it can # be updated so that we can reset the metadata if necessary. This will ensure that information about @@ -208,7 +208,7 @@ def check_for_updates( self, trans, **kwd ): """Send a request to the relevant tool shed to see if there are any updates.""" repository_id = kwd.get( 'id', None ) - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = '?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s' % \ ( web.url_for( '/', qualified=True ), @@ -234,7 +234,7 @@ status = kwd.get( 'status', 'done' ) remove_from_disk = kwd.get( 'remove_from_disk', '' ) remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) - tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + tool_shed_repository = repository_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) shed_tool_conf, tool_path, relative_install_dir = \ suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) if relative_install_dir: @@ -397,7 +397,7 @@ of the installed tool shed repository in Galaxy. We need it so that we can derive the tool shed from which it was installed. """ - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) if tool_shed_url is None or repository_name is None or repository_owner is None or changeset_revision is None: message = "Unable to retrieve tool dependencies from the Tool Shed because one or more of the following required " @@ -422,7 +422,7 @@ Send a request to the appropriate tool shed to retrieve the dictionary of information required to reinstall an updated revision of an uninstalled tool shed repository. """ - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) if tool_shed_url is None or repository_name is None or repository_owner is None or changeset_revision is None: message = "Unable to retrieve updated repository information from the Tool Shed because one or more of the following " @@ -516,7 +516,7 @@ status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) if repository_id is not None: - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) if repository is not None: tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) name = str( repository.name ) @@ -574,7 +574,7 @@ dependencies are included in the updated revision. """ updating_repository_id = kwd.get( 'updating_repository_id', None ) - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, updating_repository_id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, updating_repository_id ) # All received dependencies need to be installed - confirmed by the caller. encoded_tool_dependencies_dict = kwd.get( 'encoded_tool_dependencies_dict', None ) if encoded_tool_dependencies_dict is not None: @@ -750,7 +750,7 @@ if repository_id is None: return trans.show_error_message( 'Missing required encoded repository id.' ) operation = kwd.get( 'operation', None ) - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) if repository is None: return trans.show_error_message( 'Invalid repository specified.' ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) @@ -1279,7 +1279,7 @@ repository_id = kwd.get( 'id', None ) new_kwd = {} if repository_id is not None: - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) if repository: if repository.is_new: if kwd.get( 'purge_repository_button', False ): @@ -1317,7 +1317,7 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository_id = kwd[ 'id' ] - tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) no_changes = kwd.get( 'no_changes', '' ) no_changes_checked = CheckboxField.is_checked( no_changes ) install_repository_dependencies = CheckboxField.is_checked( kwd.get( 'install_repository_dependencies', '' ) ) @@ -1387,14 +1387,14 @@ tool_dependencies = metadata.get( 'tool_dependencies', None ) else: tool_dependencies = None - repo_info_dict = repository_maintenance_util.create_repo_info_dict( trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.changeset_revision, - ctx_rev=ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = repository_util.create_repo_info_dict( trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.changeset_revision, + ctx_rev=ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) if repo_info_dict not in repo_info_dicts: repo_info_dicts.append( repo_info_dict ) # Make sure all tool_shed_repository records exist. @@ -1460,7 +1460,7 @@ action='browse_repositories', message=message, status=status ) ) - tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) rrm = RepairRepositoryManager( trans.app ) if kwd.get( 'repair_repository_button', False ): encoded_repair_dict = kwd.get( 'repair_dict', None ) @@ -1478,7 +1478,7 @@ repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) repositories_for_repair.append( repository ) return self.repair_tool_shed_repositories( trans, rrm, repositories_for_repair, ordered_repo_info_dicts ) - tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) repair_dict = rrm.get_repair_dict( tool_shed_repository ) encoded_repair_dict = encoding_util.tool_shed_encode( repair_dict ) ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) @@ -1540,7 +1540,7 @@ repository_id = kwd.get( 'id', None ) latest_changeset_revision = kwd.get( 'latest_changeset_revision', None ) latest_ctx_rev = kwd.get( 'latest_ctx_rev', None ) - tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( trans.app, tool_shed_repository ) metadata = tool_shed_repository.metadata tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( tool_shed_repository.tool_shed ) ) @@ -1600,14 +1600,14 @@ repository_dependencies = \ rdim.get_repository_dependencies_for_installed_tool_shed_repository( trans.app, tool_shed_repository ) - repo_info_dict = repository_maintenance_util.create_repo_info_dict( trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - ctx_rev=tool_shed_repository.ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = repository_util.create_repo_info_dict( trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + ctx_rev=tool_shed_repository.ctx_rev, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) irm = trans.app.installed_repository_manager dependencies_for_repository_dict = irm.get_dependencies_for_repository( tool_shed_url, repo_info_dict, @@ -1727,7 +1727,7 @@ @web.require_admin def reset_repository_metadata( self, trans, id ): """Reset all metadata on a single installed tool shed repository.""" - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( trans.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: @@ -1766,7 +1766,7 @@ @web.require_admin def reset_to_install( self, trans, **kwd ): """An error occurred while cloning the repository, so reset everything necessary to enable another attempt.""" - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if kwd.get( 'reset_repository', False ): suc.set_repository_attributes( trans.app, repository, @@ -1792,7 +1792,7 @@ Get the tool_versions from the tool shed for each tool in the installed revision of a selected tool shed repository and update the metadata for the repository's revision in the Galaxy database. """ - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ), str( repository.owner ), @@ -2076,8 +2076,7 @@ for repository in trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ .filter( trans.install_model.ToolShedRepository.table.c.deleted == False ): ok, updated = \ - repository_maintenance_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, - repository ) + repository_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, repository ) if ok: success_count += 1 else: @@ -2093,8 +2092,7 @@ repository_id = kwd.get( 'id', None ) repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id ) ok, updated = \ - repository_maintenance_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, - repository ) + repository_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, repository ) if ok: if updated: message = "The tool shed status for repository <b>%s</b> has been updated." % str( repository.name ) @@ -2113,7 +2111,7 @@ def view_tool_metadata( self, trans, repository_id, tool_id, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_util.get_installed_tool_shed_repository( trans.app, repository_id ) repository_metadata = repository.metadata shed_config_dict = repository.get_shed_config_dict( trans.app ) tool_metadata = {} diff -r 9cce4ead10795278446af4ab84eecfc4e2b26dce -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -19,7 +19,7 @@ from tool_shed.util import basic_util from tool_shed.util import encoding_util from tool_shed.util import hg_util -from tool_shed.util import repository_maintenance_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util @@ -194,10 +194,10 @@ includes_tools_for_display_in_tool_panel, \ has_repository_dependencies, \ has_repository_dependencies_only_if_compiling_contained_td = \ - repository_maintenance_util.get_repo_info_dict( trans.app, - trans.user, - encoded_repository_id, - changeset_revision ) + repository_util.get_repo_info_dict( trans.app, + trans.user, + encoded_repository_id, + changeset_revision ) return repository_dict, repository_metadata_dict, repo_info_dict else: log.debug( "Unable to locate repository_metadata record for repository id %s and changeset_revision %s" % \ diff -r 9cce4ead10795278446af4ab84eecfc4e2b26dce -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -11,7 +11,7 @@ from tool_shed.metadata import repository_metadata_manager from tool_shed.util import metadata_util -from tool_shed.util import repository_maintenance_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc log = logging.getLogger( __name__ ) @@ -320,14 +320,14 @@ def manage_role_associations( self, trans, **kwd ): """Manage users, groups and repositories associated with a role.""" role_id = kwd.get( 'id', None ) - role = repository_maintenance_util.get_role_by_id( trans.app, role_id ) + role = repository_util.get_role_by_id( trans.app, role_id ) # We currently only have a single role associated with a repository, the repository admin role. repository_role_association = role.repositories[ 0 ] repository = repository_role_association.repository - associations_dict = repository_maintenance_util.handle_role_associations( trans.app, - role, - repository, - **kwd ) + associations_dict = repository_util.handle_role_associations( trans.app, + role, + repository, + **kwd ) in_users = associations_dict.get( 'in_users', [] ) out_users = associations_dict.get( 'out_users', [] ) in_groups = associations_dict.get( 'in_groups', [] ) diff -r 9cce4ead10795278446af4ab84eecfc4e2b26dce -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -31,7 +31,7 @@ from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import readme_util -from tool_shed.util import repository_maintenance_util +from tool_shed.util import repository_util from tool_shed.util import search_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util @@ -1070,7 +1070,7 @@ repository_type = kwd.get( 'repository_type', rt_util.UNRESTRICTED ) if kwd.get( 'create_repository_button', False ): error = False - message = repository_maintenance_util.validate_repository_name( trans.app, name, trans.user ) + message = repository_util.validate_repository_name( trans.app, name, trans.user ) if message: error = True if not description: @@ -1079,13 +1079,13 @@ if error: status = 'error' else: - repository, message = repository_maintenance_util.create_repository( trans.app, - name, - repository_type, - description, - long_description, - user_id=trans.user.id, - category_ids=category_ids ) + repository, message = repository_util.create_repository( trans.app, + name, + repository_type, + description, + long_description, + user_id=trans.user.id, + category_ids=category_ids ) trans.response.send_redirect( web.url_for( controller='repository', action='manage_repository', message=message, @@ -1820,7 +1820,7 @@ repository_id, changeset_revision = tup repo_info_dict, cur_includes_tools, cur_includes_tool_dependencies, cur_includes_tools_for_display_in_tool_panel, \ cur_has_repository_dependencies, cur_has_repository_dependencies_only_if_compiling_contained_td = \ - repository_maintenance_util.get_repo_info_dict( trans.app, trans.user, repository_id, changeset_revision ) + repository_util.get_repo_info_dict( trans.app, trans.user, repository_id, changeset_revision ) if cur_has_repository_dependencies and not has_repository_dependencies: has_repository_dependencies = True if cur_has_repository_dependencies_only_if_compiling_contained_td and not has_repository_dependencies_only_if_compiling_contained_td: @@ -1951,16 +1951,16 @@ after_changeset_revision=changeset_revision ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) - repo_info_dict = repository_maintenance_util.create_repo_info_dict( app=trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=changeset_revision, - ctx_rev=str( ctx.rev() ), - repository_owner=repository.user.username, - repository_name=repository.name, - repository=repository, - repository_metadata=repository_metadata, - tool_dependencies=None, - repository_dependencies=None ) + repo_info_dict = repository_util.create_repo_info_dict( app=trans.app, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + ctx_rev=str( ctx.rev() ), + repository_owner=repository.user.username, + repository_name=repository.name, + repository=repository, + repository_metadata=repository_metadata, + tool_dependencies=None, + repository_dependencies=None ) includes_data_managers = False includes_datatypes = False includes_tools = False @@ -2274,7 +2274,7 @@ repository.long_description = long_description flush_needed = True if repository.times_downloaded == 0 and repo_name != repository.name: - message = repository_maintenance_util.validate_repository_name( trans.app, repo_name, user ) + message = repository_util.validate_repository_name( trans.app, repo_name, user ) if message: error = True else: @@ -2284,12 +2284,12 @@ trans.app.hgweb_config_manager.change_entry( old_lhs, new_lhs, repo_dir ) # Change the entry in the repository's hgrc file. hgrc_file = os.path.join( repo_dir, '.hg', 'hgrc' ) - repository_maintenance_util.change_repository_name_in_hgrc_file( hgrc_file, repo_name ) + repository_util.change_repository_name_in_hgrc_file( hgrc_file, repo_name ) # Rename the repository's admin role to match the new repository name. repository_admin_role = repository.admin_role repository_admin_role.name = \ - repository_maintenance_util.get_repository_admin_role_name( str( repo_name ), - str( repository.user.username ) ) + repository_util.get_repository_admin_role_name( str( repo_name ), + str( repository.user.username ) ) trans.sa_session.add( repository_admin_role ) repository.name = repo_name flush_needed = True @@ -2379,7 +2379,7 @@ current_allow_push_list = current_allow_push.split( ',' ) else: current_allow_push_list = [] - allow_push_select_field = repository_maintenance_util.build_allow_push_select_field( trans, current_allow_push_list ) + allow_push_select_field = repository_util.build_allow_push_select_field( trans, current_allow_push_list ) checked = alerts_checked or user.email in email_alerts alerts_check_box = CheckboxField( 'alerts', checked=checked ) changeset_revision_select_field = grids_util.build_changeset_revision_select_field( trans, @@ -2521,10 +2521,10 @@ if repository_metadata: metadata = repository_metadata.metadata role = repository.admin_role - associations_dict = repository_maintenance_util.handle_role_associations( trans.app, - role, - repository, - **kwd ) + associations_dict = repository_util.handle_role_associations( trans.app, + role, + repository, + **kwd ) in_users = associations_dict.get( 'in_users', [] ) out_users = associations_dict.get( 'out_users', [] ) in_groups = associations_dict.get( 'in_groups', [] ) diff -r 9cce4ead10795278446af4ab84eecfc4e2b26dce -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -24,7 +24,7 @@ from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util -from tool_shed.util import repository_maintenance_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util @@ -462,13 +462,13 @@ else: category_ids.append( self.app.security.encode_id( category.id ) ) # Create the repository record in the database. - repository, create_message = repository_maintenance_util.create_repository( self.app, - name, - type, - description, - long_description, - user_id=user_id, - category_ids=category_ids ) + repository, create_message = repository_util.create_repository( self.app, + name, + type, + description, + long_description, + user_id=user_id, + category_ids=category_ids ) if create_message: results_message += create_message # Populate the new repository with the contents of exported repository archive. diff -r 9cce4ead10795278446af4ab84eecfc4e2b26dce -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -9,7 +9,7 @@ from tool_shed.metadata import metadata_generator from tool_shed.util import common_util -from tool_shed.util import repository_maintenance_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -78,7 +78,7 @@ """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" invalid_file_tups = [] metadata_dict = {} - repository = repository_maintenance_util.get_installed_tool_shed_repository( self.app, id ) + repository = repository_util.get_installed_tool_shed_repository( self.app, id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( self.app ) if relative_install_dir: @@ -118,7 +118,7 @@ unsuccessful_count = 0 for repository_id in repository_ids: try: - repository = repository_maintenance_util.get_installed_tool_shed_repository( self.app, repository_id ) + repository = repository_util.get_installed_tool_shed_repository( self.app, repository_id ) owner = str( repository.owner ) invalid_file_tups, metadata_dict = \ self.reset_all_metadata_on_installed_repository( repository_id ) diff -r 9cce4ead10795278446af4ab84eecfc4e2b26dce -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -10,7 +10,7 @@ from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import repository_maintenance_util +from tool_shed.util import repository_util from tool_shed.util import tool_dependency_util @@ -127,16 +127,16 @@ new_tool_panel_section_label=tool_panel_section_name ) else: tool_dependencies = None - repo_info_dict = repository_maintenance_util.create_repo_info_dict( app=self.app, - repository_clone_url=repository_clone_url, - changeset_revision=repository.changeset_revision, - ctx_rev=repository.ctx_rev, - repository_owner=repository.owner, - repository_name=repository.name, - repository=None, - repository_metadata=None, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies ) + repo_info_dict = repository_util.create_repo_info_dict( app=self.app, + repository_clone_url=repository_clone_url, + changeset_revision=repository.changeset_revision, + ctx_rev=repository.ctx_rev, + repository_owner=repository.owner, + repository_name=repository.name, + repository=None, + repository_metadata=None, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) return repo_info_dict, tool_panel_section_key def repair_tool_shed_repository( self, repository, repo_info_dict ): diff -r 9cce4ead10795278446af4ab84eecfc4e2b26dce -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a lib/tool_shed/util/repository_maintenance_util.py --- a/lib/tool_shed/util/repository_maintenance_util.py +++ /dev/null @@ -1,330 +0,0 @@ -import ConfigParser -import logging -import os -import re - -from galaxy import util -from galaxy import web -from galaxy.web.form_builder import build_select_field -from galaxy.webapps.tool_shed.model import directory_hash_id - -from tool_shed.dependencies.repository import relation_builder - -from tool_shed.util import common_util -from tool_shed.util import hg_util -from tool_shed.util import shed_util_common as suc - -log = logging.getLogger( __name__ ) - -VALID_REPOSITORYNAME_RE = re.compile( "^[a-z0-9\_]+$" ) - -def build_allow_push_select_field( trans, current_push_list, selected_value='none' ): - options = [] - for user in trans.sa_session.query( trans.model.User ): - if user.username not in current_push_list: - options.append( user ) - return build_select_field( trans, - objs=options, - label_attr='username', - select_field_name='allow_push', - selected_value=selected_value, - refresh_on_change=False, - multiple=True ) - -def change_repository_name_in_hgrc_file( hgrc_file, new_name ): - config = ConfigParser.ConfigParser() - config.read( hgrc_file ) - config.read( hgrc_file ) - config.set( 'web', 'name', new_name ) - new_file = open( hgrc_file, 'wb' ) - config.write( new_file ) - new_file.close() - -def check_or_update_tool_shed_status_for_installed_repository( app, repository ): - updated = False - tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( app, repository ) - if tool_shed_status_dict: - ok = True - if tool_shed_status_dict != repository.tool_shed_status: - repository.tool_shed_status = tool_shed_status_dict - app.install_model.context.add( repository ) - app.install_model.context.flush() - updated = True - else: - ok = False - return ok, updated - -def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, - repository=None, repository_metadata=None, tool_dependencies=None, repository_dependencies=None ): - """ - Return a dictionary that includes all of the information needed to install a repository into a local - Galaxy instance. The dictionary will also contain the recursive list of repository dependencies defined - for the repository, as well as the defined tool dependencies. - - This method is called from Galaxy under four scenarios: - 1. During the tool shed repository installation process via the tool shed's get_repository_information() - method. In this case both the received repository and repository_metadata will be objects, but - tool_dependencies and repository_dependencies will be None. - 2. When getting updates for an installed repository where the updates include newly defined repository - dependency definitions. This scenario is similar to 1. above. The tool shed's get_repository_information() - method is the caller, and both the received repository and repository_metadata will be objects, but - tool_dependencies and repository_dependencies will be None. - 3. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with no - updates available. In this case, both repository and repository_metadata will be None, but tool_dependencies - and repository_dependencies will be objects previously retrieved from the tool shed if the repository includes - definitions for them. - 4. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with updates - available. In this case, this method is reached via the tool shed's get_updated_repository_information() - method, and both repository and repository_metadata will be objects but tool_dependencies and - repository_dependencies will be None. - """ - repo_info_dict = {} - repository = suc.get_repository_by_name_and_owner( app, repository_name, repository_owner ) - if app.name == 'tool_shed': - # We're in the tool shed. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, - app.security.encode_id( repository.id ), - changeset_revision ) - if repository_metadata: - metadata = repository_metadata.metadata - if metadata: - tool_shed_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) - rb = relation_builder.RelationBuilder( app, repository, repository_metadata, tool_shed_url ) - # Get a dictionary of all repositories upon which the contents of the received repository depends. - repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() - tool_dependencies = metadata.get( 'tool_dependencies', {} ) - if tool_dependencies: - new_tool_dependencies = {} - for dependency_key, requirements_dict in tool_dependencies.items(): - if dependency_key in [ 'set_environment' ]: - new_set_environment_dict_list = [] - for set_environment_dict in requirements_dict: - set_environment_dict[ 'repository_name' ] = repository_name - set_environment_dict[ 'repository_owner' ] = repository_owner - set_environment_dict[ 'changeset_revision' ] = changeset_revision - new_set_environment_dict_list.append( set_environment_dict ) - new_tool_dependencies[ dependency_key ] = new_set_environment_dict_list - else: - requirements_dict[ 'repository_name' ] = repository_name - requirements_dict[ 'repository_owner' ] = repository_owner - requirements_dict[ 'changeset_revision' ] = changeset_revision - new_tool_dependencies[ dependency_key ] = requirements_dict - tool_dependencies = new_tool_dependencies - # Cast unicode to string, with the exception of description, since it is free text and can contain special characters. - repo_info_dict[ str( repository.name ) ] = ( repository.description, - str( repository_clone_url ), - str( changeset_revision ), - str( ctx_rev ), - str( repository_owner ), - repository_dependencies, - tool_dependencies ) - return repo_info_dict - -def create_repository( app, name, type, description, long_description, user_id, category_ids=[] ): - sa_session = app.model.context.current - # Add the repository record to the database. - repository = app.model.Repository( name=name, - type=type, - description=description, - long_description=long_description, - user_id=user_id ) - # Flush to get the id. - sa_session.add( repository ) - sa_session.flush() - # Create an admin role for the repository. - repository_admin_role = create_repository_admin_role( app, repository ) - # Determine the repository's repo_path on disk. - dir = os.path.join( app.config.file_path, *directory_hash_id( repository.id ) ) - # Create directory if it does not exist. - if not os.path.exists( dir ): - os.makedirs( dir ) - # Define repo name inside hashed directory. - repository_path = os.path.join( dir, "repo_%d" % repository.id ) - # Create local repository directory. - if not os.path.exists( repository_path ): - os.makedirs( repository_path ) - # Create the local repository. - repo = hg_util.get_repo_for_repository( app, repository=None, repo_path=repository_path, create=True ) - # Add an entry in the hgweb.config file for the local repository. - lhs = "repos/%s/%s" % ( repository.user.username, repository.name ) - app.hgweb_config_manager.add_entry( lhs, repository_path ) - # Create a .hg/hgrc file for the local repository. - hg_util.create_hgrc_file( app, repository ) - flush_needed = False - if category_ids: - # Create category associations - for category_id in category_ids: - category = sa_session.query( app.model.Category ) \ - .get( app.security.decode_id( category_id ) ) - rca = app.model.RepositoryCategoryAssociation( repository, category ) - sa_session.add( rca ) - flush_needed = True - if flush_needed: - sa_session.flush() - # Update the repository registry. - app.repository_registry.add_entry( repository ) - message = "Repository <b>%s</b> has been created." % str( repository.name ) - return repository, message - -def create_repository_admin_role( app, repository ): - """ - Create a new role with name-spaced name based on the repository name and its owner's public user - name. This will ensure that the tole name is unique. - """ - sa_session = app.model.context.current - name = get_repository_admin_role_name( str( repository.name ), str( repository.user.username ) ) - description = 'A user or group member with this role can administer this repository.' - role = app.model.Role( name=name, description=description, type=app.model.Role.types.SYSTEM ) - sa_session.add( role ) - sa_session.flush() - # Associate the role with the repository owner. - ura = app.model.UserRoleAssociation( repository.user, role ) - # Associate the role with the repository. - rra = app.model.RepositoryRoleAssociation( repository, role ) - sa_session.add( rra ) - sa_session.flush() - return role - -def get_installed_tool_shed_repository( app, id ): - """Get a tool shed repository record from the Galaxy database defined by the id.""" - return app.install_model.context.query( app.install_model.ToolShedRepository ) \ - .get( app.security.decode_id( id ) ) - -def get_repo_info_dict( app, user, repository_id, changeset_revision ): - repository = suc.get_repository_in_tool_shed( app, repository_id ) - repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, - repository_id, - changeset_revision ) - if not repository_metadata: - # The received changeset_revision is no longer installable, so get the next changeset_revision - # in the repository's changelog. This generally occurs only with repositories of type - # repository_suite_definition or tool_dependency_definition. - next_downloadable_changeset_revision = \ - suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - if next_downloadable_changeset_revision: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, - repository_id, - next_downloadable_changeset_revision ) - if repository_metadata: - # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption - # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the - # returned repo_info_dict. - metadata = repository_metadata.metadata - if 'tools' in metadata: - includes_tools = True - else: - includes_tools = False - includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel - repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) - repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) - has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ - suc.get_repository_dependency_types( repository_dependencies ) - if 'tool_dependencies' in metadata: - includes_tool_dependencies = True - else: - includes_tool_dependencies = False - else: - # Here's where we may have to handle enhancements to the callers. See above comment. - includes_tools = False - has_repository_dependencies = False - has_repository_dependencies_only_if_compiling_contained_td = False - includes_tool_dependencies = False - includes_tools_for_display_in_tool_panel = False - ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) - repo_info_dict = create_repo_info_dict( app=app, - repository_clone_url=repository_clone_url, - changeset_revision=changeset_revision, - ctx_rev=str( ctx.rev() ), - repository_owner=repository.user.username, - repository_name=repository.name, - repository=repository, - repository_metadata=repository_metadata, - tool_dependencies=None, - repository_dependencies=None ) - return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ - has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td - -def get_repository_admin_role_name( repository_name, repository_owner ): - return '%s_%s_admin' % ( str( repository_name ), str( repository_owner ) ) - -def get_role_by_id( app, role_id ): - """Get a Role from the database by id.""" - sa_session = app.model.context.current - return sa_session.query( app.model.Role ).get( app.security.decode_id( role_id ) ) - -def handle_role_associations( app, role, repository, **kwd ): - sa_session = app.model.context.current - message = kwd.get( 'message', '' ) - status = kwd.get( 'status', 'done' ) - repository_owner = repository.user - if kwd.get( 'manage_role_associations_button', False ): - in_users_list = util.listify( kwd.get( 'in_users', [] ) ) - in_users = [ sa_session.query( app.model.User ).get( x ) for x in in_users_list ] - # Make sure the repository owner is always associated with the repostory's admin role. - owner_associated = False - for user in in_users: - if user.id == repository_owner.id: - owner_associated = True - break - if not owner_associated: - in_users.append( repository_owner ) - message += "The repository owner must always be associated with the repository's administrator role. " - status = 'error' - in_groups_list = util.listify( kwd.get( 'in_groups', [] ) ) - in_groups = [ sa_session.query( app.model.Group ).get( x ) for x in in_groups_list ] - in_repositories = [ repository ] - app.security_agent.set_entity_role_associations( roles=[ role ], - users=in_users, - groups=in_groups, - repositories=in_repositories ) - sa_session.refresh( role ) - message += "Role <b>%s</b> has been associated with %d users, %d groups and %d repositories. " % \ - ( str( role.name ), len( in_users ), len( in_groups ), len( in_repositories ) ) - in_users = [] - out_users = [] - in_groups = [] - out_groups = [] - for user in sa_session.query( app.model.User ) \ - .filter( app.model.User.table.c.deleted==False ) \ - .order_by( app.model.User.table.c.email ): - if user in [ x.user for x in role.users ]: - in_users.append( ( user.id, user.email ) ) - else: - out_users.append( ( user.id, user.email ) ) - for group in sa_session.query( app.model.Group ) \ - .filter( app.model.Group.table.c.deleted==False ) \ - .order_by( app.model.Group.table.c.name ): - if group in [ x.group for x in role.groups ]: - in_groups.append( ( group.id, group.name ) ) - else: - out_groups.append( ( group.id, group.name ) ) - associations_dict = dict( in_users=in_users, - out_users=out_users, - in_groups=in_groups, - out_groups=out_groups, - message=message, - status=status ) - return associations_dict - -def validate_repository_name( app, name, user ): - # Repository names must be unique for each user, must be at least four characters - # in length and must contain only lower-case letters, numbers, and the '_' character. - if name in [ 'None', None, '' ]: - return 'Enter the required repository name.' - if name in [ 'repos' ]: - return "The term <b>%s</b> is a reserved word in the tool shed, so it cannot be used as a repository name." % name - check_existing = suc.get_repository_by_name_and_owner( app, name, user.username ) - if check_existing is not None: - if check_existing.deleted: - return 'You have a deleted repository named <b>%s</b>, so choose a different name.' % name - else: - return "You already have a repository named <b>%s</b>, so choose a different name." % name - if len( name ) < 4: - return "Repository names must be at least 4 characters in length." - if len( name ) > 80: - return "Repository names cannot be more than 80 characters in length." - if not( VALID_REPOSITORYNAME_RE.match( name ) ): - return "Repository names must contain only lower-case letters, numbers and underscore <b>_</b>." - return '' diff -r 9cce4ead10795278446af4ab84eecfc4e2b26dce -r 4ee9e584dbc016727e4c90f73bbf53081f775c5a lib/tool_shed/util/repository_util.py --- /dev/null +++ b/lib/tool_shed/util/repository_util.py @@ -0,0 +1,330 @@ +import ConfigParser +import logging +import os +import re + +from galaxy import util +from galaxy import web +from galaxy.web.form_builder import build_select_field +from galaxy.webapps.tool_shed.model import directory_hash_id + +from tool_shed.dependencies.repository import relation_builder + +from tool_shed.util import common_util +from tool_shed.util import hg_util +from tool_shed.util import shed_util_common as suc + +log = logging.getLogger( __name__ ) + +VALID_REPOSITORYNAME_RE = re.compile( "^[a-z0-9\_]+$" ) + +def build_allow_push_select_field( trans, current_push_list, selected_value='none' ): + options = [] + for user in trans.sa_session.query( trans.model.User ): + if user.username not in current_push_list: + options.append( user ) + return build_select_field( trans, + objs=options, + label_attr='username', + select_field_name='allow_push', + selected_value=selected_value, + refresh_on_change=False, + multiple=True ) + +def change_repository_name_in_hgrc_file( hgrc_file, new_name ): + config = ConfigParser.ConfigParser() + config.read( hgrc_file ) + config.read( hgrc_file ) + config.set( 'web', 'name', new_name ) + new_file = open( hgrc_file, 'wb' ) + config.write( new_file ) + new_file.close() + +def check_or_update_tool_shed_status_for_installed_repository( app, repository ): + updated = False + tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( app, repository ) + if tool_shed_status_dict: + ok = True + if tool_shed_status_dict != repository.tool_shed_status: + repository.tool_shed_status = tool_shed_status_dict + app.install_model.context.add( repository ) + app.install_model.context.flush() + updated = True + else: + ok = False + return ok, updated + +def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, + repository=None, repository_metadata=None, tool_dependencies=None, repository_dependencies=None ): + """ + Return a dictionary that includes all of the information needed to install a repository into a local + Galaxy instance. The dictionary will also contain the recursive list of repository dependencies defined + for the repository, as well as the defined tool dependencies. + + This method is called from Galaxy under four scenarios: + 1. During the tool shed repository installation process via the tool shed's get_repository_information() + method. In this case both the received repository and repository_metadata will be objects, but + tool_dependencies and repository_dependencies will be None. + 2. When getting updates for an installed repository where the updates include newly defined repository + dependency definitions. This scenario is similar to 1. above. The tool shed's get_repository_information() + method is the caller, and both the received repository and repository_metadata will be objects, but + tool_dependencies and repository_dependencies will be None. + 3. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with no + updates available. In this case, both repository and repository_metadata will be None, but tool_dependencies + and repository_dependencies will be objects previously retrieved from the tool shed if the repository includes + definitions for them. + 4. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with updates + available. In this case, this method is reached via the tool shed's get_updated_repository_information() + method, and both repository and repository_metadata will be objects but tool_dependencies and + repository_dependencies will be None. + """ + repo_info_dict = {} + repository = suc.get_repository_by_name_and_owner( app, repository_name, repository_owner ) + if app.name == 'tool_shed': + # We're in the tool shed. + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, + app.security.encode_id( repository.id ), + changeset_revision ) + if repository_metadata: + metadata = repository_metadata.metadata + if metadata: + tool_shed_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) + rb = relation_builder.RelationBuilder( app, repository, repository_metadata, tool_shed_url ) + # Get a dictionary of all repositories upon which the contents of the received repository depends. + repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() + tool_dependencies = metadata.get( 'tool_dependencies', {} ) + if tool_dependencies: + new_tool_dependencies = {} + for dependency_key, requirements_dict in tool_dependencies.items(): + if dependency_key in [ 'set_environment' ]: + new_set_environment_dict_list = [] + for set_environment_dict in requirements_dict: + set_environment_dict[ 'repository_name' ] = repository_name + set_environment_dict[ 'repository_owner' ] = repository_owner + set_environment_dict[ 'changeset_revision' ] = changeset_revision + new_set_environment_dict_list.append( set_environment_dict ) + new_tool_dependencies[ dependency_key ] = new_set_environment_dict_list + else: + requirements_dict[ 'repository_name' ] = repository_name + requirements_dict[ 'repository_owner' ] = repository_owner + requirements_dict[ 'changeset_revision' ] = changeset_revision + new_tool_dependencies[ dependency_key ] = requirements_dict + tool_dependencies = new_tool_dependencies + # Cast unicode to string, with the exception of description, since it is free text and can contain special characters. + repo_info_dict[ str( repository.name ) ] = ( repository.description, + str( repository_clone_url ), + str( changeset_revision ), + str( ctx_rev ), + str( repository_owner ), + repository_dependencies, + tool_dependencies ) + return repo_info_dict + +def create_repository( app, name, type, description, long_description, user_id, category_ids=[] ): + sa_session = app.model.context.current + # Add the repository record to the database. + repository = app.model.Repository( name=name, + type=type, + description=description, + long_description=long_description, + user_id=user_id ) + # Flush to get the id. + sa_session.add( repository ) + sa_session.flush() + # Create an admin role for the repository. + repository_admin_role = create_repository_admin_role( app, repository ) + # Determine the repository's repo_path on disk. + dir = os.path.join( app.config.file_path, *directory_hash_id( repository.id ) ) + # Create directory if it does not exist. + if not os.path.exists( dir ): + os.makedirs( dir ) + # Define repo name inside hashed directory. + repository_path = os.path.join( dir, "repo_%d" % repository.id ) + # Create local repository directory. + if not os.path.exists( repository_path ): + os.makedirs( repository_path ) + # Create the local repository. + repo = hg_util.get_repo_for_repository( app, repository=None, repo_path=repository_path, create=True ) + # Add an entry in the hgweb.config file for the local repository. + lhs = "repos/%s/%s" % ( repository.user.username, repository.name ) + app.hgweb_config_manager.add_entry( lhs, repository_path ) + # Create a .hg/hgrc file for the local repository. + hg_util.create_hgrc_file( app, repository ) + flush_needed = False + if category_ids: + # Create category associations + for category_id in category_ids: + category = sa_session.query( app.model.Category ) \ + .get( app.security.decode_id( category_id ) ) + rca = app.model.RepositoryCategoryAssociation( repository, category ) + sa_session.add( rca ) + flush_needed = True + if flush_needed: + sa_session.flush() + # Update the repository registry. + app.repository_registry.add_entry( repository ) + message = "Repository <b>%s</b> has been created." % str( repository.name ) + return repository, message + +def create_repository_admin_role( app, repository ): + """ + Create a new role with name-spaced name based on the repository name and its owner's public user + name. This will ensure that the tole name is unique. + """ + sa_session = app.model.context.current + name = get_repository_admin_role_name( str( repository.name ), str( repository.user.username ) ) + description = 'A user or group member with this role can administer this repository.' + role = app.model.Role( name=name, description=description, type=app.model.Role.types.SYSTEM ) + sa_session.add( role ) + sa_session.flush() + # Associate the role with the repository owner. + ura = app.model.UserRoleAssociation( repository.user, role ) + # Associate the role with the repository. + rra = app.model.RepositoryRoleAssociation( repository, role ) + sa_session.add( rra ) + sa_session.flush() + return role + +def get_installed_tool_shed_repository( app, id ): + """Get a tool shed repository record from the Galaxy database defined by the id.""" + return app.install_model.context.query( app.install_model.ToolShedRepository ) \ + .get( app.security.decode_id( id ) ) + +def get_repo_info_dict( app, user, repository_id, changeset_revision ): + repository = suc.get_repository_in_tool_shed( app, repository_id ) + repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, + repository_id, + changeset_revision ) + if not repository_metadata: + # The received changeset_revision is no longer installable, so get the next changeset_revision + # in the repository's changelog. This generally occurs only with repositories of type + # repository_suite_definition or tool_dependency_definition. + next_downloadable_changeset_revision = \ + suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + if next_downloadable_changeset_revision: + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, + repository_id, + next_downloadable_changeset_revision ) + if repository_metadata: + # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption + # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the + # returned repo_info_dict. + metadata = repository_metadata.metadata + if 'tools' in metadata: + includes_tools = True + else: + includes_tools = False + includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel + repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) + repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) + has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ + suc.get_repository_dependency_types( repository_dependencies ) + if 'tool_dependencies' in metadata: + includes_tool_dependencies = True + else: + includes_tool_dependencies = False + else: + # Here's where we may have to handle enhancements to the callers. See above comment. + includes_tools = False + has_repository_dependencies = False + has_repository_dependencies_only_if_compiling_contained_td = False + includes_tool_dependencies = False + includes_tools_for_display_in_tool_panel = False + ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) + repo_info_dict = create_repo_info_dict( app=app, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + ctx_rev=str( ctx.rev() ), + repository_owner=repository.user.username, + repository_name=repository.name, + repository=repository, + repository_metadata=repository_metadata, + tool_dependencies=None, + repository_dependencies=None ) + return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ + has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td + +def get_repository_admin_role_name( repository_name, repository_owner ): + return '%s_%s_admin' % ( str( repository_name ), str( repository_owner ) ) + +def get_role_by_id( app, role_id ): + """Get a Role from the database by id.""" + sa_session = app.model.context.current + return sa_session.query( app.model.Role ).get( app.security.decode_id( role_id ) ) + +def handle_role_associations( app, role, repository, **kwd ): + sa_session = app.model.context.current + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + repository_owner = repository.user + if kwd.get( 'manage_role_associations_button', False ): + in_users_list = util.listify( kwd.get( 'in_users', [] ) ) + in_users = [ sa_session.query( app.model.User ).get( x ) for x in in_users_list ] + # Make sure the repository owner is always associated with the repostory's admin role. + owner_associated = False + for user in in_users: + if user.id == repository_owner.id: + owner_associated = True + break + if not owner_associated: + in_users.append( repository_owner ) + message += "The repository owner must always be associated with the repository's administrator role. " + status = 'error' + in_groups_list = util.listify( kwd.get( 'in_groups', [] ) ) + in_groups = [ sa_session.query( app.model.Group ).get( x ) for x in in_groups_list ] + in_repositories = [ repository ] + app.security_agent.set_entity_role_associations( roles=[ role ], + users=in_users, + groups=in_groups, + repositories=in_repositories ) + sa_session.refresh( role ) + message += "Role <b>%s</b> has been associated with %d users, %d groups and %d repositories. " % \ + ( str( role.name ), len( in_users ), len( in_groups ), len( in_repositories ) ) + in_users = [] + out_users = [] + in_groups = [] + out_groups = [] + for user in sa_session.query( app.model.User ) \ + .filter( app.model.User.table.c.deleted==False ) \ + .order_by( app.model.User.table.c.email ): + if user in [ x.user for x in role.users ]: + in_users.append( ( user.id, user.email ) ) + else: + out_users.append( ( user.id, user.email ) ) + for group in sa_session.query( app.model.Group ) \ + .filter( app.model.Group.table.c.deleted==False ) \ + .order_by( app.model.Group.table.c.name ): + if group in [ x.group for x in role.groups ]: + in_groups.append( ( group.id, group.name ) ) + else: + out_groups.append( ( group.id, group.name ) ) + associations_dict = dict( in_users=in_users, + out_users=out_users, + in_groups=in_groups, + out_groups=out_groups, + message=message, + status=status ) + return associations_dict + +def validate_repository_name( app, name, user ): + # Repository names must be unique for each user, must be at least four characters + # in length and must contain only lower-case letters, numbers, and the '_' character. + if name in [ 'None', None, '' ]: + return 'Enter the required repository name.' + if name in [ 'repos' ]: + return "The term <b>%s</b> is a reserved word in the tool shed, so it cannot be used as a repository name." % name + check_existing = suc.get_repository_by_name_and_owner( app, name, user.username ) + if check_existing is not None: + if check_existing.deleted: + return 'You have a deleted repository named <b>%s</b>, so choose a different name.' % name + else: + return "You already have a repository named <b>%s</b>, so choose a different name." % name + if len( name ) < 4: + return "Repository names must be at least 4 characters in length." + if len( name ) > 80: + return "Repository names cannot be more than 80 characters in length." + if not( VALID_REPOSITORYNAME_RE.match( name ) ): + return "Repository names must contain only lower-case letters, numbers and underscore <b>_</b>." + return '' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a ToolDataTableManager for managing tool data tables in the Tool Shed.
by commits-noreply＠bitbucket.org 22 Jul '14

22 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9cce4ead1079/ Changeset: 9cce4ead1079 User: greg Date: 2014-07-22 21:12:40 Summary: Add a ToolDataTableManager for managing tool data tables in the Tool Shed. Affected #: 9 files diff -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 -r 9cce4ead10795278446af4ab84eecfc4e2b26dce lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -13,13 +13,13 @@ from tool_shed.dependencies import attribute_handlers from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager -import tool_shed.repository_types.util as rt_util +from tool_shed.repository_types import util as rt_util +from tool_shed.tools import data_table_manager from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import tool_util from tool_shed.util import xml_util from galaxy import eggs @@ -100,6 +100,7 @@ if uploaded_file or uploaded_directory: rdah = attribute_handlers.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False ) tdah = attribute_handlers.ToolDependencyAttributeHandler( trans.app, unpopulate=False ) + tdtm = data_table_manager.ToolDataTableManager( trans.app ) ok = True isgzip = False isbz2 = False @@ -221,7 +222,7 @@ # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded # by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables # dictionary. - error, error_message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, full_path ) + error, error_message = tdtm.handle_sample_tool_data_table_conf_file( full_path, persist=False ) if error: message = '%s<br/>%s' % ( message, error_message ) # See if the content of the change set was valid. @@ -309,7 +310,7 @@ message += invalid_repository_dependencies_message status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - tool_util.reset_tool_data_tables( trans.app ) + tdtm.reset_tool_data_tables() if uploaded_directory: basic_util.remove_dir( uploaded_directory ) trans.response.send_redirect( web.url_for( controller='repository', @@ -323,7 +324,7 @@ basic_util.remove_dir( uploaded_directory ) status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - tool_util.reset_tool_data_tables( trans.app ) + tdtm.reset_tool_data_tables() selected_categories = [ trans.security.decode_id( id ) for id in category_ids ] return trans.fill_template( '/webapps/tool_shed/repository/upload.mako', repository=repository, diff -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 -r 9cce4ead10795278446af4ab84eecfc4e2b26dce lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -35,6 +35,7 @@ from tool_shed.galaxy_install.tools import data_manager from tool_shed.galaxy_install.tools import tool_panel_manager +from tool_shed.tools import data_table_manager from tool_shed.tools import tool_version_manager log = logging.getLogger( __name__ ) @@ -508,6 +509,7 @@ reinstalling an uninstalled repository. """ shed_config_dict = self.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) + tdtm = data_table_manager.ToolDataTableManager( self.app ) irmm = InstalledRepositoryMetadataManager( self.app, self.tpm ) metadata_dict, invalid_file_tups = \ irmm.generate_metadata_for_changeset_revision( repository=tool_shed_repository, @@ -533,9 +535,9 @@ set_status=True ) if 'sample_files' in metadata_dict: sample_files = metadata_dict.get( 'sample_files', [] ) - tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) + tool_index_sample_files = tdtm.get_tool_index_sample_files( sample_files ) tool_data_table_conf_filename, tool_data_table_elems = \ - tool_util.install_tool_data_tables( self.app, tool_shed_repository, tool_index_sample_files ) + tdtm.install_tool_data_tables( tool_shed_repository, tool_index_sample_files ) if tool_data_table_elems: self.app.tool_data_tables.add_new_entries_from_config_file( tool_data_table_conf_filename, None, @@ -544,16 +546,15 @@ if 'tools' in metadata_dict: tool_panel_dict = self.tpm.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) sample_files = metadata_dict.get( 'sample_files', [] ) - tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) + tool_index_sample_files = tdtm.get_tool_index_sample_files( sample_files ) tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) sample_files_copied = [ str( s ) for s in tool_index_sample_files ] repository_tools_tups = irmm.get_repository_tools_tups( metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = tool_util.handle_missing_data_table_entry( self.app, - relative_install_dir, - tool_path, - repository_tools_tups ) + repository_tools_tups = tdtm.handle_missing_data_table_entry( relative_install_dir, + tool_path, + repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( self.app, tool_path, diff -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 -r 9cce4ead10795278446af4ab84eecfc4e2b26dce lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -17,6 +17,7 @@ from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.tools import tool_panel_manager +from tool_shed.tools import data_table_manager from tool_shed.tools import tool_version_manager from tool_shed.util import basic_util @@ -440,25 +441,27 @@ else: tool_dependencies = None if 'tools' in metadata_dict: + tdtm = data_table_manager.ToolDataTableManager( self.app ) sample_files = metadata_dict.get( 'sample_files', [] ) sample_files = [ str( s ) for s in sample_files ] - tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) + tool_index_sample_files = tdtm.get_tool_index_sample_files( sample_files ) tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=self.tool_path ) sample_files_copied = [ s for s in tool_index_sample_files ] repository_tools_tups = irmm.get_repository_tools_tups( metadata_dict ) if repository_tools_tups: - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = tool_util.handle_missing_data_table_entry( self.app, - relative_install_dir, - self.tool_path, - repository_tools_tups ) + # Handle missing data table entries for tool parameters that are dynamically + # generated select lists. + repository_tools_tups = tdtm.handle_missing_data_table_entry( relative_install_dir, + self.tool_path, + repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups, sample_files_copied ) - # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + # Copy remaining sample files included in the repository to the ~/tool-data + # directory of the local Galaxy instance. tool_util.copy_sample_files( self.app, sample_files, tool_path=self.tool_path, diff -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 -r 9cce4ead10795278446af4ab84eecfc4e2b26dce lib/tool_shed/metadata/repository_metadata_manager.py --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -897,7 +897,7 @@ # revisions from the changelog. self.reset_all_tool_versions( id, repo ) # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - tool_util.reset_tool_data_tables( self.app ) + self.app.tool_data_tables.data_tables = {} return invalid_file_tups, metadata_dict def reset_all_tool_versions( self, id, repo ): @@ -1085,7 +1085,7 @@ metadata_dict ) status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - tool_util.reset_tool_data_tables( self.app ) + self.app.tool_data_tables.data_tables = {} return message, status def set_repository_metadata_due_to_new_tip( self, host, repository, content_alert_str=None, **kwd ): diff -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 -r 9cce4ead10795278446af4ab84eecfc4e2b26dce lib/tool_shed/tools/data_table_manager.py --- /dev/null +++ b/lib/tool_shed/tools/data_table_manager.py @@ -0,0 +1,142 @@ +import logging +import os +import shutil + +from tool_shed.util import hg_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import xml_util + +log = logging.getLogger( __name__ ) + + +class ToolDataTableManager( object ): + + def __init__( self, app ): + self.app = app + + def get_tool_index_sample_files( self, sample_files ): + """ + Try to return the list of all appropriate tool data sample files included + in the repository. + """ + tool_index_sample_files = [] + for s in sample_files: + # The problem with this is that Galaxy does not follow a standard naming + # convention for file names. + if s.endswith( '.loc.sample' ) or s.endswith( '.xml.sample' ) or s.endswith( '.txt.sample' ): + tool_index_sample_files.append( str( s ) ) + return tool_index_sample_files + + def handle_missing_data_table_entry( self, relative_install_dir, tool_path, repository_tools_tups ): + """ + Inspect each tool to see if any have input parameters that are dynamically + generated select lists that require entries in the tool_data_table_conf.xml + file. This method is called only from Galaxy (not the tool shed) when a + repository is being installed or reinstalled. + """ + missing_data_table_entry = False + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, guid, repository_tool = repository_tools_tup + if repository_tool.params_with_missing_data_table_entry: + missing_data_table_entry = True + break + if missing_data_table_entry: + # The repository must contain a tool_data_table_conf.xml.sample file that includes + # all required entries for all tools in the repository. + sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', + relative_install_dir ) + if sample_tool_data_table_conf: + # Add entries to the ToolDataTableManager's in-memory data_tables dictionary. + error, message = self.handle_sample_tool_data_table_conf_file( sample_tool_data_table_conf, + persist=True ) + if error: + # TODO: Do more here than logging an exception. + log.debug( message ) + # Reload the tool into the local list of repository_tools_tups. + repository_tool = self.app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) + repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) + # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. + self.reset_tool_data_tables() + return repository_tools_tups + + def handle_sample_tool_data_table_conf_file( self, filename, persist=False ): + """ + Parse the incoming filename and add new entries to the in-memory + self.app.tool_data_tables dictionary. If persist is True (should + only occur if call is from the Galaxy side, not the tool shed), the + new entries will be appended to Galaxy's shed_tool_data_table_conf.xml + file on disk. + """ + error = False + message = '' + try: + new_table_elems, message = self.app.tool_data_tables \ + .add_new_entries_from_config_file( config_filename=filename, + tool_data_path=self.app.config.tool_data_path, + shed_tool_data_table_config=self.app.config.shed_tool_data_table_config, + persist=persist ) + if message: + error = True + except Exception, e: + message = str( e ) + error = True + return error, message + + def install_tool_data_tables( self, tool_shed_repository, tool_index_sample_files ): + TOOL_DATA_TABLE_FILE_NAME = 'tool_data_table_conf.xml' + TOOL_DATA_TABLE_FILE_SAMPLE_NAME = '%s.sample' % ( TOOL_DATA_TABLE_FILE_NAME ) + SAMPLE_SUFFIX = '.sample' + SAMPLE_SUFFIX_OFFSET = -len( SAMPLE_SUFFIX ) + tool_path, relative_target_dir = tool_shed_repository.get_tool_relative_path( self.app ) + # This is where index files will reside on a per repo/installed version basis. + target_dir = os.path.join( self.app.config.shed_tool_data_path, relative_target_dir ) + if not os.path.exists( target_dir ): + os.makedirs( target_dir ) + for sample_file in tool_index_sample_files: + path, filename = os.path.split ( sample_file ) + target_filename = filename + if target_filename.endswith( SAMPLE_SUFFIX ): + target_filename = target_filename[ : SAMPLE_SUFFIX_OFFSET ] + source_file = os.path.join( tool_path, sample_file ) + # We're not currently uninstalling index files, do not overwrite existing files. + target_path_filename = os.path.join( target_dir, target_filename ) + if not os.path.exists( target_path_filename ) or target_filename == TOOL_DATA_TABLE_FILE_NAME: + shutil.copy2( source_file, target_path_filename ) + else: + log.debug( "Did not copy sample file '%s' to install directory '%s' because file already exists.", filename, target_dir ) + # For provenance and to simplify introspection, let's keep the original data table sample file around. + if filename == TOOL_DATA_TABLE_FILE_SAMPLE_NAME: + shutil.copy2( source_file, os.path.join( target_dir, filename ) ) + tool_data_table_conf_filename = os.path.join( target_dir, TOOL_DATA_TABLE_FILE_NAME ) + elems = [] + if os.path.exists( tool_data_table_conf_filename ): + tree, error_message = xml_util.parse_xml( tool_data_table_conf_filename ) + if tree: + for elem in tree.getroot(): + # Append individual table elems or other elemes, but not tables elems. + if elem.tag == 'tables': + for table_elem in elems: + elems.append( elem ) + else: + elems.append( elem ) + else: + log.debug( "The '%s' data table file was not found, but was expected to be copied from '%s' during repository installation.", + tool_data_table_conf_filename, TOOL_DATA_TABLE_FILE_SAMPLE_NAME ) + for elem in elems: + if elem.tag == 'table': + for file_elem in elem.findall( 'file' ): + path = file_elem.get( 'path', None ) + if path: + file_elem.set( 'path', os.path.normpath( os.path.join( target_dir, os.path.split( path )[1] ) ) ) + # Store repository info in the table tag set for trace-ability. + repo_elem = suc.generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=elem ) + if elems: + # Remove old data_table + os.unlink( tool_data_table_conf_filename ) + # Persist new data_table content. + self.app.tool_data_tables.to_xml_file( tool_data_table_conf_filename, elems ) + return tool_data_table_conf_filename, elems + + def reset_tool_data_tables( self ): + # Reset the tool_data_tables to an empty dictionary. + self.app.tool_data_tables.data_tables = {} diff -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 -r 9cce4ead10795278446af4ab84eecfc4e2b26dce lib/tool_shed/tools/tool_validator.py --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -6,6 +6,8 @@ from galaxy.tools import parameters from galaxy.tools.parameters import dynamic_options +from tool_shed.tools import data_table_manager + from tool_shed.util import basic_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc @@ -18,6 +20,7 @@ def __init__( self, app ): self.app = app + self.tdtm = data_table_manager.ToolDataTableManager( self.app ) def can_use_tool_config_disk_file( self, repository, repo, file_path, changeset_revision ): """ @@ -60,7 +63,8 @@ sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) if sample_tool_data_table_conf: error, correction_msg = \ - tool_util.handle_sample_tool_data_table_conf_file( self.app, sample_tool_data_table_conf ) + self.tdtm.handle_sample_tool_data_table_conf_file( sample_tool_data_table_conf, + persist=False ) if error: invalid_files_and_errors_tups.append( ( 'tool_data_table_conf.xml.sample', correction_msg ) ) else: @@ -199,7 +203,8 @@ if 'tool_data_table_conf.xml.sample' in sample_files: # Load entries into the tool_data_tables if the tool requires them. tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - error, message = tool_util.handle_sample_tool_data_table_conf_file( self.app, tool_data_table_config ) + error, message = self.tdtm.handle_sample_tool_data_table_conf_file( tool_data_table_config, + persist=False ) tool, valid, message2 = self.load_tool_from_config( repository_id, tool_config_filepath ) message = self.concat_messages( message, message2 ) return tool, valid, message, sample_files @@ -218,7 +223,8 @@ # Load entries into the tool_data_tables if the tool requires them. tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) if tool_data_table_config: - error, message = tool_util.handle_sample_tool_data_table_conf_file( self.app, tool_data_table_config ) + error, message = self.tdtm.handle_sample_tool_data_table_conf_file( tool_data_table_config, + persist=False ) if error: log.debug( message ) manifest_ctx, ctx_file = hg_util.get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) @@ -279,7 +285,7 @@ basic_util.remove_dir( work_dir ) self.app.config.tool_data_path = original_tool_data_path # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - tool_util.reset_tool_data_tables( self.app ) + self.tdtm.reset_tool_data_tables() return repository, tool, message def load_tool_from_config( self, repository_id, full_path ): diff -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 -r 9cce4ead10795278446af4ab84eecfc4e2b26dce lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -4,11 +4,15 @@ import os import shutil import tempfile + from galaxy.datatypes import checkers + +from tool_shed.tools import data_table_manager + from tool_shed.util import basic_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import tool_util + import tool_shed.repository_types.util as rt_util log = logging.getLogger( __name__ ) @@ -184,7 +188,8 @@ # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded # by parsing the file and adding new entries to the in-memory app.tool_data_tables # dictionary. - error, message = tool_util.handle_sample_tool_data_table_conf_file( app, filename_in_archive ) + tdtm = data_table_manager.ToolDataTableManager( app ) + error, message = tdtm.handle_sample_tool_data_table_conf_file( filename_in_archive, persist=False ) if error: return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed hg_util.commit_changeset( repo.ui, diff -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 -r 9cce4ead10795278446af4ab84eecfc4e2b26dce lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -11,9 +11,6 @@ from galaxy.tools.actions.upload import UploadToolAction from tool_shed.util import basic_util -from tool_shed.util import hg_util -from tool_shed.util import xml_util -import tool_shed.util.shed_util_common as suc log = logging.getLogger( __name__ ) @@ -156,45 +153,6 @@ return tool_path, relative_install_dir return None, None -def get_tool_index_sample_files( sample_files ): - """Try to return the list of all appropriate tool data sample files included in the repository.""" - tool_index_sample_files = [] - for s in sample_files: - # The problem with this is that Galaxy does not follow a standard naming convention for file names. - if s.endswith( '.loc.sample' ) or s.endswith( '.xml.sample' ) or s.endswith( '.txt.sample' ): - tool_index_sample_files.append( str( s ) ) - return tool_index_sample_files - -def handle_missing_data_table_entry( app, relative_install_dir, tool_path, repository_tools_tups ): - """ - Inspect each tool to see if any have input parameters that are dynamically - generated select lists that require entries in the tool_data_table_conf.xml - file. This method is called only from Galaxy (not the tool shed) when a - repository is being installed or reinstalled. - """ - missing_data_table_entry = False - for index, repository_tools_tup in enumerate( repository_tools_tups ): - tup_path, guid, repository_tool = repository_tools_tup - if repository_tool.params_with_missing_data_table_entry: - missing_data_table_entry = True - break - if missing_data_table_entry: - # The repository must contain a tool_data_table_conf.xml.sample file that includes - # all required entries for all tools in the repository. - sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', relative_install_dir ) - if sample_tool_data_table_conf: - # Add entries to the ToolDataTableManager's in-memory data_tables dictionary. - error, message = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf, persist=True ) - if error: - # TODO: Do more here than logging an exception. - log.debug( message ) - # Reload the tool into the local list of repository_tools_tups. - repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) - repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) - # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - reset_tool_data_tables( app ) - return repository_tools_tups - def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups, sample_files_copied ): """ Inspect each tool to see if it has any input parameters that are dynamically @@ -222,84 +180,6 @@ repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups, sample_files_copied -def handle_sample_tool_data_table_conf_file( app, filename, persist=False ): - """ - Parse the incoming filename and add new entries to the in-memory - app.tool_data_tables dictionary. If persist is True (should only occur - if call is from the Galaxy side, not the tool shed), the new entries will - be appended to Galaxy's shed_tool_data_table_conf.xml file on disk. - """ - error = False - message = '' - try: - new_table_elems, message = \ - app.tool_data_tables.add_new_entries_from_config_file( config_filename=filename, - tool_data_path=app.config.tool_data_path, - shed_tool_data_table_config=app.config.shed_tool_data_table_config, - persist=persist ) - if message: - error = True - except Exception, e: - message = str( e ) - error = True - return error, message - -def install_tool_data_tables( app, tool_shed_repository, tool_index_sample_files ): - """Only ever called from Galaxy end when installing""" - TOOL_DATA_TABLE_FILE_NAME = 'tool_data_table_conf.xml' - TOOL_DATA_TABLE_FILE_SAMPLE_NAME = '%s.sample' % ( TOOL_DATA_TABLE_FILE_NAME ) - SAMPLE_SUFFIX = '.sample' - SAMPLE_SUFFIX_OFFSET = -len( SAMPLE_SUFFIX ) - tool_path, relative_target_dir = tool_shed_repository.get_tool_relative_path( app ) - target_dir = os.path.join( app.config.shed_tool_data_path, relative_target_dir ) #this is where index files will reside on a per repo/installed version - if not os.path.exists( target_dir ): - os.makedirs( target_dir ) - for sample_file in tool_index_sample_files: - path, filename = os.path.split ( sample_file ) - target_filename = filename - if target_filename.endswith( SAMPLE_SUFFIX ): - target_filename = target_filename[ : SAMPLE_SUFFIX_OFFSET ] - source_file = os.path.join( tool_path, sample_file ) - #we're not currently uninstalling index files, do not overwrite existing files - target_path_filename = os.path.join( target_dir, target_filename ) - if not os.path.exists( target_path_filename ) or target_filename == TOOL_DATA_TABLE_FILE_NAME: - shutil.copy2( source_file, target_path_filename ) - else: - log.debug( "Did not copy sample file '%s' to install directory '%s' because file already exists.", filename, target_dir ) - #for provenance and to simplify introspection, lets keep the original data table sample file around - if filename == TOOL_DATA_TABLE_FILE_SAMPLE_NAME: - shutil.copy2( source_file, os.path.join( target_dir, filename ) ) - tool_data_table_conf_filename = os.path.join( target_dir, TOOL_DATA_TABLE_FILE_NAME ) - elems = [] - if os.path.exists( tool_data_table_conf_filename ): - tree, error_message = xml_util.parse_xml( tool_data_table_conf_filename ) - if tree: - for elem in tree.getroot(): - #append individual table elems or other elemes, but not tables elems - if elem.tag == 'tables': - for table_elem in elems: - elems.append( elem ) - else: - elems.append( elem ) - else: - log.debug( "The '%s' data table file was not found, but was expected to be copied from '%s' during repository installation.", - tool_data_table_conf_filename, TOOL_DATA_TABLE_FILE_SAMPLE_NAME ) - for elem in elems: - if elem.tag == 'table': - for file_elem in elem.findall( 'file' ): - path = file_elem.get( 'path', None ) - if path: - file_elem.set( 'path', os.path.normpath( os.path.join( target_dir, os.path.split( path )[1] ) ) ) - #store repository info in the table tagset for traceability - repo_elem = suc.generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=elem ) - if elems: - # Remove old data_table - os.unlink( tool_data_table_conf_filename ) - # Persist new data_table content. - app.tool_data_tables.to_xml_file( tool_data_table_conf_filename, elems ) - return tool_data_table_conf_filename, elems - - def is_column_based( fname, sep='\t', skip=0, is_multi_byte=False ): """See if the file is column based with respect to a separator.""" headers = get_headers( fname, sep, is_multi_byte=is_multi_byte ) @@ -384,7 +264,3 @@ tool = app.toolbox.tools_by_id[ tool_id ] if isinstance( tool.tool_action, UploadToolAction ): app.toolbox.reload_tool_by_id( tool_id ) - -def reset_tool_data_tables( app ): - # Reset the tool_data_tables to an empty dictionary. - app.tool_data_tables.data_tables = {} diff -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 -r 9cce4ead10795278446af4ab84eecfc4e2b26dce lib/tool_shed/util/workflow_util.py --- a/lib/tool_shed/util/workflow_util.py +++ b/lib/tool_shed/util/workflow_util.py @@ -16,7 +16,6 @@ from tool_shed.util import encoding_util from tool_shed.util import metadata_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import tool_util eggs.require( "SVGFig" ) import svgfig Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a DataManagerHandler for Galaxy installs from the Tool Shed, eliminating the use of the data_manager_util.py module.
by commits-noreply＠bitbucket.org 22 Jul '14

22 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b2cdaba8715e/ Changeset: b2cdaba8715e User: greg Date: 2014-07-22 19:46:40 Summary: Add a DataManagerHandler for Galaxy installs from the Tool Shed, eliminating the use of the data_manager_util.py module. Affected #: 5 files diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -12,7 +12,6 @@ import tool_shed.repository_types.util as rt_util from tool_shed.util import common_util -from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util @@ -30,6 +29,7 @@ from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import data_manager from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.tools import tool_version_manager @@ -252,7 +252,8 @@ shed_tool_conf, uninstall=remove_from_disk_checked ) if tool_shed_repository.includes_data_managers: - data_manager_util.remove_from_data_manager( trans.app, tool_shed_repository ) + dmh = data_manager.DataManagerHandler( trans.app ) + dmh.remove_from_data_manager( tool_shed_repository ) if tool_shed_repository.includes_datatypes: # Deactivate proprietary datatypes. installed_repository_dict = datatype_util.load_installed_datatypes( trans.app, @@ -1933,7 +1934,8 @@ hg_util.update_repository( repo, latest_ctx_rev ) # Remove old Data Manager entries if repository.includes_data_managers: - data_manager_util.remove_from_data_manager( trans.app, repository ) + dmh = data_manager.DataManagerHandler( trans.app ) + dmh.remove_from_data_manager( repository ) # Update the repository metadata. tpm = tool_panel_manager.ToolPanelManager( trans.app ) irmm = InstalledRepositoryMetadataManager( trans.app, tpm ) @@ -1962,13 +1964,13 @@ new_install=False ) # Add new Data Manager entries if 'data_manager' in metadata_dict: - new_data_managers = data_manager_util.install_data_managers( trans.app, - trans.app.config.shed_data_manager_config_file, - metadata_dict, - repository.get_shed_config_dict( trans.app ), - os.path.join( relative_install_dir, name ), - repository, - repository_tools_tups ) + dmh = data_manager.DataManagerHandler( trans.app ) + new_data_managers = dmh.install_data_managers( trans.app.config.shed_data_manager_config_file, + metadata_dict, + repository.get_shed_config_dict( trans.app ), + os.path.join( relative_install_dir, name ), + repository, + repository_tools_tups ) if 'repository_dependencies' in metadata_dict or 'tool_dependencies' in metadata_dict: new_repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', {} ) new_repository_dependencies = new_repository_dependencies_dict.get( 'repository_dependencies', [] ) diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -18,7 +18,6 @@ from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import encoding_util from tool_shed.util import hg_util @@ -33,6 +32,7 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager +from tool_shed.galaxy_install.tools import data_manager from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.tools import tool_version_manager @@ -575,13 +575,13 @@ tool_panel_dict=tool_panel_dict, new_install=True ) if 'data_manager' in metadata_dict: - new_data_managers = data_manager_util.install_data_managers( self.app, - self.app.config.shed_data_manager_config_file, - metadata_dict, - shed_config_dict, - relative_install_dir, - tool_shed_repository, - repository_tools_tups ) + dmh = data_manager.DataManagerHandler( self.app ) + new_data_managers = dmh.install_data_managers( self.app.config.shed_data_manager_config_file, + metadata_dict, + shed_config_dict, + relative_install_dir, + tool_shed_repository, + repository_tools_tups ) if 'datatypes' in metadata_dict: tool_shed_repository.status = self.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -7,7 +7,6 @@ from galaxy import util from tool_shed.util import common_util from tool_shed.util import container_util -from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util @@ -16,6 +15,7 @@ from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import data_manager from tool_shed.galaxy_install.tools import tool_panel_manager log = logging.getLogger( __name__ ) @@ -99,13 +99,13 @@ tp, data_manager_relative_install_dir = repository.get_tool_relative_path( self.app ) # Hack to add repository.name here, which is actually the root of the installed repository data_manager_relative_install_dir = os.path.join( data_manager_relative_install_dir, repository.name ) - new_data_managers = data_manager_util.install_data_managers( self.app, - self.app.config.shed_data_manager_config_file, - metadata, - repository.get_shed_config_dict( self.app ), - data_manager_relative_install_dir, - repository, - repository_tools_tups ) + dmh = data_manager.DataManagerHandler( self.app ) + new_data_managers = dmh.install_data_managers( self.app.config.shed_data_manager_config_file, + metadata, + repository.get_shed_config_dict( self.app ), + data_manager_relative_install_dir, + repository, + repository_tools_tups ) self.install_model.context.add( repository ) self.install_model.context.flush() if repository.includes_datatypes: diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/tool_shed/galaxy_install/tools/data_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/tools/data_manager.py @@ -0,0 +1,150 @@ +import logging +import os + +from tool_shed.galaxy_install.tools import tool_panel_manager + +from tool_shed.util import xml_util + +log = logging.getLogger( __name__ ) + + +class DataManagerHandler( object ): + + def __init__( self, app ): + self.app = app + + def data_manager_config_elems_to_xml_file( self, config_elems, config_filename ): + """ + Persist the current in-memory list of config_elems to a file named by the value + of config_filename. + """ + fh = open( config_filename, 'wb' ) + fh.write( '<?xml version="1.0"?>\n<data_managers>\n' ) + for elem in config_elems: + fh.write( xml_util.xml_to_string( elem ) ) + fh.write( '</data_managers>\n' ) + fh.close() + + def install_data_managers( self, shed_data_manager_conf_filename, metadata_dict, shed_config_dict, + relative_install_dir, repository, repository_tools_tups ): + rval = [] + if 'data_manager' in metadata_dict: + tpm = tool_panel_manager.ToolPanelManager( self.app ) + repository_tools_by_guid = {} + for tool_tup in repository_tools_tups: + repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] ) + # Load existing data managers. + tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) + if tree is None: + return rval + config_elems = [ elem for elem in tree.getroot() ] + repo_data_manager_conf_filename = metadata_dict['data_manager'].get( 'config_filename', None ) + if repo_data_manager_conf_filename is None: + log.debug( "No data_manager_conf.xml file has been defined." ) + return rval + data_manager_config_has_changes = False + relative_repo_data_manager_dir = os.path.join( shed_config_dict.get( 'tool_path', '' ), relative_install_dir ) + repo_data_manager_conf_filename = os.path.join( relative_repo_data_manager_dir, repo_data_manager_conf_filename ) + tree, error_message = xml_util.parse_xml( repo_data_manager_conf_filename ) + if tree is None: + return rval + root = tree.getroot() + for elem in root: + if elem.tag == 'data_manager': + data_manager_id = elem.get( 'id', None ) + if data_manager_id is None: + log.error( "A data manager was defined that does not have an id and will not be installed:\n%s" % \ + xml_util.xml_to_string( elem ) ) + continue + data_manager_dict = metadata_dict['data_manager'].get( 'data_managers', {} ).get( data_manager_id, None ) + if data_manager_dict is None: + log.error( "Data manager metadata is not defined properly for '%s'." % ( data_manager_id ) ) + continue + guid = data_manager_dict.get( 'guid', None ) + if guid is None: + log.error( "Data manager guid '%s' is not set in metadata for '%s'." % ( guid, data_manager_id ) ) + continue + elem.set( 'guid', guid ) + tool_guid = data_manager_dict.get( 'tool_guid', None ) + if tool_guid is None: + log.error( "Data manager tool guid '%s' is not set in metadata for '%s'." % ( tool_guid, data_manager_id ) ) + continue + tool_dict = repository_tools_by_guid.get( tool_guid, None ) + if tool_dict is None: + log.error( "Data manager tool guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % \ + ( tool_guid, data_manager_id ) ) + continue + tool = tool_dict.get( 'tool', None ) + if tool is None: + log.error( "Data manager tool with guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % \ + ( tool_guid, data_manager_id ) ) + continue + tool_config_filename = tool_dict.get( 'tool_config_filename', None ) + if tool_config_filename is None: + log.error( "Data manager metadata is missing 'tool_config_file' for '%s'." % ( data_manager_id ) ) + continue + elem.set( 'shed_conf_file', shed_config_dict['config_filename'] ) + if elem.get( 'tool_file', None ) is not None: + del elem.attrib[ 'tool_file' ] #remove old tool_file info + tool_elem = tpm.generate_tool_elem( repository.tool_shed, + repository.name, + repository.installed_changeset_revision, + repository.owner, + tool_config_filename, + tool, + None ) + elem.insert( 0, tool_elem ) + data_manager = \ + self.app.data_managers.load_manager_from_elem( elem, + tool_path=shed_config_dict.get( 'tool_path', '' ), + replace_existing=True ) + if data_manager: + rval.append( data_manager ) + else: + log.warning( "Encountered unexpected element '%s':\n%s" % ( elem.tag, xml_util.xml_to_string( elem ) ) ) + config_elems.append( elem ) + data_manager_config_has_changes = True + # Persist the altered shed_data_manager_config file. + if data_manager_config_has_changes: + self.data_manager_config_elems_to_xml_file( config_elems, shed_data_manager_conf_filename ) + return rval + + def remove_from_data_manager( self, repository ): + metadata_dict = repository.metadata + if metadata_dict and 'data_manager' in metadata_dict: + shed_data_manager_conf_filename = self.app.config.shed_data_manager_config_file + tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) + if tree: + root = tree.getroot() + assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename ) + guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ] + load_old_data_managers_by_guid = {} + data_manager_config_has_changes = False + config_elems = [] + for elem in root: + # Match Data Manager elements by guid and installed_changeset_revision + elem_matches_removed_data_manager = False + if elem.tag == 'data_manager': + guid = elem.get( 'guid', None ) + if guid in guids: + tool_elem = elem.find( 'tool' ) + if tool_elem is not None: + installed_changeset_revision_elem = tool_elem.find( 'installed_changeset_revision' ) + if installed_changeset_revision_elem is not None: + if installed_changeset_revision_elem.text == repository.installed_changeset_revision: + elem_matches_removed_data_manager = True + else: + # This is a different version, which had been previously overridden + load_old_data_managers_by_guid[ guid ] = elem + if elem_matches_removed_data_manager: + data_manager_config_has_changes = True + else: + config_elems.append( elem ) + # Remove data managers from in memory + self.app.data_managers.remove_manager( guids ) + # Load other versions of any now uninstalled data managers, if any + for elem in load_old_data_managers_by_guid.itervalues(): + self.app.data_managers.load_manager_from_elem( elem ) + # Persist the altered shed_data_manager_config file. + if data_manager_config_has_changes: + self.data_manager_config_elems_to_xml_file( config_elems, shed_data_manager_conf_filename ) diff -r 09e86deab7932a72af7717bef9ec67ea70d694c9 -r b2cdaba8715e1e2c18dab5f365fac28b6c5365a4 lib/tool_shed/util/data_manager_util.py --- a/lib/tool_shed/util/data_manager_util.py +++ /dev/null @@ -1,135 +0,0 @@ -import logging -import os - -from tool_shed.galaxy_install.tools import tool_panel_manager - -from tool_shed.util import shed_util_common as suc -from tool_shed.util import xml_util - -log = logging.getLogger( __name__ ) - -def data_manager_config_elems_to_xml_file( app, config_elems, config_filename ):#, shed_tool_conf_filename ): - # Persist the current in-memory list of config_elems to a file named by the value of config_filename. - fh = open( config_filename, 'wb' ) - fh.write( '<?xml version="1.0"?>\n<data_managers>\n' )#% ( shed_tool_conf_filename )) - for elem in config_elems: - fh.write( xml_util.xml_to_string( elem ) ) - fh.write( '</data_managers>\n' ) - fh.close() - -def install_data_managers( app, shed_data_manager_conf_filename, metadata_dict, shed_config_dict, relative_install_dir, repository, repository_tools_tups ): - rval = [] - if 'data_manager' in metadata_dict: - tpm = tool_panel_manager.ToolPanelManager( app ) - repository_tools_by_guid = {} - for tool_tup in repository_tools_tups: - repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] ) - # Load existing data managers. - tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) - if tree is None: - return rval - config_elems = [ elem for elem in tree.getroot() ] - repo_data_manager_conf_filename = metadata_dict['data_manager'].get( 'config_filename', None ) - if repo_data_manager_conf_filename is None: - log.debug( "No data_manager_conf.xml file has been defined." ) - return rval - data_manager_config_has_changes = False - relative_repo_data_manager_dir = os.path.join( shed_config_dict.get( 'tool_path', '' ), relative_install_dir ) - repo_data_manager_conf_filename = os.path.join( relative_repo_data_manager_dir, repo_data_manager_conf_filename ) - tree, error_message = xml_util.parse_xml( repo_data_manager_conf_filename ) - if tree is None: - return rval - root = tree.getroot() - for elem in root: - if elem.tag == 'data_manager': - data_manager_id = elem.get( 'id', None ) - if data_manager_id is None: - log.error( "A data manager was defined that does not have an id and will not be installed:\n%s" % ( xml_util.xml_to_string( elem ) ) ) - continue - data_manager_dict = metadata_dict['data_manager'].get( 'data_managers', {} ).get( data_manager_id, None ) - if data_manager_dict is None: - log.error( "Data manager metadata is not defined properly for '%s'." % ( data_manager_id ) ) - continue - guid = data_manager_dict.get( 'guid', None ) - if guid is None: - log.error( "Data manager guid '%s' is not set in metadata for '%s'." % ( guid, data_manager_id ) ) - continue - elem.set( 'guid', guid ) - tool_guid = data_manager_dict.get( 'tool_guid', None ) - if tool_guid is None: - log.error( "Data manager tool guid '%s' is not set in metadata for '%s'." % ( tool_guid, data_manager_id ) ) - continue - tool_dict = repository_tools_by_guid.get( tool_guid, None ) - if tool_dict is None: - log.error( "Data manager tool guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % ( tool_guid, data_manager_id ) ) - continue - tool = tool_dict.get( 'tool', None ) - if tool is None: - log.error( "Data manager tool with guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % ( tool_guid, data_manager_id ) ) - continue - tool_config_filename = tool_dict.get( 'tool_config_filename', None ) - if tool_config_filename is None: - log.error( "Data manager metadata is missing 'tool_config_file' for '%s'." % ( data_manager_id ) ) - continue - elem.set( 'shed_conf_file', shed_config_dict['config_filename'] ) - if elem.get( 'tool_file', None ) is not None: - del elem.attrib[ 'tool_file' ] #remove old tool_file info - tool_elem = tpm.generate_tool_elem( repository.tool_shed, - repository.name, - repository.installed_changeset_revision, - repository.owner, - tool_config_filename, - tool, - None ) - elem.insert( 0, tool_elem ) - data_manager = app.data_managers.load_manager_from_elem( elem, tool_path=shed_config_dict.get( 'tool_path', '' ), replace_existing=True ) - if data_manager: - rval.append( data_manager ) - else: - log.warning( "Encountered unexpected element '%s':\n%s" % ( elem.tag, xml_util.xml_to_string( elem ) ) ) - config_elems.append( elem ) - data_manager_config_has_changes = True - # Persist the altered shed_data_manager_config file. - if data_manager_config_has_changes: - data_manager_config_elems_to_xml_file( app, config_elems, shed_data_manager_conf_filename ) - return rval - -def remove_from_data_manager( app, repository ): - metadata_dict = repository.metadata - if metadata_dict and 'data_manager' in metadata_dict: - shed_data_manager_conf_filename = app.config.shed_data_manager_config_file - tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) - if tree: - root = tree.getroot() - assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename ) - guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ] - load_old_data_managers_by_guid = {} - data_manager_config_has_changes = False - config_elems = [] - for elem in root: - # Match Data Manager elements by guid and installed_changeset_revision - elem_matches_removed_data_manager = False - if elem.tag == 'data_manager': - guid = elem.get( 'guid', None ) - if guid in guids: - tool_elem = elem.find( 'tool' ) - if tool_elem is not None: - installed_changeset_revision_elem = tool_elem.find( 'installed_changeset_revision' ) - if installed_changeset_revision_elem is not None: - if installed_changeset_revision_elem.text == repository.installed_changeset_revision: - elem_matches_removed_data_manager = True - else: - # This is a different version, which had been previously overridden - load_old_data_managers_by_guid[ guid ] = elem - if elem_matches_removed_data_manager: - data_manager_config_has_changes = True - else: - config_elems.append( elem ) - # Remove data managers from in memory - app.data_managers.remove_manager( guids ) - # Load other versions of any now uninstalled data managers, if any - for elem in load_old_data_managers_by_guid.itervalues(): - app.data_managers.load_manager_from_elem( elem ) - # Persist the altered shed_data_manager_config file. - if data_manager_config_has_changes: - data_manager_config_elems_to_xml_file( app, config_elems, shed_data_manager_conf_filename ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a ToolVersionManager to manage tool versions in the Tool Shed and Galaxy installs.
by commits-noreply＠bitbucket.org 22 Jul '14

22 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/09e86deab793/ Changeset: 09e86deab793 User: greg Date: 2014-07-22 17:33:35 Summary: Add a ToolVersionManager to manage tool versions in the Tool Shed and Galaxy installs. Affected #: 10 files diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -32,6 +32,8 @@ from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tools import tool_panel_manager +from tool_shed.tools import tool_version_manager + log = logging.getLogger( __name__ ) @@ -1799,7 +1801,8 @@ text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) if text: tool_version_dicts = json.from_json_string( text ) - tool_util.handle_tool_versions( trans.app, tool_version_dicts, repository ) + tvm = tool_version_manager.ToolVersionManager( trans.app ) + tvm.handle_tool_versions( tool_version_dicts, repository ) message = "Tool versions have been set for all included tools." status = 'done' else: @@ -1932,7 +1935,8 @@ if repository.includes_data_managers: data_manager_util.remove_from_data_manager( trans.app, repository ) # Update the repository metadata. - irmm = InstalledRepositoryMetadataManager( trans.app ) + tpm = tool_panel_manager.ToolPanelManager( trans.app ) + irmm = InstalledRepositoryMetadataManager( trans.app, tpm ) metadata_dict, invalid_file_tups = \ irmm.generate_metadata_for_changeset_revision( repository=repository, changeset_revision=latest_changeset_revision, @@ -1944,7 +1948,6 @@ updating_installed_repository=True, persist=True ) if 'tools' in metadata_dict: - tpm = tool_panel_manager.ToolPanelManager( trans.app ) tool_panel_dict = metadata_dict.get( 'tool_panel_section', None ) if tool_panel_dict is None: tool_panel_dict = tpm.generate_tool_panel_dict_from_shed_tool_conf_entries( repository ) @@ -2123,7 +2126,8 @@ tool_config = os.path.join( shed_config_dict.get( 'tool_path' ), tool_config ) tool = trans.app.toolbox.load_tool( os.path.abspath( tool_config ), guid=tool_metadata[ 'guid' ] ) if tool: - tool_version = tool_util.get_tool_version( trans.app, str( tool.id ) ) + tvm = tool_version_manager.ToolVersionManager( trans.app ) + tool_version = tvm.get_tool_version( str( tool.id ) ) tool_lineage = tool_version.get_version_ids( trans.app, reverse=True ) break return trans.fill_template( "/admin/tool_shed_repository/view_tool_metadata.mako", diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -20,7 +20,9 @@ from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager + from tool_shed.tools import tool_validator +from tool_shed.tools import tool_version_manager from tool_shed.util import basic_util from tool_shed.util import common_util @@ -3432,10 +3434,10 @@ basic_util.remove_dir( work_dir ) break if guid: - tool_lineage = tool_util.get_version_lineage_for_tool( trans.app, - repository_id, - repository_metadata, - guid ) + tvm = tool_version_manager.ToolVersionManager( trans.app ) + tool_lineage = tvm.get_version_lineage_for_tool( repository_id, + repository_metadata, + guid ) else: repository_metadata_id = None metadata = None diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/tool_shed/galaxy_install/dependency_display.py --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -1,3 +1,4 @@ +import json import logging import os import threading @@ -461,7 +462,7 @@ url = common_util.url_join( tool_shed_url, 'repository/get_readme_files%s' % params ) raw_text = common_util.tool_shed_get( self.app, tool_shed_url, url ) - readme_files_dict = json.from_json_string( raw_text ) + readme_files_dict = json.loads( raw_text ) else: readme_files_dict = readme_util.build_readme_files_dict( self.app, repository, diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -35,6 +35,8 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager from tool_shed.galaxy_install.tools import tool_panel_manager +from tool_shed.tools import tool_version_manager + log = logging.getLogger( __name__ ) @@ -438,10 +440,13 @@ class InstallRepositoryManager( object ): - def __init__( self, app ): + def __init__( self, app, tpm=None ): self.app = app self.install_model = self.app.install_model - self.tpm = tool_panel_manager.ToolPanelManager( self.app ) + if tpm is None: + self.tpm = tool_panel_manager.ToolPanelManager( self.app ) + else: + self.tpm = tpm def get_repository_components_for_installation( self, encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): @@ -503,7 +508,7 @@ reinstalling an uninstalled repository. """ shed_config_dict = self.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) - irmm = InstalledRepositoryMetadataManager( self.app ) + irmm = InstalledRepositoryMetadataManager( self.app, self.tpm ) metadata_dict, invalid_file_tups = \ irmm.generate_metadata_for_changeset_revision( repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, @@ -869,7 +874,8 @@ text = common_util.tool_shed_get( self.app, tool_shed_url, url ) if text: tool_version_dicts = json.loads( text ) - tool_util.handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) + tvm = tool_version_manager.ToolVersionManager( self.app ) + tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) else: if not error_message: error_message = "" diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -81,8 +81,8 @@ repository.deleted = False repository.status = self.install_model.ToolShedRepository.installation_status.INSTALLED if repository.includes_tools_for_display_in_tool_panel: - irmm = InstalledRepositoryMetadataManager( self.app ) tpm = tool_panel_manager.ToolPanelManager( self.app ) + irmm = InstalledRepositoryMetadataManager( self.app, tpm ) metadata = repository.metadata repository_tools_tups = irmm.get_repository_tools_tups( metadata ) # Reload tools into the appropriate tool panel section. diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -19,10 +19,13 @@ class InstalledRepositoryMetadataManager( metadata_generator.MetadataGenerator ): - def __init__( self, app ): + def __init__( self, app, tpm=None ): super( InstalledRepositoryMetadataManager, self ).__init__( app ) self.app = app - self.tpm = tool_panel_manager.ToolPanelManager( self.app ) + if tpm is None: + self.tpm = tool_panel_manager.ToolPanelManager( self.app ) + else: + self.tpm = tpm def build_repository_ids_select_field( self, name='repository_ids', multiple=True, display='checkboxes' ): """Generate the current list of repositories for resetting metadata.""" diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -150,6 +150,7 @@ metadata = repository.metadata repair_dict = {} + tpm = tool_panel_manager.ToolPanelManager( self.app ) if repository.status in [ self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED ]: try: self.app.installed_repository_manager.activate_repository( repository ) @@ -162,7 +163,6 @@ suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) # Reset the repository attributes to the New state for installation. if metadata: - tpm = tool_panel_manager.ToolPanelManager( self.app ) tool_section, tool_panel_section_key = \ tpm.handle_tool_panel_selection( self.app.toolbox, metadata, @@ -179,7 +179,7 @@ deleted=False, uninstalled=False, remove_from_disk=True ) - irm = install_manager.InstallRepositoryManager( self.app ) + irm = install_manager.InstallRepositoryManager( self.app, tpm ) irm.install_tool_shed_repository( repository, repo_info_dict, tool_panel_section_key, @@ -190,7 +190,7 @@ if repository.status in [ self.app.install_model.ToolShedRepository.installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, repository.error_message ) else: - irm = install_manager.InstallRepositoryManager( self.app ) + irm = install_manager.InstallRepositoryManager( self.app, tpm ) # We have an installed tool shed repository, so handle tool dependencies if necessary. if repository.missing_tool_dependencies and metadata and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itdep" ) diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -11,11 +11,14 @@ from galaxy import util from galaxy.tools import ToolSection +from galaxy.util.odict import odict from tool_shed.galaxy_install import install_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.tools import tool_panel_manager +from tool_shed.tools import tool_version_manager + from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import datatype_util @@ -24,7 +27,6 @@ from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util -from galaxy.util.odict import odict log = logging.getLogger( __name__ ) @@ -41,6 +43,7 @@ self.app = app self.toolbox = self.app.toolbox self.migrated_tools_config = migrated_tools_config + # Initialize the ToolPanelManager. self.tpm = tool_panel_manager.ToolPanelManager( self.app ) # If install_dependencies is True but tool_dependency_dir is not set, do not attempt # to install but print informative error message. @@ -412,7 +415,7 @@ log.exception( "Exception attempting to filter and persist non-shed-related tool panel configs:\n%s" % str( e ) ) finally: lock.release() - irmm = InstalledRepositoryMetadataManager( self.app ) + irmm = InstalledRepositoryMetadataManager( self.app, self.tpm ) metadata_dict, invalid_file_tups = \ irmm.generate_metadata_for_changeset_revision( repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, @@ -471,7 +474,7 @@ new_install=True ) if install_dependencies and tool_dependencies and has_tool_dependencies: # Install tool dependencies. - irm = install_manager.InstallRepositoryManager( self.app ) + irm = install_manager.InstallRepositoryManager( self.app, self.tpm ) itdm = install_manager.InstallToolDependencyManager( self.app ) irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) @@ -533,7 +536,7 @@ if cloned_ok and is_installed: print "Skipping automatic install of repository '", tool_shed_repository.name, "' because it has already been installed in location ", clone_dir else: - irm = install_manager.InstallRepositoryManager( self.app ) + irm = install_manager.InstallRepositoryManager( self.app, self.tpm ) repository_clone_url = os.path.join( self.tool_shed_url, 'repos', tool_shed_repository.owner, tool_shed_repository.name ) relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name ) install_dir = os.path.join( clone_dir, tool_shed_repository.name ) @@ -556,6 +559,8 @@ self.app.install_model.context.refresh( tool_shed_repository ) metadata_dict = tool_shed_repository.metadata if 'tools' in metadata_dict: + # Initialize the ToolVersionManager. + tvm = tool_version_manager.ToolVersionManager( self.app ) irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) # Get the tool_versions from the tool shed for each tool in the installed change set. @@ -564,7 +569,7 @@ text = common_util.tool_shed_get( self.app, self.tool_shed_url, url ) if text: tool_version_dicts = json.loads( text ) - tool_util.handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) + tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) else: # Set the tool versions since they seem to be missing for this repository in the tool shed. # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. @@ -573,8 +578,8 @@ tool_id = tool_dict[ 'guid' ] old_tool_id = tool_dict[ 'id' ] tool_version = tool_dict[ 'version' ] - tool_version_using_old_id = tool_util.get_tool_version( self.app, old_tool_id ) - tool_version_using_guid = tool_util.get_tool_version( self.app, tool_id ) + tool_version_using_old_id = tvm.get_tool_version( old_tool_id ) + tool_version_using_guid = tvm.get_tool_version( tool_id ) if not tool_version_using_old_id: tool_version_using_old_id = self.app.install_model.ToolVersion( tool_id=old_tool_id, tool_shed_repository=tool_shed_repository ) @@ -586,12 +591,12 @@ self.app.install_model.context.add( tool_version_using_guid ) self.app.install_model.context.flush() # Associate the two versions as parent / child. - tool_version_association = tool_util.get_tool_version_association( self.app, - tool_version_using_old_id, - tool_version_using_guid ) + tool_version_association = tvm.get_tool_version_association( tool_version_using_old_id, + tool_version_using_guid ) if not tool_version_association: - tool_version_association = self.app.install_model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, - parent_id=tool_version_using_old_id.id ) + tool_version_association = \ + self.app.install_model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, + parent_id=tool_version_using_old_id.id ) self.app.install_model.context.add( tool_version_association ) self.app.install_model.context.flush() irm.update_tool_shed_repository_status( tool_shed_repository, diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/tool_shed/tools/tool_version_manager.py --- /dev/null +++ b/lib/tool_shed/tools/tool_version_manager.py @@ -0,0 +1,103 @@ +import logging +import os + +from galaxy.model.orm import and_ + +from tool_shed.util import hg_util +from tool_shed.util import shed_util_common as suc + +log = logging.getLogger( __name__ ) + + +class ToolVersionManager( object ): + + def __init__( self, app ): + self.app = app + + def get_tool_version( self, tool_id ): + context = self.app.install_model.context + return context.query( self.app.install_model.ToolVersion ) \ + .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \ + .first() + + def get_tool_version_association( self, parent_tool_version, tool_version ): + """ + Return a ToolVersionAssociation if one exists that associates the two received + tool_versions This function is called only from Galaxy. + """ + context = self.app.install_model.context + return context.query( self.app.install_model.ToolVersionAssociation ) \ + .filter( and_( self.app.install_model.ToolVersionAssociation.table.c.parent_id == parent_tool_version.id, + self.app.install_model.ToolVersionAssociation.table.c.tool_id == tool_version.id ) ) \ + .first() + + def get_version_lineage_for_tool( self, repository_id, repository_metadata, guid ): + """ + Return the tool version lineage chain in descendant order for the received + guid contained in the received repsitory_metadata.tool_versions. This function + is called only from the Tool Shed. + """ + repository = suc.get_repository_by_id( self.app, repository_id ) + repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) + # Initialize the tool lineage + version_lineage = [ guid ] + # Get all ancestor guids of the received guid. + current_child_guid = guid + for changeset in hg_util.reversed_upper_bounded_changelog( repo, repository_metadata.changeset_revision ): + ctx = repo.changectx( changeset ) + rm = suc.get_repository_metadata_by_changeset_revision( self.app, repository_id, str( ctx ) ) + if rm: + parent_guid = rm.tool_versions.get( current_child_guid, None ) + if parent_guid: + version_lineage.append( parent_guid ) + current_child_guid = parent_guid + # Get all descendant guids of the received guid. + current_parent_guid = guid + for changeset in hg_util.reversed_lower_upper_bounded_changelog( repo, + repository_metadata.changeset_revision, + repository.tip( self.app ) ): + ctx = repo.changectx( changeset ) + rm = suc.get_repository_metadata_by_changeset_revision( self.app, repository_id, str( ctx ) ) + if rm: + tool_versions = rm.tool_versions + for child_guid, parent_guid in tool_versions.items(): + if parent_guid == current_parent_guid: + version_lineage.insert( 0, child_guid ) + current_parent_guid = child_guid + break + return version_lineage + + def handle_tool_versions( self, tool_version_dicts, tool_shed_repository ): + """ + Using the list of tool_version_dicts retrieved from the Tool Shed (one per changeset + revision up to the currently installed changeset revision), create the parent / child + pairs of tool versions. Each dictionary contains { tool id : parent tool id } pairs. + This function is called only from Galaxy. + """ + context = self.app.install_model.context + for tool_version_dict in tool_version_dicts: + for tool_guid, parent_id in tool_version_dict.items(): + tool_version_using_tool_guid = self.get_tool_version( tool_guid ) + tool_version_using_parent_id = self.get_tool_version( parent_id ) + if not tool_version_using_tool_guid: + tool_version_using_tool_guid = \ + self.app.install_model.ToolVersion( tool_id=tool_guid, + tool_shed_repository=tool_shed_repository ) + context.add( tool_version_using_tool_guid ) + context.flush() + if not tool_version_using_parent_id: + tool_version_using_parent_id = \ + self.app.install_model.ToolVersion( tool_id=parent_id, + tool_shed_repository=tool_shed_repository ) + context.add( tool_version_using_parent_id ) + context.flush() + tool_version_association = \ + self.get_tool_version_association( tool_version_using_parent_id, + tool_version_using_tool_guid ) + if not tool_version_association: + # Associate the two versions as parent / child. + tool_version_association = \ + self.app.install_model.ToolVersionAssociation( tool_id=tool_version_using_tool_guid.id, + parent_id=tool_version_using_parent_id.id ) + context.add( tool_version_association ) + context.flush() diff -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 -r 09e86deab7932a72af7717bef9ec67ea70d694c9 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -1,21 +1,19 @@ import logging import os import shutil + import galaxy.tools from galaxy import util from galaxy.datatypes import checkers -from galaxy.model.orm import and_ from galaxy.tools import parameters -from galaxy.tools.search import ToolBoxSearch from galaxy.util.expressions import ExpressionContext from galaxy.web.form_builder import SelectField from galaxy.tools.actions.upload import UploadToolAction + from tool_shed.util import basic_util -from tool_shed.util import common_util from tool_shed.util import hg_util from tool_shed.util import xml_util import tool_shed.util.shed_util_common as suc -from xml.etree import ElementTree as XmlET log = logging.getLogger( __name__ ) @@ -167,55 +165,6 @@ tool_index_sample_files.append( str( s ) ) return tool_index_sample_files -def get_tool_version( app, tool_id ): - context = app.install_model.context - return context.query( app.install_model.ToolVersion ) \ - .filter( app.install_model.ToolVersion.table.c.tool_id == tool_id ) \ - .first() - -def get_tool_version_association( app, parent_tool_version, tool_version ): - """Return a ToolVersionAssociation if one exists that associates the two received tool_versions""" - context = app.install_model.context - return context.query( app.install_model.ToolVersionAssociation ) \ - .filter( and_( app.install_model.ToolVersionAssociation.table.c.parent_id == parent_tool_version.id, - app.install_model.ToolVersionAssociation.table.c.tool_id == tool_version.id ) ) \ - .first() - -def get_version_lineage_for_tool( app, repository_id, repository_metadata, guid ): - """ - Return the tool version lineage chain in descendant order for the received - guid contained in the received repsitory_metadata.tool_versions. - """ - repository = suc.get_repository_by_id( app, repository_id ) - repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - # Initialize the tool lineage - version_lineage = [ guid ] - # Get all ancestor guids of the received guid. - current_child_guid = guid - for changeset in hg_util.reversed_upper_bounded_changelog( repo, repository_metadata.changeset_revision ): - ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( app, repository_id, str( ctx ) ) - if rm: - parent_guid = rm.tool_versions.get( current_child_guid, None ) - if parent_guid: - version_lineage.append( parent_guid ) - current_child_guid = parent_guid - # Get all descendant guids of the received guid. - current_parent_guid = guid - for changeset in hg_util.reversed_lower_upper_bounded_changelog( repo, - repository_metadata.changeset_revision, - repository.tip( app ) ): - ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( app, repository_id, str( ctx ) ) - if rm: - tool_versions = rm.tool_versions - for child_guid, parent_guid in tool_versions.items(): - if parent_guid == current_parent_guid: - version_lineage.insert( 0, child_guid ) - current_parent_guid = child_guid - break - return version_lineage - def handle_missing_data_table_entry( app, relative_install_dir, tool_path, repository_tools_tups ): """ Inspect each tool to see if any have input parameters that are dynamically @@ -295,35 +244,6 @@ error = True return error, message -def handle_tool_versions( app, tool_version_dicts, tool_shed_repository ): - """ - Using the list of tool_version_dicts retrieved from the tool shed (one per changeset - revison up to the currently installed changeset revision), create the parent / child - pairs of tool versions. Each dictionary contains { tool id : parent tool id } pairs. - """ - context = app.install_model.context - for tool_version_dict in tool_version_dicts: - for tool_guid, parent_id in tool_version_dict.items(): - tool_version_using_tool_guid = get_tool_version( app, tool_guid ) - tool_version_using_parent_id = get_tool_version( app, parent_id ) - if not tool_version_using_tool_guid: - tool_version_using_tool_guid = app.install_model.ToolVersion( tool_id=tool_guid, tool_shed_repository=tool_shed_repository ) - context.add( tool_version_using_tool_guid ) - context.flush() - if not tool_version_using_parent_id: - tool_version_using_parent_id = app.install_model.ToolVersion( tool_id=parent_id, tool_shed_repository=tool_shed_repository ) - context.add( tool_version_using_parent_id ) - context.flush() - tool_version_association = get_tool_version_association( app, - tool_version_using_parent_id, - tool_version_using_tool_guid ) - if not tool_version_association: - # Associate the two versions as parent / child. - tool_version_association = app.install_model.ToolVersionAssociation( tool_id=tool_version_using_tool_guid.id, - parent_id=tool_version_using_parent_id.id ) - context.add( tool_version_association ) - context.flush() - def install_tool_data_tables( app, tool_shed_repository, tool_index_sample_files ): """Only ever called from Galaxy end when installing""" TOOL_DATA_TABLE_FILE_NAME = 'tool_data_table_conf.xml' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: davebgx: Delete images for tools that are no longer in the distribution.
by commits-noreply＠bitbucket.org 22 Jul '14

22 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/246ad6f8e6c4/ Changeset: 246ad6f8e6c4 User: davebgx Date: 2014-07-22 17:01:23 Summary: Delete images for tools that are no longer in the distribution. Affected #: 97 files diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/emboss_icons/isochore.png Binary file static/emboss_icons/isochore.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/barcode_splitter_output_example.png Binary file static/fastx_icons/barcode_splitter_output_example.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fasta_clipping_histogram_1.png Binary file static/fastx_icons/fasta_clipping_histogram_1.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fasta_clipping_histogram_2.png Binary file static/fastx_icons/fasta_clipping_histogram_2.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fasta_clipping_histogram_3.png Binary file static/fastx_icons/fasta_clipping_histogram_3.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fasta_clipping_histogram_4.png Binary file static/fastx_icons/fasta_clipping_histogram_4.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fastq_nucleotides_distribution_1.png Binary file static/fastx_icons/fastq_nucleotides_distribution_1.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fastq_nucleotides_distribution_2.png Binary file static/fastx_icons/fastq_nucleotides_distribution_2.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fastq_nucleotides_distribution_3.png Binary file static/fastx_icons/fastq_nucleotides_distribution_3.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fastq_nucleotides_distribution_4.png Binary file static/fastx_icons/fastq_nucleotides_distribution_4.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fastq_quality_boxplot_1.png Binary file static/fastx_icons/fastq_quality_boxplot_1.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fastq_quality_boxplot_2.png Binary file static/fastx_icons/fastq_quality_boxplot_2.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fastq_quality_boxplot_3.png Binary file static/fastx_icons/fastq_quality_boxplot_3.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fastx_clipper_example.png Binary file static/fastx_icons/fastx_clipper_example.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/fastx_icons/fastx_clipper_illustration.png Binary file static/fastx_icons/fastx_clipper_illustration.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/gsummary_out.gif Binary file static/images/gsummary_out.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/gsummary_out_groups.gif Binary file static/images/gsummary_out_groups.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/hilbertvis1.png Binary file static/images/hilbertvis1.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/hilbertvis2.png Binary file static/images/hilbertvis2.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/hilbertvisDiagram.png Binary file static/images/hilbertvisDiagram.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/pileup_parser_help1.png Binary file static/images/pileup_parser_help1.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/pileup_parser_help2.png Binary file static/images/pileup_parser_help2.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/pileup_parser_help3.png Binary file static/images/pileup_parser_help3.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/pileup_parser_help4.png Binary file static/images/pileup_parser_help4.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/poisson2test_eqn1.png Binary file static/images/poisson2test_eqn1.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/poisson2test_eqn2.png Binary file static/images/poisson2test_eqn2.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2ps_autoscale.png Binary file static/images/t2ps_autoscale.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2ps_autoscale_tree.png Binary file static/images/t2ps_autoscale_tree.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2ps_heatmap.png Binary file static/images/t2ps_heatmap.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2ps_ideal.png Binary file static/images/t2ps_ideal.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2ps_ideal_ssp.png Binary file static/images/t2ps_ideal_ssp.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2ps_missing_nodes.png Binary file static/images/t2ps_missing_nodes.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2ps_node_label.png Binary file static/images/t2ps_node_label.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2t_dropdown.png Binary file static/images/t2t_dropdown.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2t_exclude.png Binary file static/images/t2t_exclude.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2t_include.png Binary file static/images/t2t_include.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2t_tree_exclude.png Binary file static/images/t2t_tree_exclude.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/t2t_tree_include.png Binary file static/images/t2t_tree_include.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/images/xy_example.jpg Binary file static/images/xy_example.jpg has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/Q_plots.png Binary file static/operation_icons/Q_plots.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/cluster.png Binary file static/operation_icons/cluster.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/clusterMerge.png Binary file static/operation_icons/clusterMerge.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/complement.png Binary file static/operation_icons/complement.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/difference.png Binary file static/operation_icons/difference.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_IvC_1.png Binary file static/operation_icons/dwt_IvC_1.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_IvC_2.png Binary file static/operation_icons/dwt_IvC_2.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_IvC_3.png Binary file static/operation_icons/dwt_IvC_3.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_IvC_4.png Binary file static/operation_icons/dwt_IvC_4.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_IvC_5.png Binary file static/operation_icons/dwt_IvC_5.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVa_1.png Binary file static/operation_icons/dwt_cor_aVa_1.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVa_2.png Binary file static/operation_icons/dwt_cor_aVa_2.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVa_3.png Binary file static/operation_icons/dwt_cor_aVa_3.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVa_4.png Binary file static/operation_icons/dwt_cor_aVa_4.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVa_5.png Binary file static/operation_icons/dwt_cor_aVa_5.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_1.png Binary file static/operation_icons/dwt_cor_aVb_all_1.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_10.png Binary file static/operation_icons/dwt_cor_aVb_all_10.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_2.png Binary file static/operation_icons/dwt_cor_aVb_all_2.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_3.png Binary file static/operation_icons/dwt_cor_aVb_all_3.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_4.png Binary file static/operation_icons/dwt_cor_aVb_all_4.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_5.png Binary file static/operation_icons/dwt_cor_aVb_all_5.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_6.png Binary file static/operation_icons/dwt_cor_aVb_all_6.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_7.png Binary file static/operation_icons/dwt_cor_aVb_all_7.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_8.png Binary file static/operation_icons/dwt_cor_aVb_all_8.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_cor_aVb_all_9.png Binary file static/operation_icons/dwt_cor_aVb_all_9.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/dwt_var_perClass.png Binary file static/operation_icons/dwt_var_perClass.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/flanks_ex1.gif Binary file static/operation_icons/flanks_ex1.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/flanks_ex2.gif Binary file static/operation_icons/flanks_ex2.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_baseCoverage.gif Binary file static/operation_icons/gops_baseCoverage.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_cluster.gif Binary file static/operation_icons/gops_cluster.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_clusterFind.gif Binary file static/operation_icons/gops_clusterFind.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_clusterMerge.gif Binary file static/operation_icons/gops_clusterMerge.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_complement.gif Binary file static/operation_icons/gops_complement.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_concatenate.gif Binary file static/operation_icons/gops_concatenate.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_intersect.gif Binary file static/operation_icons/gops_intersect.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_intersectOverlappingIntervals.gif Binary file static/operation_icons/gops_intersectOverlappingIntervals.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_intersectOverlappingPieces.gif Binary file static/operation_icons/gops_intersectOverlappingPieces.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_joinFullOuter.gif Binary file static/operation_icons/gops_joinFullOuter.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_joinInner.gif Binary file static/operation_icons/gops_joinInner.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_joinLeftOuter.gif Binary file static/operation_icons/gops_joinLeftOuter.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_joinRecordsList.gif Binary file static/operation_icons/gops_joinRecordsList.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_joinRightOuter.gif Binary file static/operation_icons/gops_joinRightOuter.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_merge.gif Binary file static/operation_icons/gops_merge.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_subtract.gif Binary file static/operation_icons/gops_subtract.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_subtractOverlappingIntervals.gif Binary file static/operation_icons/gops_subtractOverlappingIntervals.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/gops_subtractOverlappingPieces.gif Binary file static/operation_icons/gops_subtractOverlappingPieces.gif has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/intersect.png Binary file static/operation_icons/intersect.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/merge.png Binary file static/operation_icons/merge.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/overlap.png Binary file static/operation_icons/overlap.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/p_hist.png Binary file static/operation_icons/p_hist.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/proximityEx.png Binary file static/operation_icons/proximityEx.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/proximityInc.png Binary file static/operation_icons/proximityInc.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/q_hist.png Binary file static/operation_icons/q_hist.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/stacked_bars_plot.png Binary file static/operation_icons/stacked_bars_plot.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/subtract.png Binary file static/operation_icons/subtract.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/union.png Binary file static/operation_icons/union.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/vicinityEnd.png Binary file static/operation_icons/vicinityEnd.png has changed diff -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac -r 246ad6f8e6c43753a8d04ffda2f5f5f019e86fc9 static/operation_icons/vicinityStart.png Binary file static/operation_icons/vicinityStart.png has changed Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Move some functions out of the tool shed's shed_util_common module into appropriate classes.
by commits-noreply＠bitbucket.org 21 Jul '14

21 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/55b4556e0b7e/ Changeset: 55b4556e0b7e User: greg Date: 2014-07-21 22:44:32 Summary: Move some functions out of the tool shed's shed_util_common module into appropriate classes. Affected #: 16 files diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -13,6 +13,7 @@ from tool_shed.galaxy_install.install_manager import InstallRepositoryManager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager + from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -46,7 +46,7 @@ def activate_repository( self, trans, **kwd ): """Activate a repository that was deactivated but not uninstalled.""" repository_id = kwd[ 'id' ] - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) try: trans.app.installed_repository_manager.activate_repository( repository ) except Exception, e: @@ -72,7 +72,7 @@ def browse_repository( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) return trans.fill_template( '/admin/tool_shed_repository/browse_repository.mako', repository=repository, message=message, @@ -105,7 +105,7 @@ action='purge_repository', **kwd ) ) if operation == "activate or reinstall": - repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if repository.uninstalled: # Since we're reinstalling the repository we need to find the latest changeset revision to which it can # be updated so that we can reset the metadata if necessary. This will ensure that information about @@ -206,7 +206,7 @@ def check_for_updates( self, trans, **kwd ): """Send a request to the relevant tool shed to see if there are any updates.""" repository_id = kwd.get( 'id', None ) - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = '?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s' % \ ( web.url_for( '/', qualified=True ), @@ -232,7 +232,7 @@ status = kwd.get( 'status', 'done' ) remove_from_disk = kwd.get( 'remove_from_disk', '' ) remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) shed_tool_conf, tool_path, relative_install_dir = \ suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) if relative_install_dir: @@ -394,7 +394,7 @@ of the installed tool shed repository in Galaxy. We need it so that we can derive the tool shed from which it was installed. """ - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) if tool_shed_url is None or repository_name is None or repository_owner is None or changeset_revision is None: message = "Unable to retrieve tool dependencies from the Tool Shed because one or more of the following required " @@ -419,7 +419,7 @@ Send a request to the appropriate tool shed to retrieve the dictionary of information required to reinstall an updated revision of an uninstalled tool shed repository. """ - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) if tool_shed_url is None or repository_name is None or repository_owner is None or changeset_revision is None: message = "Unable to retrieve updated repository information from the Tool Shed because one or more of the following " @@ -513,7 +513,7 @@ status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) if repository_id is not None: - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) if repository is not None: tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) name = str( repository.name ) @@ -571,7 +571,7 @@ dependencies are included in the updated revision. """ updating_repository_id = kwd.get( 'updating_repository_id', None ) - repository = suc.get_installed_tool_shed_repository( trans.app, updating_repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, updating_repository_id ) # All received dependencies need to be installed - confirmed by the caller. encoded_tool_dependencies_dict = kwd.get( 'encoded_tool_dependencies_dict', None ) if encoded_tool_dependencies_dict is not None: @@ -747,7 +747,7 @@ if repository_id is None: return trans.show_error_message( 'Missing required encoded repository id.' ) operation = kwd.get( 'operation', None ) - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) if repository is None: return trans.show_error_message( 'Invalid repository specified.' ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) @@ -866,15 +866,15 @@ message += ' and restart your Galaxy server to install tool dependencies.' status = 'error' installed_tool_dependencies_select_field = \ - suc.build_tool_dependencies_select_field( trans.app, - tool_shed_repository=tool_shed_repository, - name='inst_td_ids', - uninstalled_only=False ) + tool_dependency_util.build_tool_dependencies_select_field( trans.app, + tool_shed_repository=tool_shed_repository, + name='inst_td_ids', + uninstalled_only=False ) uninstalled_tool_dependencies_select_field = \ - suc.build_tool_dependencies_select_field( trans.app, - tool_shed_repository=tool_shed_repository, - name='uninstalled_tool_dependency_ids', - uninstalled_only=True ) + tool_dependency_util.build_tool_dependencies_select_field( trans.app, + tool_shed_repository=tool_shed_repository, + name='uninstalled_tool_dependency_ids', + uninstalled_only=True ) return trans.fill_template( '/admin/tool_shed_repository/manage_repository_tool_dependencies.mako', repository=tool_shed_repository, installed_tool_dependencies_select_field=installed_tool_dependencies_select_field, @@ -1276,7 +1276,7 @@ repository_id = kwd.get( 'id', None ) new_kwd = {} if repository_id is not None: - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) if repository: if repository.is_new: if kwd.get( 'purge_repository_button', False ): @@ -1314,7 +1314,7 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository_id = kwd[ 'id' ] - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) no_changes = kwd.get( 'no_changes', '' ) no_changes_checked = CheckboxField.is_checked( no_changes ) install_repository_dependencies = CheckboxField.is_checked( kwd.get( 'install_repository_dependencies', '' ) ) @@ -1457,7 +1457,7 @@ action='browse_repositories', message=message, status=status ) ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) rrm = RepairRepositoryManager( trans.app ) if kwd.get( 'repair_repository_button', False ): encoded_repair_dict = kwd.get( 'repair_dict', None ) @@ -1475,7 +1475,7 @@ repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) repositories_for_repair.append( repository ) return self.repair_tool_shed_repositories( trans, rrm, repositories_for_repair, ordered_repo_info_dicts ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) repair_dict = rrm.get_repair_dict( tool_shed_repository ) encoded_repair_dict = encoding_util.tool_shed_encode( repair_dict ) ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) @@ -1537,7 +1537,7 @@ repository_id = kwd.get( 'id', None ) latest_changeset_revision = kwd.get( 'latest_changeset_revision', None ) latest_ctx_rev = kwd.get( 'latest_ctx_rev', None ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( trans.app, tool_shed_repository ) metadata = tool_shed_repository.metadata tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( tool_shed_repository.tool_shed ) ) @@ -1724,7 +1724,7 @@ @web.require_admin def reset_repository_metadata( self, trans, id ): """Reset all metadata on a single installed tool shed repository.""" - repository = suc.get_installed_tool_shed_repository( trans.app, id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( trans.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: @@ -1742,7 +1742,7 @@ persist=False ) repository.metadata = metadata_dict if metadata_dict != original_metadata_dict: - suc.update_in_shed_tool_config( trans.app, repository ) + irmm.update_in_shed_tool_config( repository ) trans.install_model.context.add( repository ) trans.install_model.context.flush() message = 'Metadata has been reset on repository <b>%s</b>.' % repository.name @@ -1763,7 +1763,7 @@ @web.require_admin def reset_to_install( self, trans, **kwd ): """An error occurred while cloning the repository, so reset everything necessary to enable another attempt.""" - repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if kwd.get( 'reset_repository', False ): suc.set_repository_attributes( trans.app, repository, @@ -1789,7 +1789,7 @@ Get the tool_versions from the tool shed for each tool in the installed revision of a selected tool shed repository and update the metadata for the repository's revision in the Galaxy database. """ - repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ), str( repository.owner ), @@ -1947,8 +1947,8 @@ tpm = tool_panel_manager.ToolPanelManager( trans.app ) tool_panel_dict = metadata_dict.get( 'tool_panel_section', None ) if tool_panel_dict is None: - tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( trans.app, repository ) - repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) + tool_panel_dict = tpm.generate_tool_panel_dict_from_shed_tool_conf_entries( repository ) + repository_tools_tups = irmm.get_repository_tools_tups( metadata_dict ) tpm.add_to_tool_panel( repository_name=str( repository.name ), repository_clone_url=repository_clone_url, changeset_revision=str( repository.installed_changeset_revision ), @@ -2069,8 +2069,10 @@ repository_names_not_updated = [] updated_count = 0 for repository in trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .filter( trans.install_model.ToolShedRepository.table.c.deleted == False ): - ok, updated = suc.check_or_update_tool_shed_status_for_installed_repository( trans.app, repository ) + .filter( trans.install_model.ToolShedRepository.table.c.deleted == False ): + ok, updated = \ + repository_maintenance_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, + repository ) if ok: success_count += 1 else: @@ -2085,7 +2087,9 @@ else: repository_id = kwd.get( 'id', None ) repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id ) - ok, updated = suc.check_or_update_tool_shed_status_for_installed_repository( trans.app, repository ) + ok, updated = \ + repository_maintenance_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, + repository ) if ok: if updated: message = "The tool shed status for repository <b>%s</b> has been updated." % str( repository.name ) @@ -2104,7 +2108,7 @@ def view_tool_metadata( self, trans, repository_id, tool_id, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) repository_metadata = repository.metadata shed_config_dict = repository.get_shed_config_dict( trans.app ) tool_metadata = {} diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2755,7 +2755,7 @@ # This method is called only from the ~/templates/webapps/tool_shed/repository/manage_repository.mako template. rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) invalid_file_tups, metadata_dict = \ - rmm.reset_all_metadata_on_repository_in_tool_shed( id, **kwd ) + rmm.reset_all_metadata_on_repository_in_tool_shed( id ) if invalid_file_tups: repository = suc.get_repository_in_tool_shed( trans.app, id ) message = tool_util.generate_message_for_invalid_tools( trans.app, diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -42,6 +42,7 @@ def __init__( self, app ): self.app = app + self.install_model = self.app.install_model self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', 'setup_perl_environment', 'setup_python_environment', 'setup_r_environment', 'setup_ruby_environment', 'shell_command' ] @@ -118,7 +119,7 @@ work_dir=work_dir, current_dir=current_dir, initial_download=False ) - if tool_dependency.status in [ self.app.install_model.ToolDependency.installation_status.ERROR ]: + if tool_dependency.status in [ self.install_model.ToolDependency.installation_status.ERROR ]: # If the tool_dependency status is in an error state, return it with no additional # processing. return tool_dependency @@ -130,7 +131,6 @@ return tool_dependency def install_and_build_package_via_fabric( self, tool_shed_repository, tool_dependency, actions_dict ): - sa_session = self.app.install_model.context try: # There is currently only one fabric method. tool_dependency = self.install_and_build_package( tool_shed_repository, tool_dependency, actions_dict ) @@ -221,8 +221,8 @@ tool_dependency, error_message, remove_installation_path=False ) - if tool_dependency and tool_dependency.status in [ self.app.install_model.ToolDependency.installation_status.INSTALLED, - self.app.install_model.ToolDependency.installation_status.ERROR ]: + if tool_dependency and tool_dependency.status in [ self.install_model.ToolDependency.installation_status.INSTALLED, + self.install_model.ToolDependency.installation_status.ERROR ]: installed_packages.append( tool_dependency ) if self.app.config.manage_dependency_relationships: # Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager. @@ -237,7 +237,6 @@ self.install_and_build_package(). The use of fabric is being eliminated, so some of these functions may need to be renamed at some point. """ - sa_session = self.app.install_model.context if not os.path.exists( install_dir ): os.makedirs( install_dir ) actions_dict = dict( install_dir=install_dir ) @@ -353,7 +352,7 @@ package_name=package_name, actions_elem=actions_elem, action_elem=None ) - if tool_dependency.status == self.app.install_model.ToolDependency.installation_status.INSTALLED: + if tool_dependency.status == self.install_model.ToolDependency.installation_status.INSTALLED: # If an <actions> tag was found that matches the current platform, and # self.install_via_fabric() did not result in an error state, set binary_installed # to True in order to skip any remaining platform-specific <actions> tags. @@ -396,7 +395,7 @@ actions_elem=actions_elem, action_elem=None ) if actions_elem.tag == 'action' and \ - tool_dependency.status != self.app.install_model.ToolDependency.installation_status.ERROR: + tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: # If the tool dependency is not in an error state, perform any final actions that have been # defined within the actions_group tag set, but outside of an <actions> tag, which defines # the recipe for installing and compiling from source. @@ -416,19 +415,19 @@ package_name=package_name, actions_elem=actions_elems, action_elem=None ) - if tool_dependency.status != self.app.install_model.ToolDependency.installation_status.ERROR: + if tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: log.debug( 'Tool dependency %s version %s has been installed in %s.' % \ ( str( package_name ), str( package_version ), str( install_dir ) ) ) return tool_dependency def mark_tool_dependency_installed( self, tool_dependency ): - if tool_dependency.status not in [ self.app.install_model.ToolDependency.installation_status.ERROR, - self.app.install_model.ToolDependency.installation_status.INSTALLED ]: + if tool_dependency.status not in [ self.install_model.ToolDependency.installation_status.ERROR, + self.install_model.ToolDependency.installation_status.INSTALLED ]: log.debug( 'Changing status for tool dependency %s from %s to %s.' % \ ( str( tool_dependency.name ), str( tool_dependency.status ), - str( self.app.install_model.ToolDependency.installation_status.INSTALLED ) ) ) - status = self.app.install_model.ToolDependency.installation_status.INSTALLED + str( self.install_model.ToolDependency.installation_status.INSTALLED ) ) ) + status = self.install_model.ToolDependency.installation_status.INSTALLED tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=status, @@ -441,6 +440,8 @@ def __init__( self, app ): self.app = app + self.install_model = self.app.install_model + self.tpm = tool_panel_manager.ToolPanelManager( self.app ) def get_repository_components_for_installation( self, encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): @@ -497,10 +498,10 @@ def handle_repository_contents( self, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ): """ - Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy - (never the tool shed) when an administrator is installing a new repository or reinstalling an uninstalled repository. + Generate the metadata for the installed tool shed repository, among other things. + This method is called when an administrator is installing a new repository or + reinstalling an uninstalled repository. """ - install_model = self.app.install_model shed_config_dict = self.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) irmm = InstalledRepositoryMetadataManager( self.app ) metadata_dict, invalid_file_tups = \ @@ -518,8 +519,8 @@ tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, tool_shed_repository ) if tool_shed_status_dict: tool_shed_repository.tool_shed_status = tool_shed_status_dict - install_model.context.add( tool_shed_repository ) - install_model.context.flush() + self.install_model.context.add( tool_shed_repository ) + self.install_model.context.flush() if 'tool_dependencies' in metadata_dict and not reinstalling: tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, tool_shed_repository, @@ -536,13 +537,12 @@ self.app.config.shed_tool_data_table_config, persist=True ) if 'tools' in metadata_dict: - tpm = tool_panel_manager.ToolPanelManager( self.app ) - tool_panel_dict = tpm.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) + tool_panel_dict = self.tpm.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) sample_files = metadata_dict.get( 'sample_files', [] ) tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) sample_files_copied = [ str( s ) for s in tool_index_sample_files ] - repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict ) + repository_tools_tups = irmm.get_repository_tools_tups( metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = tool_util.handle_missing_data_table_entry( self.app, @@ -561,14 +561,14 @@ sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) - tpm.add_to_tool_panel( repository_name=tool_shed_repository.name, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - repository_tools_tups=repository_tools_tups, - owner=tool_shed_repository.owner, - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=True ) + self.tpm.add_to_tool_panel( repository_name=tool_shed_repository.name, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + repository_tools_tups=repository_tools_tups, + owner=tool_shed_repository.owner, + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=True ) if 'data_manager' in metadata_dict: new_data_managers = data_manager_util.install_data_managers( self.app, self.app.config.shed_data_manager_config_file, @@ -578,11 +578,11 @@ tool_shed_repository, repository_tools_tups ) if 'datatypes' in metadata_dict: - tool_shed_repository.status = install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES + tool_shed_repository.status = self.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True - install_model.context.add( tool_shed_repository ) - install_model.context.flush() + self.install_model.context.add( tool_shed_repository ) + self.install_model.context.flush() files_dir = relative_install_dir if shed_config_dict.get( 'tool_path' ): files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) @@ -628,7 +628,6 @@ return created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts def initiate_repository_installation( self, installation_dict ): - install_model = self.app.install_model # The following installation_dict entries are all required. created_or_updated_tool_shed_repositories = installation_dict[ 'created_or_updated_tool_shed_repositories' ] filtered_repo_info_dicts = installation_dict[ 'filtered_repo_info_dicts' ] @@ -648,11 +647,10 @@ tool_shed_url = installation_dict[ 'tool_shed_url' ] # Handle contained tools. if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section_label or tool_panel_section_id ): - tpm = tool_panel_manager.ToolPanelManager( self.app ) tool_panel_section_key, tool_section = \ - tpm.handle_tool_panel_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + self.tpm.handle_tool_panel_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) else: tool_panel_section_key = None tool_section = None @@ -675,12 +673,12 @@ tsr_ids = [ r.id for r in created_or_updated_tool_shed_repositories ] tool_shed_repositories = [] for tsr_id in tsr_ids: - tsr = install_model.context.query( install_model.ToolShedRepository ).get( tsr_id ) + tsr = self.install_model.context.query( self.install_model.ToolShedRepository ).get( tsr_id ) tool_shed_repositories.append( tsr ) clause_list = [] for tsr_id in tsr_ids: - clause_list.append( install_model.ToolShedRepository.table.c.id == tsr_id ) - query = install_model.context.query( install_model.ToolShedRepository ).filter( or_( *clause_list ) ) + clause_list.append( self.install_model.ToolShedRepository.table.c.id == tsr_id ) + query = self.install_model.context.query( self.install_model.ToolShedRepository ).filter( or_( *clause_list ) ) return encoded_kwd, query, tool_shed_repositories, encoded_repository_ids def install( self, tool_shed_url, name, owner, changeset_revision, install_options ): @@ -726,7 +724,7 @@ shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: # Get the tool_path setting. - index, shed_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. @@ -792,11 +790,10 @@ self.order_components_for_installation( tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) # Install the repositories, keeping track of each one for later display. for index, tsr_id in enumerate( ordered_tsr_ids ): - install_model = self.app.install_model - tool_shed_repository = install_model.context.query( install_model.ToolShedRepository ) \ - .get( self.app.security.decode_id( tsr_id ) ) - if tool_shed_repository.status in [ install_model.ToolShedRepository.installation_status.NEW, - install_model.ToolShedRepository.installation_status.UNINSTALLED ]: + tool_shed_repository = self.install_model.context.query( self.install_model.ToolShedRepository ) \ + .get( self.app.security.decode_id( tsr_id ) ) + if tool_shed_repository.status in [ self.install_model.ToolShedRepository.installation_status.NEW, + self.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: repo_info_dict = ordered_repo_info_dicts[ index ] tool_panel_section_key = ordered_tool_panel_section_keys[ index ] self.install_tool_shed_repository( tool_shed_repository, @@ -814,7 +811,6 @@ def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies, reinstalling=False ): - install_model = self.app.install_model if tool_panel_section_key: try: tool_section = self.app.toolbox.tool_panel[ tool_panel_section_key ] @@ -827,9 +823,8 @@ if isinstance( repo_info_dict, basestring ): repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict ) # Clone each repository to the configured location. - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, - install_model.ToolShedRepository.installation_status.CLONING ) + self.update_tool_shed_repository_status( tool_shed_repository, + self.install_model.ToolShedRepository.installation_status.CLONING ) repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, @@ -859,13 +854,12 @@ tool_section=tool_section, shed_tool_conf=shed_tool_conf, reinstalling=reinstalling ) - install_model.context.refresh( tool_shed_repository ) + self.install_model.context.refresh( tool_shed_repository ) metadata = tool_shed_repository.metadata if 'tools' in metadata: # Get the tool_versions from the tool shed for each tool in the installed change set. - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, - install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) + self.update_tool_shed_repository_status( tool_shed_repository, + self.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, str( tool_shed_repository.tool_shed ) ) params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( tool_shed_repository.name ), str( tool_shed_repository.owner ), @@ -885,9 +879,8 @@ if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" ) # Install tool dependencies. - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, - install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + self.update_tool_shed_repository_status( tool_shed_repository, + self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', install_dir ) itdm = InstallToolDependencyManager( self.app ) @@ -896,9 +889,8 @@ tool_dependencies=tool_shed_repository.tool_dependencies, from_tool_migration_manager=False ) basic_util.remove_dir( work_dir ) - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, - install_model.ToolShedRepository.installation_status.INSTALLED ) + self.update_tool_shed_repository_status( tool_shed_repository, + self.install_model.ToolShedRepository.installation_status.INSTALLED ) if self.app.config.manage_dependency_relationships: # Add the installed repository and any tool dependencies to the in-memory dictionaries # in the installed_repository_manager. @@ -907,7 +899,7 @@ # An error occurred while cloning the repository, so reset everything necessary to enable another attempt. suc.set_repository_attributes( self.app, tool_shed_repository, - status=install_model.ToolShedRepository.installation_status.ERROR, + status=self.install_model.ToolShedRepository.installation_status.ERROR, error_message=error_message, deleted=False, uninstalled=False, @@ -962,3 +954,13 @@ ordered_repo_info_dicts.append( repo_info_dict ) ordered_tool_panel_section_keys.append( tool_panel_section_key ) return ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys + + def update_tool_shed_repository_status( self, tool_shed_repository, status, error_message=None ): + """ + Update the status of a tool shed repository in the process of being installed into Galaxy. + """ + tool_shed_repository.status = status + if error_message: + tool_shed_repository.error_message = str( error_message ) + self.install_model.context.add( tool_shed_repository ) + self.install_model.context.flush() diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -14,6 +14,7 @@ from tool_shed.util import xml_util from galaxy.model.orm import and_ +from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tools import tool_panel_manager @@ -80,9 +81,10 @@ repository.deleted = False repository.status = self.install_model.ToolShedRepository.installation_status.INSTALLED if repository.includes_tools_for_display_in_tool_panel: + irmm = InstalledRepositoryMetadataManager( self.app ) tpm = tool_panel_manager.ToolPanelManager( self.app ) metadata = repository.metadata - repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata ) + repository_tools_tups = irmm.get_repository_tools_tups( metadata ) # Reload tools into the appropriate tool panel section. tool_panel_dict = repository.metadata[ 'tool_panel_section' ] tpm.add_to_tool_panel( repository.name, diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -5,11 +5,14 @@ from galaxy.util import inflector from galaxy.web.form_builder import SelectField +from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.metadata import metadata_generator from tool_shed.util import common_util +from tool_shed.util import repository_maintenance_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util +from tool_shed.util import xml_util log = logging.getLogger( __name__ ) @@ -19,6 +22,7 @@ def __init__( self, app ): super( InstalledRepositoryMetadataManager, self ).__init__( app ) self.app = app + self.tpm = tool_panel_manager.ToolPanelManager( self.app ) def build_repository_ids_select_field( self, name='repository_ids', multiple=True, display='checkboxes' ): """Generate the current list of repositories for resetting metadata.""" @@ -46,11 +50,32 @@ return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ .filter( self.app.install_model.ToolShedRepository.table.c.uninstalled == False ) + def get_repository_tools_tups( self, metadata_dict ): + """ + Return a list of tuples of the form (relative_path, guid, tool) for each tool defined + in the received tool shed repository metadata. + """ + repository_tools_tups = [] + index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( metadata_dict.get( 'shed_config_filename' ) ) + if 'tools' in metadata_dict: + for tool_dict in metadata_dict[ 'tools' ]: + load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) + if shed_conf_dict.get( 'tool_path' ): + load_relative_path = os.path.join( shed_conf_dict.get( 'tool_path' ), relative_path ) + guid = tool_dict.get( 'guid', None ) + if relative_path and guid: + tool = self.app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid ) + else: + tool = None + if tool: + repository_tools_tups.append( ( relative_path, guid, tool ) ) + return repository_tools_tups + def reset_all_metadata_on_installed_repository( self, id ): """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" invalid_file_tups = [] metadata_dict = {} - repository = suc.get_installed_tool_shed_repository( self.app, id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( self.app, id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( self.app ) if relative_install_dir: @@ -67,7 +92,7 @@ persist=False ) repository.metadata = metadata_dict if metadata_dict != original_metadata_dict: - suc.update_in_shed_tool_config( self.app, repository ) + self.update_in_shed_tool_config( repository ) self.app.install_model.context.add( repository ) self.app.install_model.context.flush() log.debug( 'Metadata has been reset on repository %s.' % repository.name ) @@ -90,7 +115,7 @@ unsuccessful_count = 0 for repository_id in repository_ids: try: - repository = suc.get_installed_tool_shed_repository( self.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( self.app, repository_id ) owner = str( repository.owner ) invalid_file_tups, metadata_dict = \ self.reset_all_metadata_on_installed_repository( repository_id ) @@ -117,3 +142,49 @@ message = 'Select at least one repository to on which to reset all metadata.' status = 'error' return message, status + + def tool_shed_from_repository_clone_url( self, repository_clone_url ): + """Given a repository clone URL, return the tool shed that contains the repository.""" + return common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ).split( '/repos/' )[ 0 ].rstrip( '/' ) + + def update_in_shed_tool_config( self, repository ): + """ + A tool shed repository is being updated so change the shed_tool_conf file. Parse the config + file to generate the entire list of config_elems instead of using the in-memory list. + """ + shed_conf_dict = repository.get_shed_config_dict( self.app ) + shed_tool_conf = shed_conf_dict[ 'config_filename' ] + tool_path = shed_conf_dict[ 'tool_path' ] + tool_panel_dict = self.tpm.generate_tool_panel_dict_from_shed_tool_conf_entries( repository ) + repository_tools_tups = self.get_repository_tools_tups( repository.metadata ) + clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) + cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( clone_url ) + tool_shed = self.tool_shed_from_repository_clone_url( cleaned_repository_clone_url ) + owner = repository.owner + if not owner: + owner = suc.get_repository_owner( cleaned_repository_clone_url ) + guid_to_tool_elem_dict = {} + for tool_config_filename, guid, tool in repository_tools_tups: + guid_to_tool_elem_dict[ guid ] = self.tpm.generate_tool_elem( tool_shed, + repository.name, + repository.changeset_revision, + repository.owner or '', + tool_config_filename, + tool, + None ) + config_elems = [] + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + if elem.tag == 'section': + for i, tool_elem in enumerate( elem ): + guid = tool_elem.attrib.get( 'guid' ) + if guid in guid_to_tool_elem_dict: + elem[i] = guid_to_tool_elem_dict[ guid ] + elif elem.tag == 'tool': + guid = elem.attrib.get( 'guid' ) + if guid in guid_to_tool_elem_dict: + elem = guid_to_tool_elem_dict[ guid ] + config_elems.append( elem ) + self.tpm.config_elems_to_xml_file( config_elems, shed_tool_conf, tool_path ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -190,6 +190,7 @@ if repository.status in [ self.app.install_model.ToolShedRepository.installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, repository.error_message ) else: + irm = install_manager.InstallRepositoryManager( self.app ) # We have an installed tool shed repository, so handle tool dependencies if necessary. if repository.missing_tool_dependencies and metadata and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itdep" ) @@ -204,8 +205,7 @@ error_message=None, remove_from_disk=True ) # Install tool dependencies. - suc.update_tool_shed_repository_status( self.app, - repository, + irm.update_tool_shed_repository_status( repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path( self.app ) ) @@ -218,5 +218,6 @@ if installed_tool_dependency.status in [ self.app.install_model.ToolDependency.installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message ) basic_util.remove_dir( work_dir ) - suc.update_tool_shed_repository_status( self.app, repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLED ) + irm.update_tool_shed_repository_status( repository, + self.app.install_model.ToolShedRepository.installation_status.INSTALLED ) return repair_dict \ No newline at end of file diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -206,7 +206,7 @@ # Set changeset_revision here so suc.create_or_update_tool_shed_repository will find # the previously installed and uninstalled repository instead of creating a new record. changeset_revision = repository_db_record.installed_changeset_revision - suc.reset_previously_installed_repository( install_model, repository_db_record ) + self.reset_previously_installed_repository( repository_db_record ) can_update_db_record = True else: # No record exists in the database for the repository currently being processed. @@ -409,3 +409,26 @@ return False return True return False + + def reset_previously_installed_repository( self, repository ): + """ + Reset the attributes of a tool_shed_repository that was previously installed. + The repository will be in some state other than INSTALLED, so all attributes + will be set to the default NEW state. This will enable the repository to be + freshly installed. + """ + debug_msg = "Resetting tool_shed_repository '%s' for installation.\n" % str( repository.name ) + debug_msg += "The current state of the tool_shed_repository is:\n" + debug_msg += "deleted: %s\n" % str( repository.deleted ) + debug_msg += "tool_shed_status: %s\n" % str( repository.tool_shed_status ) + debug_msg += "uninstalled: %s\n" % str( repository.uninstalled ) + debug_msg += "status: %s\n" % str( repository.status ) + debug_msg += "error_message: %s\n" % str( repository.error_message ) + log.debug( debug_msg ) + repository.deleted = False + repository.tool_shed_status = None + repository.uninstalled = False + repository.status = self.app.install_model.ToolShedRepository.installation_status.NEW + repository.error_message = None + self.app.install_model.context.add( repository ) + self.app.install_model.context.flush() diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -74,7 +74,7 @@ self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url ) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER - index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config ) + index, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed @@ -442,7 +442,7 @@ tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=self.tool_path ) sample_files_copied = [ s for s in tool_index_sample_files ] - repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict ) + repository_tools_tups = irmm.get_repository_tools_tups( metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = tool_util.handle_missing_data_table_entry( self.app, @@ -471,12 +471,12 @@ new_install=True ) if install_dependencies and tool_dependencies and has_tool_dependencies: # Install tool dependencies. - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, + irm = install_manager.InstallRepositoryManager( self.app ) + itdm = install_manager.InstallToolDependencyManager( self.app ) + irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from disk. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir ) - itdm = install_manager.InstallToolDependencyManager( self.app ) installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=tool_dependencies, @@ -533,6 +533,7 @@ if cloned_ok and is_installed: print "Skipping automatic install of repository '", tool_shed_repository.name, "' because it has already been installed in location ", clone_dir else: + irm = install_manager.InstallRepositoryManager( self.app ) repository_clone_url = os.path.join( self.tool_shed_url, 'repos', tool_shed_repository.owner, tool_shed_repository.name ) relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name ) install_dir = os.path.join( clone_dir, tool_shed_repository.name ) @@ -542,8 +543,7 @@ tool_shed_repository.owner, tool_shed_repository.installed_changeset_revision ) if not cloned_ok: - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, + irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.CLONING ) cloned_ok, error_message = hg_util.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok and not is_installed: @@ -556,8 +556,7 @@ self.app.install_model.context.refresh( tool_shed_repository ) metadata_dict = tool_shed_repository.metadata if 'tools' in metadata_dict: - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, + irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) # Get the tool_versions from the tool shed for each tool in the installed change set. url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ @@ -595,11 +594,11 @@ parent_id=tool_version_using_old_id.id ) self.app.install_model.context.add( tool_version_association ) self.app.install_model.context.flush() - suc.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLED ) + irm.update_tool_shed_repository_status( tool_shed_repository, + self.app.install_model.ToolShedRepository.installation_status.INSTALLED ) else: print 'Error attempting to clone repository %s: %s' % ( str( tool_shed_repository.name ), str( error_message ) ) - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, + irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.ERROR, error_message=error_message ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -1,5 +1,7 @@ import logging import os +import shutil +import tempfile import galaxy.tools from galaxy.tools.search import ToolBoxSearch @@ -19,9 +21,11 @@ self.app = app def add_to_shed_tool_config( self, shed_tool_conf_dict, elem_list ): - # A tool shed repository is being installed so change the shed_tool_conf file. Parse the - # config file to generate the entire list of config_elems instead of using the in-memory list - # since it will be a subset of the entire list if one or more repositories have been deactivated. + """ + "A tool shed repository is being installed so change the shed_tool_conf file. Parse the + config file to generate the entire list of config_elems instead of using the in-memory list + since it will be a subset of the entire list if one or more repositories have been deactivated. + """ shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] tool_path = shed_tool_conf_dict[ 'tool_path' ] config_elems = [] @@ -34,13 +38,13 @@ for elem_entry in elem_list: config_elems.append( elem_entry ) # Persist the altered shed_tool_config file. - suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path ) + self.config_elems_to_xml_file( config_elems, shed_tool_conf, tool_path ) def add_to_tool_panel( self, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner, shed_tool_conf, tool_panel_dict, new_install=True ): """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" # We need to change the in-memory version and the file system version of the shed_tool_conf file. - index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_tool_conf_dict[ 'tool_path' ] # Generate the list of ElementTree Element objects for each section or tool. elem_list = self.generate_tool_panel_elem_list( repository_name, @@ -81,6 +85,44 @@ self.app.toolbox.write_integrated_tool_panel_config_file() self.app.toolbox_search = ToolBoxSearch( self.app.toolbox ) + def config_elems_to_xml_file( self, config_elems, config_filename, tool_path ): + """ + Persist the current in-memory list of config_elems to a file named by the + value of config_filename. + """ + fd, filename = tempfile.mkstemp( prefix="tmp-toolshed-cetxf" ) + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<toolbox tool_path="%s">\n' % str( tool_path ) ) + for elem in config_elems: + os.write( fd, '%s' % xml_util.xml_to_string( elem, use_indent=True ) ) + os.write( fd, '</toolbox>\n' ) + os.close( fd ) + shutil.move( filename, os.path.abspath( config_filename ) ) + os.chmod( config_filename, 0644 ) + + def generate_tool_elem( self, tool_shed, repository_name, changeset_revision, owner, tool_file_path, + tool, tool_section ): + """Create and return an ElementTree tool Element.""" + if tool_section is not None: + tool_elem = XmlET.SubElement( tool_section, 'tool' ) + else: + tool_elem = XmlET.Element( 'tool' ) + tool_elem.attrib[ 'file' ] = tool_file_path + tool_elem.attrib[ 'guid' ] = tool.guid + tool_shed_elem = XmlET.SubElement( tool_elem, 'tool_shed' ) + tool_shed_elem.text = tool_shed + repository_name_elem = XmlET.SubElement( tool_elem, 'repository_name' ) + repository_name_elem.text = repository_name + repository_owner_elem = XmlET.SubElement( tool_elem, 'repository_owner' ) + repository_owner_elem.text = owner + changeset_revision_elem = XmlET.SubElement( tool_elem, 'installed_changeset_revision' ) + changeset_revision_elem.text = changeset_revision + id_elem = XmlET.SubElement( tool_elem, 'id' ) + id_elem.text = tool.id + version_elem = XmlET.SubElement( tool_elem, 'version' ) + version_elem.text = tool.version + return tool_elem + def generate_tool_panel_dict_for_new_install( self, tool_dicts, tool_section=None ): """ When installing a repository that contains tools, all tools must currently be defined @@ -125,6 +167,60 @@ tool_panel_dict[ guid ] = tool_section_dicts return tool_panel_dict + def generate_tool_panel_dict_from_shed_tool_conf_entries( self, repository ): + """ + Keep track of the section in the tool panel in which this repository's tools + will be contained by parsing the shed_tool_conf in which the repository's tools + are defined and storing the tool panel definition of each tool in the repository. + This method is called only when the repository is being deactivated or un-installed + and allows for activation or re-installation using the original layout. + """ + tool_panel_dict = {} + shed_tool_conf, tool_path, relative_install_dir = \ + suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) + metadata = repository.metadata + # Create a dictionary of tool guid and tool config file name for each tool in the repository. + guids_and_configs = {} + if 'tools' in metadata: + for tool_dict in metadata[ 'tools' ]: + guid = tool_dict[ 'guid' ] + tool_config = tool_dict[ 'tool_config' ] + file_name = basic_util.strip_path( tool_config ) + guids_and_configs[ guid ] = file_name + # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree is None: + return tool_panel_dict + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + guid = elem.get( 'guid' ) + if guid in guids_and_configs: + # The tool is displayed in the tool panel outside of any tool sections. + tool_section_dict = dict( tool_config=guids_and_configs[ guid ], id='', name='', version='' ) + if guid in tool_panel_dict: + tool_panel_dict[ guid ].append( tool_section_dict ) + else: + tool_panel_dict[ guid ] = [ tool_section_dict ] + elif elem.tag == 'section': + section_id = elem.get( 'id' ) or '' + section_name = elem.get( 'name' ) or '' + section_version = elem.get( 'version' ) or '' + for section_elem in elem: + if section_elem.tag == 'tool': + guid = section_elem.get( 'guid' ) + if guid in guids_and_configs: + # The tool is displayed in the tool panel inside the current tool section. + tool_section_dict = dict( tool_config=guids_and_configs[ guid ], + id=section_id, + name=section_name, + version=section_version ) + if guid in tool_panel_dict: + tool_panel_dict[ guid ].append( tool_section_dict ) + else: + tool_panel_dict[ guid ] = [ tool_section_dict ] + return tool_panel_dict + def generate_tool_panel_elem_list( self, repository_name, repository_clone_url, changeset_revision, tool_panel_dict, repository_tools_tups, owner='' ): """Generate a list of ElementTree Element objects for each section or tool.""" @@ -157,21 +253,21 @@ if tup_guid == guid: break if inside_section: - tool_elem = suc.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - tool_section ) + tool_elem = self.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + tool_section ) else: - tool_elem = suc.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - None ) + tool_elem = self.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + None ) if inside_section: if section_in_elem_list: elem_list[ index ] = tool_section @@ -238,6 +334,19 @@ log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) return tool_panel_section_key, tool_section + def get_shed_tool_conf_dict( self, shed_tool_conf ): + """ + Return the in-memory version of the shed_tool_conf file, which is stored in + the config_elems entry in the shed_tool_conf_dict associated with the file. + """ + for index, shed_tool_conf_dict in enumerate( self.app.toolbox.shed_tool_confs ): + if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]: + return index, shed_tool_conf_dict + else: + file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] ) + if shed_tool_conf == file_name: + return index, shed_tool_conf_dict + def handle_tool_panel_section( self, toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ): """Return a ToolSection object retrieved from the current in-memory tool_panel.""" # If tool_panel_section_id is received, the section exists in the tool panel. In this @@ -299,10 +408,12 @@ return tool_section, tool_panel_section_key def remove_from_shed_tool_config( self, shed_tool_conf_dict, guids_to_remove ): - # A tool shed repository is being uninstalled so change the shed_tool_conf file. - # Parse the config file to generate the entire list of config_elems instead of - # using the in-memory list since it will be a subset of the entire list if one - # or more repositories have been deactivated. + """ + A tool shed repository is being uninstalled so change the shed_tool_conf file. + Parse the config file to generate the entire list of config_elems instead of + using the in-memory list since it will be a subset of the entire list if one + or more repositories have been deactivated. + """ shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] tool_path = shed_tool_conf_dict[ 'tool_path' ] config_elems = [] @@ -330,7 +441,7 @@ for config_elem in config_elems_to_remove: config_elems.remove( config_elem ) # Persist the altered in-memory version of the tool config. - suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path ) + self.config_elems_to_xml_file( config_elems, shed_tool_conf, tool_path ) def remove_from_tool_panel( self, repository, shed_tool_conf, uninstall ): """ @@ -340,7 +451,7 @@ # Determine where the tools are currently defined in the tool panel and store this # information so the tools can be displayed in the same way when the repository is # activated or reinstalled. - tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( self.app, repository ) + tool_panel_dict = self.generate_tool_panel_dict_from_shed_tool_conf_entries( repository ) repository.metadata[ 'tool_panel_section' ] = tool_panel_dict self.app.install_model.context.add( repository ) self.app.install_model.context.flush() @@ -351,7 +462,7 @@ for guid_to_remove in guids_to_remove: if guid_to_remove in self.app.toolbox.tools_by_id: del self.app.toolbox.tools_by_id[ guid_to_remove ] - index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/metadata/repository_metadata_manager.py --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -951,11 +951,9 @@ unsuccessful_count = 0 for repository_id in repository_ids: try: - # We're in the tool shed. repository = suc.get_repository_in_tool_shed( self.app, repository_id ) - owner = str( repository.user.username ) invalid_file_tups, metadata_dict = \ - self.reset_all_metadata_on_repository_in_tool_shed( self.user, repository_id ) + self.reset_all_metadata_on_repository_in_tool_shed( repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( self.app, invalid_file_tups, @@ -966,7 +964,7 @@ unsuccessful_count += 1 else: log.debug( "Successfully reset metadata on repository %s owned by %s" % \ - ( str( repository.name ), owner ) ) + ( str( repository.name ), str( repository.user.username ) ) ) successful_count += 1 except: log.exception( "Error attempting to reset metadata on repository %s" % str( repository.name ) ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/data_manager_util.py --- a/lib/tool_shed/util/data_manager_util.py +++ b/lib/tool_shed/util/data_manager_util.py @@ -1,7 +1,10 @@ import logging import os + +from tool_shed.galaxy_install.tools import tool_panel_manager + +from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util -import tool_shed.util.shed_util_common as suc log = logging.getLogger( __name__ ) @@ -17,6 +20,7 @@ def install_data_managers( app, shed_data_manager_conf_filename, metadata_dict, shed_config_dict, relative_install_dir, repository, repository_tools_tups ): rval = [] if 'data_manager' in metadata_dict: + tpm = tool_panel_manager.ToolPanelManager( app ) repository_tools_by_guid = {} for tool_tup in repository_tools_tups: repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] ) @@ -70,13 +74,13 @@ elem.set( 'shed_conf_file', shed_config_dict['config_filename'] ) if elem.get( 'tool_file', None ) is not None: del elem.attrib[ 'tool_file' ] #remove old tool_file info - tool_elem = suc.generate_tool_elem( repository.tool_shed, - repository.name, - repository.installed_changeset_revision, - repository.owner, - tool_config_filename, - tool, - None ) + tool_elem = tpm.generate_tool_elem( repository.tool_shed, + repository.name, + repository.installed_changeset_revision, + repository.owner, + tool_config_filename, + tool, + None ) elem.insert( 0, tool_elem ) data_manager = app.data_managers.load_manager_from_elem( elem, tool_path=shed_config_dict.get( 'tool_path', '' ), replace_existing=True ) if data_manager: diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/repository_maintenance_util.py --- a/lib/tool_shed/util/repository_maintenance_util.py +++ b/lib/tool_shed/util/repository_maintenance_util.py @@ -40,6 +40,20 @@ config.write( new_file ) new_file.close() +def check_or_update_tool_shed_status_for_installed_repository( app, repository ): + updated = False + tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( app, repository ) + if tool_shed_status_dict: + ok = True + if tool_shed_status_dict != repository.tool_shed_status: + repository.tool_shed_status = tool_shed_status_dict + app.install_model.context.add( repository ) + app.install_model.context.flush() + updated = True + else: + ok = False + return ok, updated + def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, repository=None, repository_metadata=None, tool_dependencies=None, repository_dependencies=None ): """ @@ -171,6 +185,11 @@ sa_session.flush() return role +def get_installed_tool_shed_repository( app, id ): + """Get a tool shed repository record from the Galaxy database defined by the id.""" + return app.install_model.context.query( app.install_model.ToolShedRepository ) \ + .get( app.security.decode_id( id ) ) + def get_repo_info_dict( app, user, repository_id, changeset_revision ): repository = suc.get_repository_in_tool_shed( app, repository_id ) repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -1,24 +1,22 @@ +import json import logging import os import re import shutil import string -import tempfile +import sqlalchemy.orm.exc + from galaxy import util -from galaxy.util import asbool -from galaxy.util import json from galaxy.web import url_for -from galaxy.web.form_builder import SelectField from galaxy.datatypes import checkers from galaxy.model.orm import and_ from galaxy.model.orm import or_ -import sqlalchemy.orm.exc + from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util -from tool_shed.util import xml_util -import tool_shed.repository_types.util as rt_util + from xml.etree import ElementTree as XmlET from urllib2 import HTTPError @@ -84,49 +82,6 @@ '${host}' """ -def build_tool_dependencies_select_field( app, tool_shed_repository, name, multiple=True, display='checkboxes', uninstalled_only=False ): - """Method called from Galaxy to generate the current list of tool dependency ids for an installed tool shed repository.""" - tool_dependencies_select_field = SelectField( name=name, multiple=multiple, display=display ) - for tool_dependency in tool_shed_repository.tool_dependencies: - if uninstalled_only: - if tool_dependency.status not in [ app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, - app.install_model.ToolDependency.installation_status.UNINSTALLED ]: - continue - else: - if tool_dependency.status in [ app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, - app.install_model.ToolDependency.installation_status.UNINSTALLED ]: - continue - option_label = '%s version %s' % ( str( tool_dependency.name ), str( tool_dependency.version ) ) - option_value = app.security.encode_id( tool_dependency.id ) - tool_dependencies_select_field.add_option( option_label, option_value ) - return tool_dependencies_select_field - -def check_or_update_tool_shed_status_for_installed_repository( app, repository ): - updated = False - tool_shed_status_dict = get_tool_shed_status_for_installed_repository( app, repository ) - if tool_shed_status_dict: - ok = True - if tool_shed_status_dict != repository.tool_shed_status: - repository.tool_shed_status = tool_shed_status_dict - app.install_model.context.add( repository ) - app.install_model.context.flush() - updated = True - else: - ok = False - return ok, updated - -def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): - """Persist the current in-memory list of config_elems to a file named by the value of config_filename.""" - fd, filename = tempfile.mkstemp( prefix="tmp-toolshed-cetxf" ) - os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, '<toolbox tool_path="%s">\n' % str( tool_path ) ) - for elem in config_elems: - os.write( fd, '%s' % xml_util.xml_to_string( elem, use_indent=True ) ) - os.write( fd, '</toolbox>\n' ) - os.close( fd ) - shutil.move( filename, os.path.abspath( config_filename ) ) - os.chmod( config_filename, 0644 ) - def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): """ @@ -248,28 +203,6 @@ sharable_url += '/%s' % changeset_revision return sharable_url -def generate_tool_elem( tool_shed, repository_name, changeset_revision, owner, tool_file_path, tool, tool_section ): - """Create and return an ElementTree tool Element.""" - if tool_section is not None: - tool_elem = XmlET.SubElement( tool_section, 'tool' ) - else: - tool_elem = XmlET.Element( 'tool' ) - tool_elem.attrib[ 'file' ] = tool_file_path - tool_elem.attrib[ 'guid' ] = tool.guid - tool_shed_elem = XmlET.SubElement( tool_elem, 'tool_shed' ) - tool_shed_elem.text = tool_shed - repository_name_elem = XmlET.SubElement( tool_elem, 'repository_name' ) - repository_name_elem.text = repository_name - repository_owner_elem = XmlET.SubElement( tool_elem, 'repository_owner' ) - repository_owner_elem.text = owner - changeset_revision_elem = XmlET.SubElement( tool_elem, 'installed_changeset_revision' ) - changeset_revision_elem.text = changeset_revision - id_elem = XmlET.SubElement( tool_elem, 'id' ) - id_elem.text = tool.id - version_elem = XmlET.SubElement( tool_elem, 'version' ) - version_elem.text = tool.version - return tool_elem - def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for @@ -279,57 +212,6 @@ tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) -def generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ): - """ - Keep track of the section in the tool panel in which this repository's tools will be contained by parsing the shed_tool_conf in - which the repository's tools are defined and storing the tool panel definition of each tool in the repository. This method is called - only when the repository is being deactivated or uninstalled and allows for activation or reinstallation using the original layout. - """ - tool_panel_dict = {} - shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( app, repository ) - metadata = repository.metadata - # Create a dictionary of tool guid and tool config file name for each tool in the repository. - guids_and_configs = {} - if 'tools' in metadata: - for tool_dict in metadata[ 'tools' ]: - guid = tool_dict[ 'guid' ] - tool_config = tool_dict[ 'tool_config' ] - file_name = basic_util.strip_path( tool_config ) - guids_and_configs[ guid ] = file_name - # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. - tree, error_message = xml_util.parse_xml( shed_tool_conf ) - if tree is None: - return tool_panel_dict - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - guid = elem.get( 'guid' ) - if guid in guids_and_configs: - # The tool is displayed in the tool panel outside of any tool sections. - tool_section_dict = dict( tool_config=guids_and_configs[ guid ], id='', name='', version='' ) - if guid in tool_panel_dict: - tool_panel_dict[ guid ].append( tool_section_dict ) - else: - tool_panel_dict[ guid ] = [ tool_section_dict ] - elif elem.tag == 'section': - section_id = elem.get( 'id' ) or '' - section_name = elem.get( 'name' ) or '' - section_version = elem.get( 'version' ) or '' - for section_elem in elem: - if section_elem.tag == 'tool': - guid = section_elem.get( 'guid' ) - if guid in guids_and_configs: - # The tool is displayed in the tool panel inside the current tool section. - tool_section_dict = dict( tool_config=guids_and_configs[ guid ], - id=section_id, - name=section_name, - version=section_version ) - if guid in tool_panel_dict: - tool_panel_dict[ guid ].append( tool_section_dict ) - else: - tool_panel_dict[ guid ] = [ tool_section_dict ] - return tool_panel_dict - def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_revision ): """ Generate a repository installation directory that guarantees repositories with the same @@ -483,10 +365,6 @@ return ','.join( installing_repository_ids ) return installing_repository_ids -def get_installed_tool_shed_repository( app, id ): - """Get a tool shed repository record from the Galaxy database defined by the id.""" - return app.install_model.context.query( app.install_model.ToolShedRepository ).get( app.security.decode_id( id ) ) - def get_latest_changeset_revision( app, repository, repo ): repository_tip = repository.tip( app ) repository_metadata = get_repository_metadata_by_changeset_revision( app, @@ -565,8 +443,10 @@ def get_or_create_tool_shed_repository( app, tool_shed, name, owner, changeset_revision ): """ - Return a tool shed repository database record defined by the combination of tool shed, repository name, repository owner and changeset_revision - or installed_changeset_revision. A new tool shed repository record will be created if one is not located. + Return a tool shed repository database record defined by the combination of + tool shed, repository name, repository owner and changeset_revision or + installed_changeset_revision. A new tool shed repository record will be + created if one is not located. """ install_model = app.install_model # We store the port in the database. @@ -697,7 +577,7 @@ for rd_tup in repository_dependencies: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) - if not asbool( only_if_compiling_contained_td ): + if not util.asbool( only_if_compiling_contained_td ): has_repository_dependencies = True break # Set has_repository_dependencies_only_if_compiling_contained_td, which will be True only if at @@ -706,7 +586,7 @@ for rd_tup in repository_dependencies: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) - if asbool( only_if_compiling_contained_td ): + if util.asbool( only_if_compiling_contained_td ): has_repository_dependencies_only_if_compiling_contained_td = True break return has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td @@ -838,8 +718,8 @@ # where the repository dependency is not installed prior to the dependent repository's tool dependency compilation process, the tool # dependency compilation framework will install the repository dependency prior to compilation of the dependent repository's tool # dependency. - if not asbool( only_if_compiling_contained_td ): - if asbool( prior_installation_required ): + if not util.asbool( only_if_compiling_contained_td ): + if util.asbool( prior_installation_required ): if is_tool_shed_client( app ): # We store the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) @@ -899,34 +779,6 @@ query = app.model.context.query( app.model.Repository ) return query -def get_repository_tools_tups( app, metadata_dict ): - """Return a list of tuples of the form (relative_path, guid, tool) for each tool defined in the received tool shed repository metadata.""" - repository_tools_tups = [] - index, shed_conf_dict = get_shed_tool_conf_dict( app, metadata_dict.get( 'shed_config_filename' ) ) - if 'tools' in metadata_dict: - for tool_dict in metadata_dict[ 'tools' ]: - load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) - if shed_conf_dict.get( 'tool_path' ): - load_relative_path = os.path.join( shed_conf_dict.get( 'tool_path' ), relative_path ) - guid = tool_dict.get( 'guid', None ) - if relative_path and guid: - tool = app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid ) - else: - tool = None - if tool: - repository_tools_tups.append( ( relative_path, guid, tool ) ) - return repository_tools_tups - -def get_shed_tool_conf_dict( app, shed_tool_conf ): - """Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry in the shed_tool_conf_dict associated with the file.""" - for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): - if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]: - return index, shed_tool_conf_dict - else: - file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] ) - if shed_tool_conf == file_name: - return index, shed_tool_conf_dict - def get_skip_tool_test_by_changeset_revision( app, changeset_revision ): """Return a skip_tool_test record whose initial_changeset_revision is the received changeset_revision.""" # There should only be one, but we'll use first() so callers won't have to handle exceptions. @@ -1238,7 +1090,7 @@ email_alerts.append( user.email ) else: subject = "Galaxy tool shed update alert for repository named %s" % str( repository.name ) - email_alerts = json.from_json_string( repository.email_alerts ) + email_alerts = json.loads( repository.email_alerts ) for email in email_alerts: to = email.strip() # Send it @@ -1262,7 +1114,8 @@ return False def is_tool_shed_client( app ): - """ The tool shed and clients to the tool (i.e. Galaxy) require a lot + """ + The tool shed and clients to the tool (i.e. Galaxy) require a lot of similar functionality in this file but with small differences. This method should determine if the app performing the action is the tool shed or a client of the tool shed. @@ -1338,32 +1191,12 @@ return tool_shed_repository, previous_changeset_revision return None, None -def reset_previously_installed_repository( install_model, repository ): - """ - Reset the atrributes of a tool_shed_repository that was previsouly installed. The repository will be in some state other than INSTALLED, - so all atributes will be set to the default NEW state. This will enable the repository to be freshly installed. - """ - debug_msg = "Resetting tool_shed_repository '%s' for installation.\n" % str( repository.name ) - debug_msg += "The current state of the tool_shed_repository is:\n" - debug_msg += "deleted: %s\n" % str( repository.deleted ) - debug_msg += "tool_shed_status: %s\n" % str( repository.tool_shed_status ) - debug_msg += "uninstalled: %s\n" % str( repository.uninstalled ) - debug_msg += "status: %s\n" % str( repository.status ) - debug_msg += "error_message: %s\n" % str( repository.error_message ) - log.debug( debug_msg ) - repository.deleted = False - repository.tool_shed_status = None - repository.uninstalled = False - repository.status = install_model.ToolShedRepository.installation_status.NEW - repository.error_message = None - install_model.context.add( repository ) - install_model.context.flush() - def set_image_paths( app, encoded_repository_id, text ): """ - Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy as well as image - display in repository README files. This method will determine the location of the image file and return the path to it that will enable - the caller to open the file. + Handle tool help image display for tools that are contained in repositories in + the tool shed or installed into Galaxy as well as image display in repository + README files. This method will determine the location of the image file and + return the path to it that will enable the caller to open the file. """ if text: if is_tool_shed_client( app ): @@ -1373,22 +1206,32 @@ route_to_images = '/repository/static/images/%s' % encoded_repository_id # We used to require $PATH_TO_IMAGES, but we now eliminate it if it's used. text = text.replace( '$PATH_TO_IMAGES', '' ) - # Eliminate the invalid setting of ./static/images since the routes will properly display images contained in that directory. + # Eliminate the invalid setting of ./static/images since the routes will + # properly display images contained in that directory. text = text.replace( './static/images', '' ) - # Eliminate the default setting of /static/images since the routes will properly display images contained in that directory. + # Eliminate the default setting of /static/images since the routes will + # properly display images contained in that directory. text = text.replace( '/static/images', '' ) - # Use regex to instantiate routes into the defined image paths, but replace paths that start with neither http:// nor https://, - # which will allow for settings like .. images:: http_files/images/help.png + # Use regex to instantiate routes into the defined image paths, but replace + # paths that start with neither http:// nor https://, which will allow for + # settings like .. images:: http_files/images/help.png for match in re.findall( '.. image:: (?!http)/?(.+)', text ): text = text.replace( match, match.replace( '/', '%2F' ) ) text = re.sub( r'\.\. image:: (?!https?://)/?(.+)', r'.. image:: %s/\1' % route_to_images, text ) return text def set_only_if_compiling_contained_td( repository, required_repository ): - """Return True if the received required_repository is only needed to compile a tool dependency defined for the received repository.""" - # This method is called only from Galaxy when rendering repository dependencies for an installed tool shed repository. + """ + Return True if the received required_repository is only needed to compile a tool + dependency defined for the received repository. + """ + # This method is called only from Galaxy when rendering repository dependencies + # for an installed tool shed repository. # TODO: Do we need to check more than changeset_revision here? - required_repository_tup = [ required_repository.tool_shed, required_repository.name, required_repository.owner, required_repository.changeset_revision ] + required_repository_tup = [ required_repository.tool_shed, \ + required_repository.name, \ + required_repository.owner, \ + required_repository.changeset_revision ] for tup in repository.tuples_of_repository_dependencies_needed_for_compiling_td: partial_tup = tup[ 0:4 ] if partial_tup == required_repository_tup: @@ -1430,63 +1273,8 @@ app.install_model.context.add( repository ) app.install_model.context.flush() -def tool_shed_from_repository_clone_url( repository_clone_url ): - """Given a repository clone URL, return the tool shed that contains the repository.""" - return common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ).split( '/repos/' )[ 0 ].rstrip( '/' ) - def tool_shed_is_this_tool_shed( toolshed_base_url ): """Determine if a tool shed is the current tool shed.""" cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url ) cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url( str( url_for( '/', qualified=True ) ) ) return cleaned_toolshed_base_url == cleaned_tool_shed - -def update_in_shed_tool_config( app, repository ): - """ - A tool shed repository is being updated so change the shed_tool_conf file. Parse the config - file to generate the entire list of config_elems instead of using the in-memory list. - """ - shed_conf_dict = repository.get_shed_config_dict( app ) - shed_tool_conf = shed_conf_dict[ 'config_filename' ] - tool_path = shed_conf_dict[ 'tool_path' ] - tool_panel_dict = generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ) - repository_tools_tups = get_repository_tools_tups( app, repository.metadata ) - clone_url = common_util.generate_clone_url_for_installed_repository( app, repository ) - cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( clone_url ) - tool_shed = tool_shed_from_repository_clone_url( cleaned_repository_clone_url ) - owner = repository.owner - if not owner: - owner = get_repository_owner( cleaned_repository_clone_url ) - guid_to_tool_elem_dict = {} - for tool_config_filename, guid, tool in repository_tools_tups: - guid_to_tool_elem_dict[ guid ] = generate_tool_elem( tool_shed, - repository.name, - repository.changeset_revision, - repository.owner or '', - tool_config_filename, - tool, - None ) - config_elems = [] - tree, error_message = xml_util.parse_xml( shed_tool_conf ) - if tree: - root = tree.getroot() - for elem in root: - if elem.tag == 'section': - for i, tool_elem in enumerate( elem ): - guid = tool_elem.attrib.get( 'guid' ) - if guid in guid_to_tool_elem_dict: - elem[i] = guid_to_tool_elem_dict[ guid ] - elif elem.tag == 'tool': - guid = elem.attrib.get( 'guid' ) - if guid in guid_to_tool_elem_dict: - elem = guid_to_tool_elem_dict[ guid ] - config_elems.append( elem ) - config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) - -def update_tool_shed_repository_status( app, tool_shed_repository, status, error_message=None ): - """Update the status of a tool shed repository in the process of being installed into Galaxy.""" - context = app.install_model.context - tool_shed_repository.status = status - if error_message: - tool_shed_repository.error_message = str( error_message ) - context.add( tool_shed_repository ) - context.flush() diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -4,11 +4,34 @@ from galaxy import util from galaxy.model.orm import and_ +from galaxy.web.form_builder import SelectField + from tool_shed.util import hg_util from tool_shed.util import xml_util log = logging.getLogger( __name__ ) +def build_tool_dependencies_select_field( app, tool_shed_repository, name, multiple=True, display='checkboxes', + uninstalled_only=False ): + """ + Generate a SelectField consisting of the current list of tool dependency ids + for an installed tool shed repository. + """ + tool_dependencies_select_field = SelectField( name=name, multiple=multiple, display=display ) + for tool_dependency in tool_shed_repository.tool_dependencies: + if uninstalled_only: + if tool_dependency.status not in [ app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, + app.install_model.ToolDependency.installation_status.UNINSTALLED ]: + continue + else: + if tool_dependency.status in [ app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, + app.install_model.ToolDependency.installation_status.UNINSTALLED ]: + continue + option_label = '%s version %s' % ( str( tool_dependency.name ), str( tool_dependency.version ) ) + option_value = app.security.encode_id( tool_dependency.id ) + tool_dependencies_select_field.add_option( option_label, option_value ) + return tool_dependencies_select_field + def create_or_update_tool_dependency( app, tool_shed_repository, name, version, type, status, set_status=True ): """Create or update a tool_dependency record in the Galaxy database.""" # Called from Galaxy (never the tool shed) when a new repository is being installed or when an uninstalled diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/xml_util.py --- a/lib/tool_shed/util/xml_util.py +++ b/lib/tool_shed/util/xml_util.py @@ -137,8 +137,8 @@ def xml_to_string( elem, encoding='utf-8', use_indent=False, level=0 ): if elem is not None: if use_indent: - # We were called from suc.config_elems_to_xml_file(), so set the level to 1 since level 0 - # is the <toolbox> tag set. + # We were called from ToolPanelManager.config_elems_to_xml_file(), so + # set the level to 1 since level 0 is the <toolbox> tag set. indent( elem, level=level ) if using_python_27: xml_str = '%s\n' % xml.etree.ElementTree.tostring( elem, encoding=encoding, method="xml" ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 21 Jul '14

21 Jul '14

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c118476b5823/ Changeset: c118476b5823 User: jgoecks Date: 2014-07-21 16:19:14 Summary: Use jquery to remove event handler. Affected #: 1 file diff -r cbca66cc16bfed165f3b38399857ad0b85d14489 -r c118476b5823c676267dbe5d48f7105711e3c15d static/scripts/viz/trackster.js --- a/static/scripts/viz/trackster.js +++ b/static/scripts/viz/trackster.js @@ -356,7 +356,7 @@ buttons: { "Cancel": function() { Galaxy.modal.hide(); }, "Leave without Saving" : function() { - window.onbeforeunload = undefined; + $(window).off('beforeunload'); window.location = galaxy_config.root + 'visualization'; }, "Save" : function() { https://bitbucket.org/galaxy/galaxy-central/commits/bca25a3bc9c7/ Changeset: bca25a3bc9c7 User: jgoecks Date: 2014-07-21 16:26:18 Summary: Trackster: remove bottom label track because it is redundant. Affected #: 1 file diff -r c118476b5823c676267dbe5d48f7105711e3c15d -r bca25a3bc9c7618eb37fb467ae32564fa213aac5 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -885,8 +885,6 @@ }); }); }); - // Another label track at bottom - this.nav_labeltrack = $("<div/>").addClass("nav-labeltrack").appendTo(this.bottom_container); // Navigation at top this.nav_container = $("<div/>").addClass("trackster-nav-container").prependTo(this.top_container); this.nav = $("<div/>").addClass("trackster-nav").appendTo(this.nav_container); @@ -1042,7 +1040,7 @@ // Dragging in the top label track allows selecting a region to zoom in on selected region. this.top_labeltrack.bind( "dragstart", function( e, d ) { return $("<div/>").addClass('zoom-area').css( - "height", view.browser_content_div.height() + view.top_labeltrack.height() + view.nav_labeltrack.height() + 1 + "height", view.browser_content_div.height() + view.top_labeltrack.height() + 1 ).appendTo( $(this) ); }).bind( "drag", function( e, d ) { $( d.proxy ).css({ left: Math.min( e.pageX, d.startX ) - view.container.offset().left, width: Math.abs( e.pageX - d.startX ) }); @@ -1091,7 +1089,6 @@ */ this.add_label_track( new LabelTrack( this, { content_div: this.top_labeltrack } ) ); - this.add_label_track( new LabelTrack( this, { content_div: this.nav_labeltrack } ) ); $(window).bind("resize", function() { // Stop previous timer. https://bitbucket.org/galaxy/galaxy-central/commits/f9b47a91b7f2/ Changeset: f9b47a91b7f2 User: jgoecks Date: 2014-07-21 16:27:44 Summary: Trackster: some fixes for vertical position element. Affected #: 1 file diff -r bca25a3bc9c7618eb37fb467ae32564fa213aac5 -r f9b47a91b7f2611c87c9dc7a0ceafc733f6bc181 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -1062,11 +1062,9 @@ view.request_redraw(); }); + // FIXME: this is still wonky for embedded visualizations. + /* // For vertical alignment, track mouse with simple line. - // Fixes needed for this to work: - // (a) make work with embedded visualizations; - // (b) seems to get stuck on tile overlaps. - /* var mouse_tracker_div = $('<div/>').addClass('mouse-pos').appendTo(parent_element); // Show tracker only when hovering over view. @@ -1075,9 +1073,8 @@ mouse_tracker_div.show(); parent_element.mousemove(function(e) { mouse_tracker_div.css({ - // -1 makes it appear next to the mouse w/o obscuring clicking - // and dragging on view elements. - left: e.pageX - 1 + // -1 makes line appear next to the mouse w/o preventing mouse actions. + left: e.pageX - parent_element.offset().left - 1 }); }); }, https://bitbucket.org/galaxy/galaxy-central/commits/972a80267009/ Changeset: 972a80267009 User: jgoecks Date: 2014-07-21 19:23:41 Summary: Trackster: do not reset overview when changing chromosome. Affected #: 1 file diff -r f9b47a91b7f2611c87c9dc7a0ceafc733f6bc181 -r 972a80267009126e558ba14142df48b897db48c8 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -1287,7 +1287,6 @@ } } - view.reset_overview(); view.request_redraw(); } }, https://bitbucket.org/galaxy/galaxy-central/commits/e9de99ef7710/ Changeset: e9de99ef7710 User: jgoecks Date: 2014-07-21 19:42:14 Summary: Automated merge Affected #: 19 files diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -413,12 +413,12 @@ # Make sure the current user's API key proves he is an admin user in this Galaxy instance. if not trans.user_is_admin(): raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) + irmm = InstalledRepositoryMetadataManager( trans.app ) + query = irmm.get_query_for_setting_metadata_on_repositories( order=False ) # Now reset metadata on all remaining repositories. for repository in query: repository_id = trans.security.encode_id( repository.id ) try: - irmm = InstalledRepositoryMetadataManager( trans.app ) invalid_file_tups, metadata_dict = irmm.reset_all_metadata_on_installed_repository( repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -30,6 +30,7 @@ from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import tool_panel_manager log = logging.getLogger( __name__ ) @@ -244,10 +245,10 @@ if kwd.get( 'deactivate_or_uninstall_repository_button', False ): if tool_shed_repository.includes_tools_for_display_in_tool_panel: # Handle tool panel alterations. - tool_util.remove_from_tool_panel( trans.app, - tool_shed_repository, - shed_tool_conf, - uninstall=remove_from_disk_checked ) + tpm = tool_panel_manager.ToolPanelManager( trans.app ) + tpm.remove_from_tool_panel( tool_shed_repository, + shed_tool_conf, + uninstall=remove_from_disk_checked ) if tool_shed_repository.includes_data_managers: data_manager_util.remove_from_data_manager( trans.app, tool_shed_repository ) if tool_shed_repository.includes_datatypes: @@ -1336,13 +1337,14 @@ # its repository dependencies. includes_tool_dependencies = tool_shed_repository.includes_tool_dependencies if tool_shed_repository.includes_tools_for_display_in_tool_panel: + tpm = tool_panel_manager.ToolPanelManager( trans.app ) # Handle the selected tool panel location for loading tools included in the tool shed repository. tool_section, tool_panel_section_key = \ - tool_util.handle_tool_panel_selection( toolbox=trans.app.toolbox, - metadata=metadata, - no_changes_checked=no_changes_checked, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + tpm.handle_tool_panel_selection( toolbox=trans.app.toolbox, + metadata=metadata, + no_changes_checked=no_changes_checked, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) if tool_section is not None: # Just in case the tool_section.id differs from tool_panel_section_id, which it shouldn't... tool_panel_section_id = str( tool_section.id ) @@ -1706,13 +1708,13 @@ @web.expose @web.require_admin def reset_metadata_on_selected_installed_repositories( self, trans, **kwd ): + irmm = InstalledRepositoryMetadataManager( trans.app ) if 'reset_metadata_on_selected_repositories_button' in kwd: - irmm = InstalledRepositoryMetadataManager( trans.app ) message, status = irmm.reset_metadata_on_selected_repositories( trans.user, **kwd ) else: message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repositories_select_field = suc.build_repository_ids_select_field( trans ) + repositories_select_field = irmm.build_repository_ids_select_field( trans ) return trans.fill_template( '/admin/tool_shed_repository/reset_metadata_on_selected_repositories.mako', repositories_select_field=repositories_select_field, message=message, @@ -1942,19 +1944,19 @@ updating_installed_repository=True, persist=True ) if 'tools' in metadata_dict: + tpm = tool_panel_manager.ToolPanelManager( trans.app ) tool_panel_dict = metadata_dict.get( 'tool_panel_section', None ) if tool_panel_dict is None: tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( trans.app, repository ) repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) - tool_util.add_to_tool_panel( app=trans.app, - repository_name=str( repository.name ), - repository_clone_url=repository_clone_url, - changeset_revision=str( repository.installed_changeset_revision ), - repository_tools_tups=repository_tools_tups, - owner=str( repository.owner ), - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=False ) + tpm.add_to_tool_panel( repository_name=str( repository.name ), + repository_clone_url=repository_clone_url, + changeset_revision=str( repository.installed_changeset_revision ), + repository_tools_tups=repository_tools_tups, + owner=str( repository.owner ), + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=False ) # Add new Data Manager entries if 'data_manager' in metadata_dict: new_data_managers = data_manager_util.install_data_managers( trans.app, diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -392,7 +392,8 @@ my_writable = True handled_repository_ids = [] repository_ids = [] - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) + query = rmm.get_query_for_setting_metadata_on_repositories( my_writable=my_writable, order=False ) # Make sure repositories of type tool_dependency_definition are first in the list. for repository in query: if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: @@ -426,13 +427,12 @@ This param can be used as an alternative to the above encoded_ids_to_skip. """ - def handle_repository( trans, repository, results ): + def handle_repository( trans, rmm, repository, results ): log.debug( "Resetting metadata on repository %s" % str( repository.name ) ) repository_id = trans.security.encode_id( repository.id ) try: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) invalid_file_tups, metadata_dict = \ - rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, repository_id ) + rmm.reset_all_metadata_on_repository_in_tool_shed( repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, @@ -451,7 +451,7 @@ status = '%s : %s' % ( str( repository.name ), message ) results[ 'repository_status' ].append( status ) return results - + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) start_time = strftime( "%Y-%m-%d %H:%M:%S" ) results = dict( start_time=start_time, repository_status=[], @@ -473,7 +473,7 @@ my_writable = util.asbool( payload.get( 'my_writable', False ) ) else: my_writable = True - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + query = rmm.get_query_for_setting_metadata_on_repositories( my_writable=my_writable, order=False ) # First reset metadata on all repositories of type repository_dependency_definition. for repository in query: encoded_id = trans.security.encode_id( repository.id ) @@ -481,7 +481,7 @@ log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ ( str( repository.id ), str( encoded_ids_to_skip ) ) ) elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) + results = handle_repository( trans, rmm, repository, results ) # Now reset metadata on all remaining repositories. for repository in query: encoded_id = trans.security.encode_id( repository.id ) @@ -489,7 +489,7 @@ log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ ( str( repository.id ), str( encoded_ids_to_skip ) ) ) elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) + results = handle_repository( trans, rmm, repository, results ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time return json.dumps( results, sort_keys=True, indent=4 ) @@ -511,10 +511,9 @@ results = dict( start_time=start_time, repository_status=[] ) try: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) invalid_file_tups, metadata_dict = \ - rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, - trans.security.encode_id( repository.id ) ) + rmm.reset_all_metadata_on_repository_in_tool_shed( trans.security.encode_id( repository.id ) ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -348,13 +348,16 @@ @web.expose @web.require_admin def reset_metadata_on_selected_repositories_in_tool_shed( self, trans, **kwd ): + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) if 'reset_metadata_on_selected_repositories_button' in kwd: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) - message, status = rmm.reset_metadata_on_selected_repositories( trans.user, **kwd ) + message, status = rmm.reset_metadata_on_selected_repositories( **kwd ) else: message = util.restore_text( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repositories_select_field = suc.build_repository_ids_select_field( trans ) + repositories_select_field = rmm.build_repository_ids_select_field( name='repository_ids', + multiple=True, + display='checkboxes', + my_writable=False ) return trans.fill_template( '/webapps/tool_shed/common/reset_metadata_on_selected_repositories.mako', repositories_select_field=repositories_select_field, message=message, diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/galaxy/webapps/tool_shed/controllers/hg.py --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -48,8 +48,8 @@ repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) ) update_repository( repo, ctx_rev=None ) # Set metadata using the repository files on disk. - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) - error_message, status = rmm.set_repository_metadata( trans.request.host, trans.user, repository ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) + error_message, status = rmm.set_repository_metadata( trans.request.host, repository ) if status == 'ok' and error_message: log.debug( "Successfully reset metadata on repository %s owned by %s, but encountered problem: %s" % \ ( str( repository.name ), str( repository.user.username ), error_message ) ) diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2753,9 +2753,9 @@ def reset_all_metadata( self, trans, id, **kwd ): """Reset all metadata on the complete changelog for a single repository in the tool shed.""" # This method is called only from the ~/templates/webapps/tool_shed/repository/manage_repository.mako template. - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) invalid_file_tups, metadata_dict = \ - rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, id, **kwd ) + rmm.reset_all_metadata_on_repository_in_tool_shed( id, **kwd ) if invalid_file_tups: repository = suc.get_repository_in_tool_shed( trans.app, id ) message = tool_util.generate_message_for_invalid_tools( trans.app, @@ -2774,13 +2774,16 @@ @web.expose def reset_metadata_on_my_writable_repositories_in_tool_shed( self, trans, **kwd ): + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) if 'reset_metadata_on_selected_repositories_button' in kwd: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) - message, status = rmm.reset_metadata_on_selected_repositories( trans.user, **kwd ) + message, status = rmm.reset_metadata_on_selected_repositories( **kwd ) else: message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repositories_select_field = suc.build_repository_ids_select_field( trans, my_writable=True ) + repositories_select_field = rmm.build_repository_ids_select_field( name='repository_ids', + multiple=True, + display='checkboxes', + my_writable=True ) return trans.fill_template( '/webapps/tool_shed/common/reset_metadata_on_selected_repositories.mako', repositories_select_field=repositories_select_field, message=message, @@ -2840,9 +2843,8 @@ if tip == repository.tip( trans.app ): message += 'No changes to repository. ' else: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) status, error_message = rmm.set_repository_metadata_due_to_new_tip( trans.request.host, - trans.user, repository, **kwd ) if error_message: diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -262,10 +262,9 @@ ( len( files_to_remove ), upload_point ) else: message += " %d files were removed from the repository root. " % len( files_to_remove ) - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) status, error_message = \ rmm.set_repository_metadata_due_to_new_tip( trans.request.host, - trans.user, repository, content_alert_str=content_alert_str, **kwd ) diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -727,9 +727,8 @@ results_dict[ 'ok' ] = False results_dict[ 'error_message' ] += error_message try: - rmm = repository_metadata_manager.RepositoryMetadataManager( self.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( self.app, self.user ) status, error_message = rmm.set_repository_metadata_due_to_new_tip( self.host, - self.user, repository, content_alert_str=content_alert_str ) if error_message: diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -33,6 +33,7 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager +from tool_shed.galaxy_install.tools import tool_panel_manager log = logging.getLogger( __name__ ) @@ -535,7 +536,8 @@ self.app.config.shed_tool_data_table_config, persist=True ) if 'tools' in metadata_dict: - tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) + tpm = tool_panel_manager.ToolPanelManager( self.app ) + tool_panel_dict = tpm.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) sample_files = metadata_dict.get( 'sample_files', [] ) tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) @@ -555,16 +557,18 @@ sample_files_copied ) # Copy remaining sample files included in the repository to the ~/tool-data directory of the # local Galaxy instance. - tool_util.copy_sample_files( self.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) - tool_util.add_to_tool_panel( app=self.app, - repository_name=tool_shed_repository.name, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - repository_tools_tups=repository_tools_tups, - owner=tool_shed_repository.owner, - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=True ) + tool_util.copy_sample_files( self.app, + sample_files, + tool_path=tool_path, + sample_files_copied=sample_files_copied ) + tpm.add_to_tool_panel( repository_name=tool_shed_repository.name, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + repository_tools_tups=repository_tools_tups, + owner=tool_shed_repository.owner, + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=True ) if 'data_manager' in metadata_dict: new_data_managers = data_manager_util.install_data_managers( self.app, self.app.config.shed_data_manager_config_file, @@ -644,10 +648,11 @@ tool_shed_url = installation_dict[ 'tool_shed_url' ] # Handle contained tools. if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section_label or tool_panel_section_id ): + tpm = tool_panel_manager.ToolPanelManager( self.app ) tool_panel_section_key, tool_section = \ - tool_util.handle_tool_panel_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + tpm.handle_tool_panel_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) else: tool_panel_section_key = None tool_section = None diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -11,11 +11,11 @@ from tool_shed.util import datatype_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util -from tool_shed.util import tool_util from tool_shed.util import xml_util from galaxy.model.orm import and_ from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import tool_panel_manager log = logging.getLogger( __name__ ) @@ -80,19 +80,19 @@ repository.deleted = False repository.status = self.install_model.ToolShedRepository.installation_status.INSTALLED if repository.includes_tools_for_display_in_tool_panel: + tpm = tool_panel_manager.ToolPanelManager( self.app ) metadata = repository.metadata repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata ) # Reload tools into the appropriate tool panel section. tool_panel_dict = repository.metadata[ 'tool_panel_section' ] - tool_util.add_to_tool_panel( self.app, - repository.name, - repository_clone_url, - repository.installed_changeset_revision, - repository_tools_tups, - repository.owner, - shed_tool_conf, - tool_panel_dict, - new_install=False ) + tpm.add_to_tool_panel( repository.name, + repository_clone_url, + repository.installed_changeset_revision, + repository_tools_tups, + repository.owner, + shed_tool_conf, + tool_panel_dict, + new_install=False ) if repository.includes_data_managers: tp, data_manager_relative_install_dir = repository.get_tool_relative_path( self.app ) # Hack to add repository.name here, which is actually the root of the installed repository diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -3,6 +3,7 @@ from galaxy import util from galaxy.util import inflector +from galaxy.web.form_builder import SelectField from tool_shed.metadata import metadata_generator @@ -19,6 +20,32 @@ super( InstalledRepositoryMetadataManager, self ).__init__( app ) self.app = app + def build_repository_ids_select_field( self, name='repository_ids', multiple=True, display='checkboxes' ): + """Generate the current list of repositories for resetting metadata.""" + repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) + query = self.get_query_for_setting_metadata_on_repositories( order=True ) + for repository in query: + owner = str( repository.owner ) + option_label = '%s (%s)' % ( str( repository.name ), owner ) + option_value = '%s' % self.app.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) + return repositories_select_field + + def get_query_for_setting_metadata_on_repositories( self, order=True ): + """ + Return a query containing repositories for resetting metadata. The order parameter + is used for displaying the list of repositories ordered alphabetically for display on + a page. When called from the Galaxy API, order is False. + """ + if order: + return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ + .filter( self.app.install_model.ToolShedRepository.table.c.uninstalled == False ) \ + .order_by( self.app.install_model.ToolShedRepository.table.c.name, + self.app.install_model.ToolShedRepository.table.c.owner ) + else: + return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ + .filter( self.app.install_model.ToolShedRepository.table.c.uninstalled == False ) + def reset_all_metadata_on_installed_repository( self, id ): """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" invalid_file_tups = [] diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -5,13 +5,14 @@ from tool_shed.galaxy_install import install_manager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import shed_util_common as suc from tool_shed.util import repository_maintenance_util from tool_shed.util import tool_dependency_util -from tool_shed.util import tool_util + class RepairRepositoryManager(): @@ -119,10 +120,11 @@ tool_panel_section_id = section_dict[ 'id' ] tool_panel_section_name = section_dict[ 'name' ] if tool_panel_section_id: + tpm = tool_panel_manager.ToolPanelManager( self.app ) tool_panel_section_key, tool_panel_section = \ - tool_util.get_or_create_tool_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=tool_panel_section_name ) + tpm.get_or_create_tool_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=tool_panel_section_name ) else: tool_dependencies = None repo_info_dict = repository_maintenance_util.create_repo_info_dict( app=self.app, @@ -156,15 +158,17 @@ log.debug( error_message ) repair_dict [ repository.name ] = error_message elif repository.status not in [ self.app.install_model.ToolShedRepository.installation_status.INSTALLED ]: - shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) + shed_tool_conf, tool_path, relative_install_dir = \ + suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) # Reset the repository attributes to the New state for installation. if metadata: + tpm = tool_panel_manager.ToolPanelManager( self.app ) tool_section, tool_panel_section_key = \ - tool_util.handle_tool_panel_selection( self.app.toolbox, - metadata, - no_changes_checked=True, - tool_panel_section_id=None, - new_tool_panel_section_label=None ) + tpm.handle_tool_panel_selection( self.app.toolbox, + metadata, + no_changes_checked=True, + tool_panel_section_id=None, + new_tool_panel_section_label=None ) else: # The tools will be loaded outside of any sections in the tool panel. tool_panel_section_key = None diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -10,11 +10,12 @@ from galaxy.util import asbool +from tool_shed.galaxy_install.tools import tool_panel_manager + from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import encoding_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import tool_util log = logging.getLogger( __name__ ) @@ -216,19 +217,20 @@ if can_update_db_record: # The database record for the tool shed repository currently being processed can be updated. # Get the repository metadata to see where it was previously located in the tool panel. + tpm = tool_panel_manager.ToolPanelManager( self.app ) if repository_db_record and repository_db_record.metadata: tool_section, tool_panel_section_key = \ - tool_util.handle_tool_panel_selection( toolbox=self.app.toolbox, - metadata=repository_db_record.metadata, - no_changes_checked=no_changes_checked, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + tpm.handle_tool_panel_selection( toolbox=self.app.toolbox, + metadata=repository_db_record.metadata, + no_changes_checked=no_changes_checked, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) else: # We're installing a new tool shed repository that does not yet have a database record. tool_panel_section_key, tool_section = \ - tool_util.handle_tool_panel_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + tpm.handle_tool_panel_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) tool_shed_repository = \ suc.create_or_update_tool_shed_repository( app=self.app, name=name, diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -14,6 +14,7 @@ from tool_shed.galaxy_install import install_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager +from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.util import basic_util from tool_shed.util import common_util @@ -30,23 +31,31 @@ class ToolMigrationManager( object ): - def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ): + def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, + install_dependencies ): """ - Check tool settings in tool_shed_install_config and install all repositories that are not already installed. The tool - panel configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. + Check tool settings in tool_shed_install_config and install all repositories + that are not already installed. The tool panel configuration file is the received + migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. """ self.app = app self.toolbox = self.app.toolbox self.migrated_tools_config = migrated_tools_config - # If install_dependencies is True but tool_dependency_dir is not set, do not attempt to install but print informative error message. + self.tpm = tool_panel_manager.ToolPanelManager( self.app ) + # If install_dependencies is True but tool_dependency_dir is not set, do not attempt + # to install but print informative error message. if install_dependencies and app.config.tool_dependency_dir is None: - message = 'You are attempting to install tool dependencies but do not have a value for "tool_dependency_dir" set in your universe_wsgi.ini ' - message += 'file. Set this location value to the path where you want tool dependencies installed and rerun the migration script.' + message = 'You are attempting to install tool dependencies but do not have a value ' + message += 'for "tool_dependency_dir" set in your universe_wsgi.ini file. Set this ' + message += 'location value to the path where you want tool dependencies installed and ' + message += 'rerun the migration script.' raise Exception( message ) - # Get the local non-shed related tool panel configs (there can be more than one, and the default name is tool_conf.xml). + # Get the local non-shed related tool panel configs (there can be more than one, and the + # default name is tool_conf.xml). self.proprietary_tool_confs = self.non_shed_tool_panel_configs self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number ) - # Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config. + # Set the location where the repositories will be installed by retrieving the tool_path + # setting from migrated_tools_config. tree, error_message = xml_util.parse_xml( migrated_tools_config ) if tree is None: print error_message @@ -66,21 +75,24 @@ self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config ) - # Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in - # tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk. - # The default behavior is that the tool shed is down. + # Since tool migration scripts can be executed any number of times, we need to + # make sure the appropriate tools are defined in tool_conf.xml. If no tools + # associated with the migration stage are defined, no repositories will be installed + # on disk. The default behavior is that the tool shed is down. tool_shed_accessible = False tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app ) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} - tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, - tool_panel_configs, - latest_migration_script_number ) + tool_shed_accessible, missing_tool_configs_dict = \ + common_util.check_for_missing_tools( app, + tool_panel_configs, + latest_migration_script_number ) else: - # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but - # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in - # the database. + # It doesn't matter if the tool shed is accessible since there are no migrated + # tools defined in the local Galaxy instance, but we have to set the value of + # tool_shed_accessible to True so that the value of migrate_tools.version can + # be correctly set in the database. tool_shed_accessible = True missing_tool_configs_dict = odict() if tool_shed_accessible: @@ -93,15 +105,18 @@ if missing_tool_configs_dict: for proprietary_tool_conf in self.proprietary_tool_confs: # Create a backup of the tool configuration in the un-migrated state. - shutil.copy( proprietary_tool_conf, '%s-pre-stage-%04d' % ( proprietary_tool_conf, latest_migration_script_number ) ) + shutil.copy( proprietary_tool_conf, '%s-pre-stage-%04d' % ( proprietary_tool_conf, + latest_migration_script_number ) ) for repository_elem in root: # Make sure we have a valid repository tag. if self.__is_valid_repository_tag( repository_elem ): - # Get all repository dependencies for the repository defined by the current repository_elem. Repository dependency - # definitions contained in tool shed repositories with migrated tools must never define a relationship to a repository - # dependency that contains a tool. The repository dependency can only contain items that are not loaded into the Galaxy - # tool panel (e.g., tool dependency definitions, custom datatypes, etc). This restriction must be followed down the - # entire dependency hierarchy. + # Get all repository dependencies for the repository defined by the + # current repository_elem. Repository dependency definitions contained + # in tool shed repositories with migrated tools must never define a + # relationship to a repository dependency that contains a tool. The + # repository dependency can only contain items that are not loaded into + # the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes, + # etc). This restriction must be followed down the entire dependency hierarchy. name = repository_elem.get( 'name' ) changeset_revision = repository_elem.get( 'changeset_revision' ) tool_shed_accessible, repository_dependencies_dict = \ @@ -110,30 +125,38 @@ name, self.repository_owner, changeset_revision ) - # Make sure all repository dependency records exist (as tool_shed_repository table rows) in the Galaxy database. - created_tool_shed_repositories = self.create_or_update_tool_shed_repository_records( name, - changeset_revision, - repository_dependencies_dict ) + # Make sure all repository dependency records exist (as tool_shed_repository + # table rows) in the Galaxy database. + created_tool_shed_repositories = \ + self.create_or_update_tool_shed_repository_records( name, + changeset_revision, + repository_dependencies_dict ) # Order the repositories for proper installation. This process is similar to the # process used when installing tool shed repositories, but does not handle managing # tool panel sections and other components since repository dependency definitions # contained in tool shed repositories with migrated tools must never define a relationship # to a repository dependency that contains a tool. - ordered_tool_shed_repositories = self.order_repositories_for_installation( created_tool_shed_repositories, - repository_dependencies_dict ) + ordered_tool_shed_repositories = \ + self.order_repositories_for_installation( created_tool_shed_repositories, + repository_dependencies_dict ) for tool_shed_repository in ordered_tool_shed_repositories: - is_repository_dependency = self.__is_repository_dependency( name, changeset_revision, tool_shed_repository ) + is_repository_dependency = self.__is_repository_dependency( name, + changeset_revision, + tool_shed_repository ) self.install_repository( repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency=is_repository_dependency ) else: - message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number ) - message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names ) + message = "\nNo tools associated with migration stage %s are defined in your " % \ + str( latest_migration_script_number ) + message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \ + ( plural, file_names ) print message else: - message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number ) + message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \ + str( latest_migration_script_number ) message += "Try again later.\n" print message @@ -354,16 +377,21 @@ repo_install_dir = relative_install_dir if not is_repository_dependency: for tool_elem in repository_elem: - # The tool_elem looks something like this: <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> + # The tool_elem looks something like this: + # <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> tool_config = tool_elem.get( 'file' ) guid = self.get_guid( repository_clone_url, relative_install_dir, tool_config ) # See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems. is_displayed, tool_sections = self.get_containing_tool_sections( tool_config ) if is_displayed: tool_panel_dict_for_tool_config = \ - tool_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections ) + self.tpm.generate_tool_panel_dict_for_tool_config( guid, + tool_config, + tool_sections=tool_sections ) # The tool-panel_dict has the following structure. - # {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, + # {<Tool guid> : [{ tool_config : <tool_config_file>, + # id: <ToolSection id>, + # version : <ToolSection version>, # name : <TooSection name>}]} for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[ k ] = v @@ -433,15 +461,14 @@ tool_path=self.tool_path, sample_files_copied=sample_files_copied ) if not is_repository_dependency: - tool_util.add_to_tool_panel( self.app, - tool_shed_repository.name, - repository_clone_url, - tool_shed_repository.installed_changeset_revision, - repository_tools_tups, - self.repository_owner, - self.migrated_tools_config, - tool_panel_dict=tool_panel_dict_for_display, - new_install=True ) + self.tpm.add_to_tool_panel( tool_shed_repository.name, + repository_clone_url, + tool_shed_repository.installed_changeset_revision, + repository_tools_tups, + self.repository_owner, + self.migrated_tools_config, + tool_panel_dict=tool_panel_dict_for_display, + new_install=True ) if install_dependencies and tool_dependencies and has_tool_dependencies: # Install tool dependencies. suc.update_tool_shed_repository_status( self.app, diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -0,0 +1,465 @@ +import logging +import os + +import galaxy.tools +from galaxy.tools.search import ToolBoxSearch +from xml.etree import ElementTree as XmlET + +from tool_shed.util import basic_util +from tool_shed.util import common_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import xml_util + +log = logging.getLogger( __name__ ) + + +class ToolPanelManager( object ): + + def __init__( self, app ): + self.app = app + + def add_to_shed_tool_config( self, shed_tool_conf_dict, elem_list ): + # A tool shed repository is being installed so change the shed_tool_conf file. Parse the + # config file to generate the entire list of config_elems instead of using the in-memory list + # since it will be a subset of the entire list if one or more repositories have been deactivated. + shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] + tool_path = shed_tool_conf_dict[ 'tool_path' ] + config_elems = [] + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + config_elems.append( elem ) + # Add the elements to the in-memory list of config_elems. + for elem_entry in elem_list: + config_elems.append( elem_entry ) + # Persist the altered shed_tool_config file. + suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path ) + + def add_to_tool_panel( self, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner, + shed_tool_conf, tool_panel_dict, new_install=True ): + """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" + # We need to change the in-memory version and the file system version of the shed_tool_conf file. + index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + tool_path = shed_tool_conf_dict[ 'tool_path' ] + # Generate the list of ElementTree Element objects for each section or tool. + elem_list = self.generate_tool_panel_elem_list( repository_name, + repository_clone_url, + changeset_revision, + tool_panel_dict, + repository_tools_tups, + owner=owner ) + if new_install: + # Add the new elements to the shed_tool_conf file on disk. + self.add_to_shed_tool_config( shed_tool_conf_dict, elem_list ) + # Use the new elements to add entries to the + config_elems = shed_tool_conf_dict[ 'config_elems' ] + for config_elem in elem_list: + # Add the new elements to the in-memory list of config_elems. + config_elems.append( config_elem ) + # Load the tools into the in-memory tool panel. + if config_elem.tag == 'section': + self.app.toolbox.load_section_tag_set( config_elem, tool_path, load_panel_dict=True ) + elif config_elem.tag == 'workflow': + self.app.toolbox.load_workflow_tag_set( config_elem, + self.app.toolbox.tool_panel, + self.app.toolbox.integrated_tool_panel, + load_panel_dict=True ) + elif config_elem.tag == 'tool': + guid = config_elem.get( 'guid' ) + self.app.toolbox.load_tool_tag_set( config_elem, + self.app.toolbox.tool_panel, + self.app.toolbox.integrated_tool_panel, + tool_path, + load_panel_dict=True, + guid=guid ) + # Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict. + shed_tool_conf_dict[ 'config_elems' ] = config_elems + self.app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict + if self.app.config.update_integrated_tool_panel: + # Write the current in-memory version of the integrated_tool_panel.xml file to disk. + self.app.toolbox.write_integrated_tool_panel_config_file() + self.app.toolbox_search = ToolBoxSearch( self.app.toolbox ) + + def generate_tool_panel_dict_for_new_install( self, tool_dicts, tool_section=None ): + """ + When installing a repository that contains tools, all tools must currently be defined + within the same tool section in the tool panel or outside of any sections. + """ + tool_panel_dict = {} + if tool_section: + section_id = tool_section.id + section_name = tool_section.name + section_version = tool_section.version or '' + else: + section_id = '' + section_name = '' + section_version = '' + for tool_dict in tool_dicts: + if tool_dict.get( 'add_to_tool_panel', True ): + guid = tool_dict[ 'guid' ] + tool_config = tool_dict[ 'tool_config' ] + tool_section_dict = dict( tool_config=tool_config, id=section_id, name=section_name, version=section_version ) + if guid in tool_panel_dict: + tool_panel_dict[ guid ].append( tool_section_dict ) + else: + tool_panel_dict[ guid ] = [ tool_section_dict ] + return tool_panel_dict + + def generate_tool_panel_dict_for_tool_config( self, guid, tool_config, tool_sections=None ): + """ + Create a dictionary of the following type for a single tool config file name. + The intent is to call this method for every tool config in a repository and + append each of these as entries to a tool panel dictionary for the repository. + This enables each tool to be loaded into a different section in the tool panel. + {<Tool guid> : + [{ tool_config : <tool_config_file>, + id: <ToolSection id>, + version : <ToolSection version>, + name : <TooSection name>}]} + """ + tool_panel_dict = {} + file_name = basic_util.strip_path( tool_config ) + tool_section_dicts = self. generate_tool_section_dicts( tool_config=file_name, + tool_sections=tool_sections ) + tool_panel_dict[ guid ] = tool_section_dicts + return tool_panel_dict + + def generate_tool_panel_elem_list( self, repository_name, repository_clone_url, changeset_revision, + tool_panel_dict, repository_tools_tups, owner='' ): + """Generate a list of ElementTree Element objects for each section or tool.""" + elem_list = [] + tool_elem = None + cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) + if not owner: + owner = suc.get_repository_owner( cleaned_repository_clone_url ) + tool_shed = cleaned_repository_clone_url.split( '/repos/' )[ 0 ].rstrip( '/' ) + for guid, tool_section_dicts in tool_panel_dict.items(): + for tool_section_dict in tool_section_dicts: + tool_section = None + inside_section = False + section_in_elem_list = False + if tool_section_dict[ 'id' ]: + inside_section = True + # Create a new section element only if we haven't already created it. + for index, elem in enumerate( elem_list ): + if elem.tag == 'section': + section_id = elem.get( 'id', None ) + if section_id == tool_section_dict[ 'id' ]: + section_in_elem_list = True + tool_section = elem + break + if tool_section is None: + tool_section = self.generate_tool_section_element_from_dict( tool_section_dict ) + # Find the tuple containing the current guid from the list of repository_tools_tups. + for repository_tool_tup in repository_tools_tups: + tool_file_path, tup_guid, tool = repository_tool_tup + if tup_guid == guid: + break + if inside_section: + tool_elem = suc.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + tool_section ) + else: + tool_elem = suc.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + None ) + if inside_section: + if section_in_elem_list: + elem_list[ index ] = tool_section + else: + elem_list.append( tool_section ) + else: + elem_list.append( tool_elem ) + return elem_list + + def generate_tool_section_dicts( self, tool_config=None, tool_sections=None ): + tool_section_dicts = [] + if tool_config is None: + tool_config = '' + if tool_sections: + for tool_section in tool_sections: + # The value of tool_section will be None if the tool is displayed outside + # of any sections in the tool panel. + if tool_section: + section_id = tool_section.id or '' + section_version = tool_section.version or '' + section_name = tool_section.name or '' + else: + section_id = '' + section_version = '' + section_name = '' + tool_section_dicts.append( dict( tool_config=tool_config, + id=section_id, + version=section_version, + name=section_name ) ) + else: + tool_section_dicts.append( dict( tool_config=tool_config, id='', version='', name='' ) ) + return tool_section_dicts + + def generate_tool_section_element_from_dict( self, tool_section_dict ): + # The value of tool_section_dict looks like the following. + # { id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>} + if tool_section_dict[ 'id' ]: + # Create a new tool section. + tool_section = XmlET.Element( 'section' ) + tool_section.attrib[ 'id' ] = tool_section_dict[ 'id' ] + tool_section.attrib[ 'name' ] = tool_section_dict[ 'name' ] + tool_section.attrib[ 'version' ] = tool_section_dict[ 'version' ] + else: + tool_section = None + return tool_section + + def get_or_create_tool_section( self, toolbox, tool_panel_section_id, new_tool_panel_section_label=None ): + tool_panel_section_key = str( tool_panel_section_id ) + if tool_panel_section_key in toolbox.tool_panel: + # Appending a tool to an existing section in toolbox.tool_panel + tool_section = toolbox.tool_panel[ tool_panel_section_key ] + log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) + else: + # Appending a new section to toolbox.tool_panel + if new_tool_panel_section_label is None: + # This might add an ugly section label to the tool panel, but, oh well... + new_tool_panel_section_label = tool_panel_section_id + elem = XmlET.Element( 'section' ) + elem.attrib[ 'name' ] = new_tool_panel_section_label + elem.attrib[ 'id' ] = tool_panel_section_id + elem.attrib[ 'version' ] = '' + tool_section = galaxy.tools.ToolSection( elem ) + toolbox.tool_panel[ tool_panel_section_key ] = tool_section + log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) + return tool_panel_section_key, tool_section + + def handle_tool_panel_section( self, toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ): + """Return a ToolSection object retrieved from the current in-memory tool_panel.""" + # If tool_panel_section_id is received, the section exists in the tool panel. In this + # case, the value of the received tool_panel_section_id must be the id retrieved from a + # tool panel config (e.g., tool_conf.xml, which may have getext). If new_tool_panel_section_label + # is received, a new section will be added to the tool panel. + if new_tool_panel_section_label: + section_id = str( new_tool_panel_section_label.lower().replace( ' ', '_' ) ) + tool_panel_section_key, tool_section = \ + self.get_or_create_tool_section( toolbox, + tool_panel_section_id=section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) + elif tool_panel_section_id: + tool_panel_section_key = str( tool_panel_section_id ) + tool_section = toolbox.tool_panel[ tool_panel_section_key ] + else: + return None, None + return tool_panel_section_key, tool_section + + def handle_tool_panel_selection( self, toolbox, metadata, no_changes_checked, tool_panel_section_id, + new_tool_panel_section_label ): + """ + Handle the selected tool panel location for loading tools included in tool shed + repositories when installing or reinstalling them. + """ + # Get the location in the tool panel in which each tool was originally loaded. + tool_section = None + tool_panel_section_key = None + if 'tools' in metadata: + # This forces everything to be loaded into the same section (or no section) + # in the tool panel. + if no_changes_checked: + # Make sure the no_changes check box overrides the new_tool_panel_section_label + # if the user checked the check box and entered something into the field. + new_tool_panel_section_label = None + if 'tool_panel_section' in metadata: + tool_panel_dict = metadata[ 'tool_panel_section' ] + if not tool_panel_dict: + tool_panel_dict = self.generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) + else: + tool_panel_dict = self.generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) + if tool_panel_dict: + # The tool_panel_dict is empty when tools exist but are not installed into a tool panel section. + tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] + tool_section_dict = tool_section_dicts[ 0 ] + original_section_id = tool_section_dict[ 'id' ] + original_section_name = tool_section_dict[ 'name' ] + if original_section_id: + tool_panel_section_key, tool_section = \ + self.get_or_create_tool_section( toolbox, + tool_panel_section_id=original_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) + else: + # The user elected to change the tool panel section to contain the tools. + tool_panel_section_key, tool_section = \ + self.handle_tool_panel_section( toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) + return tool_section, tool_panel_section_key + + def remove_from_shed_tool_config( self, shed_tool_conf_dict, guids_to_remove ): + # A tool shed repository is being uninstalled so change the shed_tool_conf file. + # Parse the config file to generate the entire list of config_elems instead of + # using the in-memory list since it will be a subset of the entire list if one + # or more repositories have been deactivated. + shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] + tool_path = shed_tool_conf_dict[ 'tool_path' ] + config_elems = [] + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + config_elems.append( elem ) + config_elems_to_remove = [] + for config_elem in config_elems: + if config_elem.tag == 'section': + tool_elems_to_remove = [] + for tool_elem in config_elem: + if tool_elem.get( 'guid' ) in guids_to_remove: + tool_elems_to_remove.append( tool_elem ) + for tool_elem in tool_elems_to_remove: + # Remove all of the appropriate tool sub-elements from the section element. + config_elem.remove( tool_elem ) + if len( config_elem ) < 1: + # Keep a list of all empty section elements so they can be removed. + config_elems_to_remove.append( config_elem ) + elif config_elem.tag == 'tool': + if config_elem.get( 'guid' ) in guids_to_remove: + config_elems_to_remove.append( config_elem ) + for config_elem in config_elems_to_remove: + config_elems.remove( config_elem ) + # Persist the altered in-memory version of the tool config. + suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path ) + + def remove_from_tool_panel( self, repository, shed_tool_conf, uninstall ): + """ + A tool shed repository is being deactivated or uninstalled, so handle tool panel + alterations accordingly. + """ + # Determine where the tools are currently defined in the tool panel and store this + # information so the tools can be displayed in the same way when the repository is + # activated or reinstalled. + tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( self.app, repository ) + repository.metadata[ 'tool_panel_section' ] = tool_panel_dict + self.app.install_model.context.add( repository ) + self.app.install_model.context.flush() + # Create a list of guids for all tools that will be removed from the in-memory tool panel + # and config file on disk. + guids_to_remove = [ k for k in tool_panel_dict.keys() ] + # Remove the tools from the toolbox's tools_by_id dictionary. + for guid_to_remove in guids_to_remove: + if guid_to_remove in self.app.toolbox.tools_by_id: + del self.app.toolbox.tools_by_id[ guid_to_remove ] + index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + if uninstall: + # Remove from the shed_tool_conf file on disk. + self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) + config_elems = shed_tool_conf_dict[ 'config_elems' ] + config_elems_to_remove = [] + for config_elem in config_elems: + if config_elem.tag == 'section': + # Get the section key for the in-memory tool panel. + section_key = str( config_elem.get( "id" ) ) + # Generate the list of tool elements to remove. + tool_elems_to_remove = [] + for tool_elem in config_elem: + if tool_elem.get( 'guid' ) in guids_to_remove: + tool_elems_to_remove.append( tool_elem ) + for tool_elem in tool_elems_to_remove: + if tool_elem in config_elem: + # Remove the tool sub-element from the section element. + config_elem.remove( tool_elem ) + # Remove the tool from the section in the in-memory tool panel. + if section_key in self.app.toolbox.tool_panel: + tool_section = self.app.toolbox.tool_panel[ section_key ] + guid = tool_elem.get( 'guid' ) + tool_key = 'tool_%s' % str( guid ) + # Get the list of versions of this tool that are currently available in the toolbox. + available_tool_versions = self.app.toolbox.get_loaded_tools_by_lineage( guid ) + if tool_key in tool_section.elems: + if available_tool_versions: + available_tool_versions.reverse() + replacement_tool_key = None + replacement_tool_version = None + # Since we are going to remove the tool from the section, replace it with the + # newest loaded version of the tool. + for available_tool_version in available_tool_versions: + available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() + if available_tool_version.id in tool_section.elems.keys() or section_key == available_tool_section_id: + replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) + replacement_tool_version = available_tool_version + break + if replacement_tool_key and replacement_tool_version: + # Get the index of the tool_key in the tool_section. + for tool_section_elems_index, key in enumerate( tool_section.elems.keys() ): + if key == tool_key: + break + # Remove the tool from the tool section. + del tool_section.elems[ tool_key ] + # Add the replacement tool at the same location in the tool section. + tool_section.elems.insert( tool_section_elems_index, + replacement_tool_key, + replacement_tool_version ) + else: + del tool_section.elems[ tool_key ] + else: + del tool_section.elems[ tool_key ] + if uninstall: + # Remove the tool from the section in the in-memory integrated tool panel. + if section_key in self.app.toolbox.integrated_tool_panel: + tool_section = self.app.toolbox.integrated_tool_panel[ section_key ] + tool_key = 'tool_%s' % str( tool_elem.get( 'guid' ) ) + if tool_key in tool_section.elems: + del tool_section.elems[ tool_key ] + if len( config_elem ) < 1: + # Keep a list of all empty section elements so they can be removed. + config_elems_to_remove.append( config_elem ) + elif config_elem.tag == 'tool': + guid = config_elem.get( 'guid' ) + if guid in guids_to_remove: + tool_key = 'tool_%s' % str( config_elem.get( 'guid' ) ) + # Get the list of versions of this tool that are currently available in the toolbox. + available_tool_versions = self.app.toolbox.get_loaded_tools_by_lineage( guid ) + if tool_key in self.app.toolbox.tool_panel: + if available_tool_versions: + available_tool_versions.reverse() + replacement_tool_key = None + replacement_tool_version = None + # Since we are going to remove the tool from the section, replace it with + # the newest loaded version of the tool. + for available_tool_version in available_tool_versions: + available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() + if available_tool_version.id in self.app.toolbox.tool_panel.keys() or not available_tool_section_id: + replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) + replacement_tool_version = available_tool_version + break + if replacement_tool_key and replacement_tool_version: + # Get the index of the tool_key in the tool_section. + for tool_panel_index, key in enumerate( self.app.toolbox.tool_panel.keys() ): + if key == tool_key: + break + # Remove the tool from the tool panel. + del self.app.toolbox.tool_panel[ tool_key ] + # Add the replacement tool at the same location in the tool panel. + self.app.toolbox.tool_panel.insert( tool_panel_index, + replacement_tool_key, + replacement_tool_version ) + else: + del self.app.toolbox.tool_panel[ tool_key ] + else: + del self.app.toolbox.tool_panel[ tool_key ] + if uninstall: + if tool_key in self.app.toolbox.integrated_tool_panel: + del self.app.toolbox.integrated_tool_panel[ tool_key ] + config_elems_to_remove.append( config_elem ) + for config_elem in config_elems_to_remove: + # Remove the element from the in-memory list of elements. + config_elems.remove( config_elem ) + # Update the config_elems of the in-memory shed_tool_conf_dict. + shed_tool_conf_dict[ 'config_elems' ] = config_elems + self.app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict + self.app.toolbox_search = ToolBoxSearch( self.app.toolbox ) + if uninstall and self.app.config.update_integrated_tool_panel: + # Write the current in-memory version of the integrated_tool_panel.xml file to disk. + self.app.toolbox.write_integrated_tool_panel_config_file() diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/metadata/repository_metadata_manager.py --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -2,11 +2,13 @@ import os import tempfile +from galaxy import util from galaxy.util import inflector -from galaxy import util +from galaxy.web.form_builder import SelectField from tool_shed.metadata import metadata_generator from tool_shed.repository_types.metadata import TipOnly +from tool_shed.repository_types import util as rt_util from tool_shed.util import basic_util from tool_shed.util import common_util @@ -20,9 +22,10 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): - def __init__( self, app ): + def __init__( self, app, user ): super( RepositoryMetadataManager, self ).__init__( app ) self.app = app + self.user = user # Repository metadata comparisons for changeset revisions. self.EQUAL = 'equal' self.NO_METADATA = 'no metadata' @@ -47,6 +50,18 @@ self.sa_session.add( repository_metadata ) self.sa_session.flush() + def build_repository_ids_select_field( self, name='repository_ids', multiple=True, display='checkboxes', + my_writable=False ): + """Generate the current list of repositories for resetting metadata.""" + repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) + query = self.get_query_for_setting_metadata_on_repositories( my_writable=my_writable, order=True ) + for repository in query: + owner = str( repository.user.username ) + option_label = '%s (%s)' % ( str( repository.name ), owner ) + option_value = '%s' % self.app.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) + return repositories_select_field + def clean_repository_metadata( self, id, changeset_revisions ): # Delete all repository_metadata records associated with the repository that have # a changeset_revision that is not in changeset_revisions. We sometimes see multiple @@ -458,6 +473,54 @@ # The tool did not change through all of the changeset revisions. return old_id + def get_query_for_setting_metadata_on_repositories( self, my_writable=False, order=True ): + """ + Return a query containing repositories for resetting metadata. The order parameter + is used for displaying the list of repositories ordered alphabetically for display on + a page. When called from the Tool Shed API, order is False. + """ + # When called from the Tool Shed API, the metadata is reset on all repositories of types + # repository_suite_definition and tool_dependency_definition in addition to other selected + # repositories. + if my_writable: + username = self.user.username + clause_list = [] + for repository in self.sa_session.query( self.app.model.Repository ) \ + .filter( self.app.model.Repository.table.c.deleted == False ): + # Always reset metadata on all repositories of types repository_suite_definition and + # tool_dependency_definition. + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: + clause_list.append( self.app.model.Repository.table.c.id == repository.id ) + else: + allow_push = repository.allow_push( self.app ) + if allow_push: + # Include all repositories that are writable by the current user. + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + clause_list.append( self.app.model.Repository.table.c.id == repository.id ) + if clause_list: + if order: + return self.sa_session.query( self.app.model.Repository ) \ + .filter( or_( *clause_list ) ) \ + .order_by( self.app.model.Repository.table.c.name, + self.app.model.Repository.table.c.user_id ) + else: + return self.sa_session.query( self.app.model.Repository ) \ + .filter( or_( *clause_list ) ) + else: + # Return an empty query. + return self.sa_session.query( self.app.model.Repository ) \ + .filter( self.app.model.Repository.table.c.id == -1 ) + else: + if order: + return self.sa_session.query( self.app.model.Repository ) \ + .filter( self.app.model.Repository.table.c.deleted == False ) \ + .order_by( self.app.model.Repository.table.c.name, + self.app.model.Repository.table.c.user_id ) + else: + return self.sa_session.query( self.app.model.Repository ) \ + .filter( self.app.model.Repository.table.c.deleted == False ) + def new_datatypes_metadata_required( self, repository_metadata, metadata_dict ): """ Compare the last saved metadata for each datatype in the repository with the new metadata @@ -732,13 +795,13 @@ # repository_metadata table record is not needed. return False - def reset_all_metadata_on_repository_in_tool_shed( self, user, id ): + def reset_all_metadata_on_repository_in_tool_shed( self, id ): """Reset all metadata on a single repository in a tool shed.""" repository = suc.get_repository_in_tool_shed( self.app, id ) log.debug( "Resetting all metadata on repository: %s" % repository.name ) repo_dir = repository.repo_path( self.app ) repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( self.user, repository ) # The list of changeset_revisions refers to repository_metadata records that have been created # or updated. When the following loop completes, we'll delete all repository_metadata records # for this repository that do not have a changeset_revision value in this list. @@ -875,7 +938,7 @@ self.sa_session.add( repository_metadata ) self.sa_session.flush() - def reset_metadata_on_selected_repositories( self, user, **kwd ): + def reset_metadata_on_selected_repositories( self, **kwd ): """ Inspect the repository changelog to reset metadata for all appropriate changeset revisions. This method is called from both Galaxy and the Tool Shed. @@ -892,7 +955,7 @@ repository = suc.get_repository_in_tool_shed( self.app, repository_id ) owner = str( repository.user.username ) invalid_file_tups, metadata_dict = \ - self.reset_all_metadata_on_repository_in_tool_shed( user, repository_id ) + self.reset_all_metadata_on_repository_in_tool_shed( self.user, repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( self.app, invalid_file_tups, @@ -918,7 +981,7 @@ status = 'error' return message, status - def set_repository_metadata( self, host, user, repository, content_alert_str='', **kwd ): + def set_repository_metadata( self, host, repository, content_alert_str='', **kwd ): """ Set metadata using the repository's current disk files, returning specific error messages (if any) to alert the repository owner that the changeset has problems. @@ -926,7 +989,7 @@ message = '' status = 'done' encoded_id = self.app.security.encode_id( repository.id ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( self.user, repository ) repo_dir = repository.repo_path( self.app ) repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False ) metadata_dict, invalid_file_tups = \ @@ -1027,10 +1090,9 @@ tool_util.reset_tool_data_tables( self.app ) return message, status - def set_repository_metadata_due_to_new_tip( self, host, user, repository, content_alert_str=None, **kwd ): + def set_repository_metadata_due_to_new_tip( self, host, repository, content_alert_str=None, **kwd ): """Set metadata on the repository tip in the tool shed.""" error_message, status = self.set_repository_metadata( host, - user, repository, content_alert_str=content_alert_str, **kwd ) diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -84,20 +84,6 @@ '${host}' """ -def build_repository_ids_select_field( trans, name='repository_ids', multiple=True, display='checkboxes', my_writable=False ): - """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" - repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) - query = get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=True ) - for repository in query: - if trans.webapp.name == 'tool_shed': - owner = str( repository.user.username ) - else: - owner = str( repository.owner ) - option_label = '%s (%s)' % ( str( repository.name ), owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - return repositories_select_field - def build_tool_dependencies_select_field( app, tool_shed_repository, name, multiple=True, display='checkboxes', uninstalled_only=False ): """Method called from Galaxy to generate the current list of tool dependency ids for an installed tool shed repository.""" tool_dependencies_select_field = SelectField( name=name, multiple=multiple, display=display ) @@ -645,66 +631,6 @@ prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids return prior_import_or_install_required_dict -def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=True ): - """ - Return a query containing repositories for resetting metadata. This method is called from both the - Tool Shed and Galaxy. The my_writable parameter is ignored unless called from the Tool Shed, and the - order parameter is used for displaying the list of repositories ordered alphabetically for display on - a page. When called from either the Tool Shed or Galaxy API, order is False. - """ - if trans.webapp.name == 'tool_shed': - # When called from the Tool Shed API, the metadata is reset on all repositories of types - # repository_suite_definition and tool_dependency_definition in addition to other selected - # repositories. - if my_writable: - username = trans.user.username - clause_list = [] - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ): - # Always reset metadata on all repositories of types repository_suite_definition and - # tool_dependency_definition. - if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: - clause_list.append( trans.model.Repository.table.c.id == repository.id ) - else: - allow_push = repository.allow_push( trans.app ) - if allow_push: - # Include all repositories that are writable by the current user. - allow_push_usernames = allow_push.split( ',' ) - if username in allow_push_usernames: - clause_list.append( trans.model.Repository.table.c.id == repository.id ) - if clause_list: - if order: - return trans.sa_session.query( trans.model.Repository ) \ - .filter( or_( *clause_list ) ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ) - else: - return trans.sa_session.query( trans.model.Repository ) \ - .filter( or_( *clause_list ) ) - else: - # Return an empty query. - return trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.id == -1 ) - else: - if order: - return trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ) - else: - return trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) - else: - # We're in Galaxy. - if order: - return trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .filter( trans.install_model.ToolShedRepository.table.c.uninstalled == False ) \ - .order_by( trans.install_model.ToolShedRepository.table.c.name, - trans.install_model.ToolShedRepository.table.c.owner ) - else: - return trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .filter( trans.install_model.ToolShedRepository.table.c.uninstalled == False ) - def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: diff -r 972a80267009126e558ba14142df48b897db48c8 -r e9de99ef77105d221df55771724ca56b86485a4b lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -19,69 +19,6 @@ log = logging.getLogger( __name__ ) -def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ): - # A tool shed repository is being installed so change the shed_tool_conf file. Parse the - # config file to generate the entire list of config_elems instead of using the in-memory list - # since it will be a subset of the entire list if one or more repositories have been deactivated. - shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] - tool_path = shed_tool_conf_dict[ 'tool_path' ] - config_elems = [] - tree, error_message = xml_util.parse_xml( shed_tool_conf ) - if tree: - root = tree.getroot() - for elem in root: - config_elems.append( elem ) - # Add the elements to the in-memory list of config_elems. - for elem_entry in elem_list: - config_elems.append( elem_entry ) - # Persist the altered shed_tool_config file. - suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) - -def add_to_tool_panel( app, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner, - shed_tool_conf, tool_panel_dict, new_install=True ): - """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" - # We need to change the in-memory version and the file system version of the shed_tool_conf file. - index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( app, shed_tool_conf ) - tool_path = shed_tool_conf_dict[ 'tool_path' ] - # Generate the list of ElementTree Element objects for each section or tool. - elem_list = generate_tool_panel_elem_list( repository_name, - repository_clone_url, - changeset_revision, - tool_panel_dict, - repository_tools_tups, - owner=owner ) - if new_install: - # Add the new elements to the shed_tool_conf file on disk. - add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ) - # Use the new elements to add entries to the - config_elems = shed_tool_conf_dict[ 'config_elems' ] - for config_elem in elem_list: - # Add the new elements to the in-memory list of config_elems. - config_elems.append( config_elem ) - # Load the tools into the in-memory tool panel. - if config_elem.tag == 'section': - app.toolbox.load_section_tag_set( config_elem, tool_path, load_panel_dict=True ) - elif config_elem.tag == 'workflow': - app.toolbox.load_workflow_tag_set( config_elem, - app.toolbox.tool_panel, - app.toolbox.integrated_tool_panel, - load_panel_dict=True ) - elif config_elem.tag == 'tool': - guid = config_elem.get( 'guid' ) - app.toolbox.load_tool_tag_set( config_elem, - app.toolbox.tool_panel, - app.toolbox.integrated_tool_panel, - tool_path, - load_panel_dict=True, - guid=guid ) - # Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict. - shed_tool_conf_dict[ 'config_elems' ] = config_elems - app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict - if app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - app.toolbox.write_integrated_tool_panel_config_file() - app.toolbox_search = ToolBoxSearch( app.toolbox ) - def build_shed_tool_conf_select_field( app ): """Build a SelectField whose options are the keys in app.toolbox.shed_tool_confs.""" options = [] @@ -192,142 +129,6 @@ message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) return message -def generate_tool_panel_dict_for_new_install( tool_dicts, tool_section=None ): - """ - When installing a repository that contains tools, all tools must currently be defined - within the same tool section in the tool panel or outside of any sections. - """ - tool_panel_dict = {} - if tool_section: - section_id = tool_section.id - section_name = tool_section.name - section_version = tool_section.version or '' - else: - section_id = '' - section_name = '' - section_version = '' - for tool_dict in tool_dicts: - if tool_dict.get( 'add_to_tool_panel', True ): - guid = tool_dict[ 'guid' ] - tool_config = tool_dict[ 'tool_config' ] - tool_section_dict = dict( tool_config=tool_config, id=section_id, name=section_name, version=section_version ) - if guid in tool_panel_dict: - tool_panel_dict[ guid ].append( tool_section_dict ) - else: - tool_panel_dict[ guid ] = [ tool_section_dict ] - return tool_panel_dict - -def generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=None ): - """ - Create a dictionary of the following type for a single tool config file name. - The intent is to call this method for every tool config in a repository and - append each of these as entries to a tool panel dictionary for the repository. - This allows for each tool to be loaded into a different section in the tool panel. - {<Tool guid> : - [{ tool_config : <tool_config_file>, - id: <ToolSection id>, - version : <ToolSection version>, - name : <TooSection name>}]} - """ - tool_panel_dict = {} - file_name = basic_util.strip_path( tool_config ) - tool_section_dicts = generate_tool_section_dicts( tool_config=file_name, tool_sections=tool_sections ) - tool_panel_dict[ guid ] = tool_section_dicts - return tool_panel_dict - -def generate_tool_panel_elem_list( repository_name, repository_clone_url, changeset_revision, tool_panel_dict, - repository_tools_tups, owner='' ): - """Generate a list of ElementTree Element objects for each section or tool.""" - elem_list = [] - tool_elem = None - cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) - if not owner: - owner = suc.get_repository_owner( cleaned_repository_clone_url ) - tool_shed = cleaned_repository_clone_url.split( '/repos/' )[ 0 ].rstrip( '/' ) - for guid, tool_section_dicts in tool_panel_dict.items(): - for tool_section_dict in tool_section_dicts: - tool_section = None - inside_section = False - section_in_elem_list = False - if tool_section_dict[ 'id' ]: - inside_section = True - # Create a new section element only if we haven't already created it. - for index, elem in enumerate( elem_list ): - if elem.tag == 'section': - section_id = elem.get( 'id', None ) - if section_id == tool_section_dict[ 'id' ]: - section_in_elem_list = True - tool_section = elem - break - if tool_section is None: - tool_section = generate_tool_section_element_from_dict( tool_section_dict ) - # Find the tuple containing the current guid from the list of repository_tools_tups. - for repository_tool_tup in repository_tools_tups: - tool_file_path, tup_guid, tool = repository_tool_tup - if tup_guid == guid: - break - if inside_section: - tool_elem = suc.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - tool_section ) - else: - tool_elem = suc.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - None ) - if inside_section: - if section_in_elem_list: - elem_list[ index ] = tool_section - else: - elem_list.append( tool_section ) - else: - elem_list.append( tool_elem ) - return elem_list - -def generate_tool_section_dicts( tool_config=None, tool_sections=None ): - tool_section_dicts = [] - if tool_config is None: - tool_config = '' - if tool_sections: - for tool_section in tool_sections: - # The value of tool_section will be None if the tool is displayed outside - # of any sections in the tool panel. - if tool_section: - section_id = tool_section.id or '' - section_version = tool_section.version or '' - section_name = tool_section.name or '' - else: - section_id = '' - section_version = '' - section_name = '' - tool_section_dicts.append( dict( tool_config=tool_config, - id=section_id, - version=section_version, - name=section_name ) ) - else: - tool_section_dicts.append( dict( tool_config=tool_config, id='', version='', name='' ) ) - return tool_section_dicts - -def generate_tool_section_element_from_dict( tool_section_dict ): - # The value of tool_section_dict looks like the following. - # { id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>} - if tool_section_dict[ 'id' ]: - # Create a new tool section. - tool_section = XmlET.Element( 'section' ) - tool_section.attrib[ 'id' ] = tool_section_dict[ 'id' ] - tool_section.attrib[ 'name' ] = tool_section_dict[ 'name' ] - tool_section.attrib[ 'version' ] = tool_section_dict[ 'version' ] - else: - tool_section = None - return tool_section - def get_headers( fname, sep, count=60, is_multi_byte=False ): """Returns a list with the first 'count' lines split by 'sep'.""" headers = [] @@ -341,26 +142,6 @@ break return headers -def get_or_create_tool_section( toolbox, tool_panel_section_id, new_tool_panel_section_label=None ): - tool_panel_section_key = str( tool_panel_section_id ) - if tool_panel_section_key in toolbox.tool_panel: - # Appending a tool to an existing section in toolbox.tool_panel - tool_section = toolbox.tool_panel[ tool_panel_section_key ] - log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) - else: - # Appending a new section to toolbox.tool_panel - if new_tool_panel_section_label is None: - # This might add an ugly section label to the tool panel, but, oh well... - new_tool_panel_section_label = tool_panel_section_id - elem = XmlET.Element( 'section' ) - elem.attrib[ 'name' ] = new_tool_panel_section_label - elem.attrib[ 'id' ] = tool_panel_section_id - elem.attrib[ 'version' ] = '' - tool_section = galaxy.tools.ToolSection( elem ) - toolbox.tool_panel[ tool_panel_section_key ] = tool_section - log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) - return tool_panel_section_key, tool_section - def get_tool_path_install_dir( partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems ): for elem in config_elems: if elem.tag == 'tool': @@ -514,63 +295,6 @@ error = True return error, message -def handle_tool_panel_selection( toolbox, metadata, no_changes_checked, tool_panel_section_id, new_tool_panel_section_label ): - """ - Handle the selected tool panel location for loading tools included in tool shed repositories when installing - or reinstalling them. - """ - # Get the location in the tool panel in which each tool was originally loaded. - tool_section = None - tool_panel_section_key = None - if 'tools' in metadata: - # This forces everything to be loaded into the same section (or no section) in the tool panel. - if no_changes_checked: - # Make sure the no_changes check box overrides the new_tool_panel_section_label if the user checked the check - # box and entered something into the field. - new_tool_panel_section_label = None - if 'tool_panel_section' in metadata: - tool_panel_dict = metadata[ 'tool_panel_section' ] - if not tool_panel_dict: - tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) - else: - tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) - if tool_panel_dict: - # The tool_panel_dict is empty when tools exist but are not installed into a tool panel section. - tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] - tool_section_dict = tool_section_dicts[ 0 ] - original_section_id = tool_section_dict[ 'id' ] - original_section_name = tool_section_dict[ 'name' ] - if original_section_id: - tool_panel_section_key, tool_section = \ - get_or_create_tool_section( toolbox, - tool_panel_section_id=original_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) - else: - # The user elected to change the tool panel section to contain the tools. - tool_panel_section_key, tool_section = handle_tool_panel_section( toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) - return tool_section, tool_panel_section_key - -def handle_tool_panel_section( toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ): - """Return a ToolSection object retrieved from the current in-memory tool_panel.""" - # If tool_panel_section_id is received, the section exists in the tool panel. In this - # case, the value of the received tool_panel_section_id must be the id retrieved from a - # tool panel config (e.g., tool_conf.xml, which may have getext). If new_tool_panel_section_label - # is received, a new section will be added to the tool panel. - if new_tool_panel_section_label: - section_id = str( new_tool_panel_section_label.lower().replace( ' ', '_' ) ) - tool_panel_section_key, tool_section = \ - get_or_create_tool_section( toolbox, - tool_panel_section_id=section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) - elif tool_panel_section_id: - tool_panel_section_key = str( tool_panel_section_id ) - tool_section = toolbox.tool_panel[ tool_panel_section_key ] - else: - return None, None - return tool_panel_section_key, tool_section - def handle_tool_versions( app, tool_version_dicts, tool_shed_repository ): """ Using the list of tool_version_dicts retrieved from the tool shed (one per changeset @@ -741,170 +465,6 @@ if isinstance( tool.tool_action, UploadToolAction ): app.toolbox.reload_tool_by_id( tool_id ) -def remove_from_shed_tool_config( app, shed_tool_conf_dict, guids_to_remove ): - # A tool shed repository is being uninstalled so change the shed_tool_conf file. - # Parse the config file to generate the entire list of config_elems instead of - # using the in-memory list since it will be a subset of the entire list if one - # or more repositories have been deactivated. - shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] - tool_path = shed_tool_conf_dict[ 'tool_path' ] - config_elems = [] - tree, error_message = xml_util.parse_xml( shed_tool_conf ) - if tree: - root = tree.getroot() - for elem in root: - config_elems.append( elem ) - config_elems_to_remove = [] - for config_elem in config_elems: - if config_elem.tag == 'section': - tool_elems_to_remove = [] - for tool_elem in config_elem: - if tool_elem.get( 'guid' ) in guids_to_remove: - tool_elems_to_remove.append( tool_elem ) - for tool_elem in tool_elems_to_remove: - # Remove all of the appropriate tool sub-elements from the section element. - config_elem.remove( tool_elem ) - if len( config_elem ) < 1: - # Keep a list of all empty section elements so they can be removed. - config_elems_to_remove.append( config_elem ) - elif config_elem.tag == 'tool': - if config_elem.get( 'guid' ) in guids_to_remove: - config_elems_to_remove.append( config_elem ) - for config_elem in config_elems_to_remove: - config_elems.remove( config_elem ) - # Persist the altered in-memory version of the tool config. - suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) - -def remove_from_tool_panel( app, repository, shed_tool_conf, uninstall ): - """ - A tool shed repository is being deactivated or uninstalled, so handle tool panel - alterations accordingly. - """ - # Determine where the tools are currently defined in the tool panel and store this - # information so the tools can be displayed in the same way when the repository is - # activated or reinstalled. - tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ) - repository.metadata[ 'tool_panel_section' ] = tool_panel_dict - app.install_model.context.add( repository ) - app.install_model.context.flush() - # Create a list of guids for all tools that will be removed from the in-memory tool panel - # and config file on disk. - guids_to_remove = [ k for k in tool_panel_dict.keys() ] - # Remove the tools from the toolbox's tools_by_id dictionary. - for guid_to_remove in guids_to_remove: - if guid_to_remove in app.toolbox.tools_by_id: - del app.toolbox.tools_by_id[ guid_to_remove ] - index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( app, shed_tool_conf ) - if uninstall: - # Remove from the shed_tool_conf file on disk. - remove_from_shed_tool_config( app, shed_tool_conf_dict, guids_to_remove ) - config_elems = shed_tool_conf_dict[ 'config_elems' ] - config_elems_to_remove = [] - for config_elem in config_elems: - if config_elem.tag == 'section': - # Get the section key for the in-memory tool panel. - section_key = str( config_elem.get( "id" ) ) - # Generate the list of tool elements to remove. - tool_elems_to_remove = [] - for tool_elem in config_elem: - if tool_elem.get( 'guid' ) in guids_to_remove: - tool_elems_to_remove.append( tool_elem ) - for tool_elem in tool_elems_to_remove: - if tool_elem in config_elem: - # Remove the tool sub-element from the section element. - config_elem.remove( tool_elem ) - # Remove the tool from the section in the in-memory tool panel. - if section_key in app.toolbox.tool_panel: - tool_section = app.toolbox.tool_panel[ section_key ] - guid = tool_elem.get( 'guid' ) - tool_key = 'tool_%s' % str( guid ) - # Get the list of versions of this tool that are currently available in the toolbox. - available_tool_versions = app.toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in tool_section.elems: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with the - # newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - if available_tool_version.id in tool_section.elems.keys() or section_key == available_tool_section_id: - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_section_elems_index, key in enumerate( tool_section.elems.keys() ): - if key == tool_key: - break - # Remove the tool from the tool section. - del tool_section.elems[ tool_key ] - # Add the replacement tool at the same location in the tool section. - tool_section.elems.insert( tool_section_elems_index, - replacement_tool_key, - replacement_tool_version ) - else: - del tool_section.elems[ tool_key ] - else: - del tool_section.elems[ tool_key ] - if uninstall: - # Remove the tool from the section in the in-memory integrated tool panel. - if section_key in app.toolbox.integrated_tool_panel: - tool_section = app.toolbox.integrated_tool_panel[ section_key ] - tool_key = 'tool_%s' % str( tool_elem.get( 'guid' ) ) - if tool_key in tool_section.elems: - del tool_section.elems[ tool_key ] - if len( config_elem ) < 1: - # Keep a list of all empty section elements so they can be removed. - config_elems_to_remove.append( config_elem ) - elif config_elem.tag == 'tool': - guid = config_elem.get( 'guid' ) - if guid in guids_to_remove: - tool_key = 'tool_%s' % str( config_elem.get( 'guid' ) ) - # Get the list of versions of this tool that are currently available in the toolbox. - available_tool_versions = app.toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in app.toolbox.tool_panel: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with - # the newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - if available_tool_version.id in app.toolbox.tool_panel.keys() or not available_tool_section_id: - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( app.toolbox.tool_panel.keys() ): - if key == tool_key: - break - # Remove the tool from the tool panel. - del app.toolbox.tool_panel[ tool_key ] - # Add the replacement tool at the same location in the tool panel. - app.toolbox.tool_panel.insert( tool_panel_index, replacement_tool_key, replacement_tool_version ) - else: - del app.toolbox.tool_panel[ tool_key ] - else: - del app.toolbox.tool_panel[ tool_key ] - if uninstall: - if tool_key in app.toolbox.integrated_tool_panel: - del app.toolbox.integrated_tool_panel[ tool_key ] - config_elems_to_remove.append( config_elem ) - for config_elem in config_elems_to_remove: - # Remove the element from the in-memory list of elements. - config_elems.remove( config_elem ) - # Update the config_elems of the in-memory shed_tool_conf_dict. - shed_tool_conf_dict[ 'config_elems' ] = config_elems - app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict - app.toolbox_search = ToolBoxSearch( app.toolbox ) - if uninstall and app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - app.toolbox.write_integrated_tool_panel_config_file() - def reset_tool_data_tables( app ): # Reset the tool_data_tables to an empty dictionary. app.tool_data_tables.data_tables = {} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Some code cleanup for setting metadata on tool shed repositories.
by commits-noreply＠bitbucket.org 21 Jul '14

21 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0eaf1edd08e3/ Changeset: 0eaf1edd08e3 User: greg Date: 2014-07-21 19:38:43 Summary: Some code cleanup for setting metadata on tool shed repositories. Affected #: 11 files diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -413,12 +413,12 @@ # Make sure the current user's API key proves he is an admin user in this Galaxy instance. if not trans.user_is_admin(): raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) + irmm = InstalledRepositoryMetadataManager( trans.app ) + query = irmm.get_query_for_setting_metadata_on_repositories( order=False ) # Now reset metadata on all remaining repositories. for repository in query: repository_id = trans.security.encode_id( repository.id ) try: - irmm = InstalledRepositoryMetadataManager( trans.app ) invalid_file_tups, metadata_dict = irmm.reset_all_metadata_on_installed_repository( repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1708,13 +1708,13 @@ @web.expose @web.require_admin def reset_metadata_on_selected_installed_repositories( self, trans, **kwd ): + irmm = InstalledRepositoryMetadataManager( trans.app ) if 'reset_metadata_on_selected_repositories_button' in kwd: - irmm = InstalledRepositoryMetadataManager( trans.app ) message, status = irmm.reset_metadata_on_selected_repositories( trans.user, **kwd ) else: message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repositories_select_field = suc.build_repository_ids_select_field( trans ) + repositories_select_field = irmm.build_repository_ids_select_field( trans ) return trans.fill_template( '/admin/tool_shed_repository/reset_metadata_on_selected_repositories.mako', repositories_select_field=repositories_select_field, message=message, diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -392,7 +392,8 @@ my_writable = True handled_repository_ids = [] repository_ids = [] - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) + query = rmm.get_query_for_setting_metadata_on_repositories( my_writable=my_writable, order=False ) # Make sure repositories of type tool_dependency_definition are first in the list. for repository in query: if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: @@ -426,13 +427,12 @@ This param can be used as an alternative to the above encoded_ids_to_skip. """ - def handle_repository( trans, repository, results ): + def handle_repository( trans, rmm, repository, results ): log.debug( "Resetting metadata on repository %s" % str( repository.name ) ) repository_id = trans.security.encode_id( repository.id ) try: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) invalid_file_tups, metadata_dict = \ - rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, repository_id ) + rmm.reset_all_metadata_on_repository_in_tool_shed( repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, @@ -451,7 +451,7 @@ status = '%s : %s' % ( str( repository.name ), message ) results[ 'repository_status' ].append( status ) return results - + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) start_time = strftime( "%Y-%m-%d %H:%M:%S" ) results = dict( start_time=start_time, repository_status=[], @@ -473,7 +473,7 @@ my_writable = util.asbool( payload.get( 'my_writable', False ) ) else: my_writable = True - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + query = rmm.get_query_for_setting_metadata_on_repositories( my_writable=my_writable, order=False ) # First reset metadata on all repositories of type repository_dependency_definition. for repository in query: encoded_id = trans.security.encode_id( repository.id ) @@ -481,7 +481,7 @@ log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ ( str( repository.id ), str( encoded_ids_to_skip ) ) ) elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) + results = handle_repository( trans, rmm, repository, results ) # Now reset metadata on all remaining repositories. for repository in query: encoded_id = trans.security.encode_id( repository.id ) @@ -489,7 +489,7 @@ log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ ( str( repository.id ), str( encoded_ids_to_skip ) ) ) elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) + results = handle_repository( trans, rmm, repository, results ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time return json.dumps( results, sort_keys=True, indent=4 ) @@ -511,10 +511,9 @@ results = dict( start_time=start_time, repository_status=[] ) try: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) invalid_file_tups, metadata_dict = \ - rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, - trans.security.encode_id( repository.id ) ) + rmm.reset_all_metadata_on_repository_in_tool_shed( trans.security.encode_id( repository.id ) ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -348,13 +348,16 @@ @web.expose @web.require_admin def reset_metadata_on_selected_repositories_in_tool_shed( self, trans, **kwd ): + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) if 'reset_metadata_on_selected_repositories_button' in kwd: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) - message, status = rmm.reset_metadata_on_selected_repositories( trans.user, **kwd ) + message, status = rmm.reset_metadata_on_selected_repositories( **kwd ) else: message = util.restore_text( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repositories_select_field = suc.build_repository_ids_select_field( trans ) + repositories_select_field = rmm.build_repository_ids_select_field( name='repository_ids', + multiple=True, + display='checkboxes', + my_writable=False ) return trans.fill_template( '/webapps/tool_shed/common/reset_metadata_on_selected_repositories.mako', repositories_select_field=repositories_select_field, message=message, diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/galaxy/webapps/tool_shed/controllers/hg.py --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -48,8 +48,8 @@ repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) ) update_repository( repo, ctx_rev=None ) # Set metadata using the repository files on disk. - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) - error_message, status = rmm.set_repository_metadata( trans.request.host, trans.user, repository ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) + error_message, status = rmm.set_repository_metadata( trans.request.host, repository ) if status == 'ok' and error_message: log.debug( "Successfully reset metadata on repository %s owned by %s, but encountered problem: %s" % \ ( str( repository.name ), str( repository.user.username ), error_message ) ) diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2753,9 +2753,9 @@ def reset_all_metadata( self, trans, id, **kwd ): """Reset all metadata on the complete changelog for a single repository in the tool shed.""" # This method is called only from the ~/templates/webapps/tool_shed/repository/manage_repository.mako template. - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) invalid_file_tups, metadata_dict = \ - rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, id, **kwd ) + rmm.reset_all_metadata_on_repository_in_tool_shed( id, **kwd ) if invalid_file_tups: repository = suc.get_repository_in_tool_shed( trans.app, id ) message = tool_util.generate_message_for_invalid_tools( trans.app, @@ -2774,13 +2774,16 @@ @web.expose def reset_metadata_on_my_writable_repositories_in_tool_shed( self, trans, **kwd ): + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) if 'reset_metadata_on_selected_repositories_button' in kwd: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) - message, status = rmm.reset_metadata_on_selected_repositories( trans.user, **kwd ) + message, status = rmm.reset_metadata_on_selected_repositories( **kwd ) else: message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repositories_select_field = suc.build_repository_ids_select_field( trans, my_writable=True ) + repositories_select_field = rmm.build_repository_ids_select_field( name='repository_ids', + multiple=True, + display='checkboxes', + my_writable=True ) return trans.fill_template( '/webapps/tool_shed/common/reset_metadata_on_selected_repositories.mako', repositories_select_field=repositories_select_field, message=message, @@ -2840,9 +2843,8 @@ if tip == repository.tip( trans.app ): message += 'No changes to repository. ' else: - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) status, error_message = rmm.set_repository_metadata_due_to_new_tip( trans.request.host, - trans.user, repository, **kwd ) if error_message: diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -262,10 +262,9 @@ ( len( files_to_remove ), upload_point ) else: message += " %d files were removed from the repository root. " % len( files_to_remove ) - rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) status, error_message = \ rmm.set_repository_metadata_due_to_new_tip( trans.request.host, - trans.user, repository, content_alert_str=content_alert_str, **kwd ) diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/tool_shed/capsule/capsule_manager.py --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -727,9 +727,8 @@ results_dict[ 'ok' ] = False results_dict[ 'error_message' ] += error_message try: - rmm = repository_metadata_manager.RepositoryMetadataManager( self.app ) + rmm = repository_metadata_manager.RepositoryMetadataManager( self.app, self.user ) status, error_message = rmm.set_repository_metadata_due_to_new_tip( self.host, - self.user, repository, content_alert_str=content_alert_str ) if error_message: diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -3,6 +3,7 @@ from galaxy import util from galaxy.util import inflector +from galaxy.web.form_builder import SelectField from tool_shed.metadata import metadata_generator @@ -19,6 +20,32 @@ super( InstalledRepositoryMetadataManager, self ).__init__( app ) self.app = app + def build_repository_ids_select_field( self, name='repository_ids', multiple=True, display='checkboxes' ): + """Generate the current list of repositories for resetting metadata.""" + repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) + query = self.get_query_for_setting_metadata_on_repositories( order=True ) + for repository in query: + owner = str( repository.owner ) + option_label = '%s (%s)' % ( str( repository.name ), owner ) + option_value = '%s' % self.app.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) + return repositories_select_field + + def get_query_for_setting_metadata_on_repositories( self, order=True ): + """ + Return a query containing repositories for resetting metadata. The order parameter + is used for displaying the list of repositories ordered alphabetically for display on + a page. When called from the Galaxy API, order is False. + """ + if order: + return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ + .filter( self.app.install_model.ToolShedRepository.table.c.uninstalled == False ) \ + .order_by( self.app.install_model.ToolShedRepository.table.c.name, + self.app.install_model.ToolShedRepository.table.c.owner ) + else: + return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ + .filter( self.app.install_model.ToolShedRepository.table.c.uninstalled == False ) + def reset_all_metadata_on_installed_repository( self, id ): """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" invalid_file_tups = [] diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/tool_shed/metadata/repository_metadata_manager.py --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -2,11 +2,13 @@ import os import tempfile +from galaxy import util from galaxy.util import inflector -from galaxy import util +from galaxy.web.form_builder import SelectField from tool_shed.metadata import metadata_generator from tool_shed.repository_types.metadata import TipOnly +from tool_shed.repository_types import util as rt_util from tool_shed.util import basic_util from tool_shed.util import common_util @@ -20,9 +22,10 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): - def __init__( self, app ): + def __init__( self, app, user ): super( RepositoryMetadataManager, self ).__init__( app ) self.app = app + self.user = user # Repository metadata comparisons for changeset revisions. self.EQUAL = 'equal' self.NO_METADATA = 'no metadata' @@ -47,6 +50,18 @@ self.sa_session.add( repository_metadata ) self.sa_session.flush() + def build_repository_ids_select_field( self, name='repository_ids', multiple=True, display='checkboxes', + my_writable=False ): + """Generate the current list of repositories for resetting metadata.""" + repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) + query = self.get_query_for_setting_metadata_on_repositories( my_writable=my_writable, order=True ) + for repository in query: + owner = str( repository.user.username ) + option_label = '%s (%s)' % ( str( repository.name ), owner ) + option_value = '%s' % self.app.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) + return repositories_select_field + def clean_repository_metadata( self, id, changeset_revisions ): # Delete all repository_metadata records associated with the repository that have # a changeset_revision that is not in changeset_revisions. We sometimes see multiple @@ -458,6 +473,54 @@ # The tool did not change through all of the changeset revisions. return old_id + def get_query_for_setting_metadata_on_repositories( self, my_writable=False, order=True ): + """ + Return a query containing repositories for resetting metadata. The order parameter + is used for displaying the list of repositories ordered alphabetically for display on + a page. When called from the Tool Shed API, order is False. + """ + # When called from the Tool Shed API, the metadata is reset on all repositories of types + # repository_suite_definition and tool_dependency_definition in addition to other selected + # repositories. + if my_writable: + username = self.user.username + clause_list = [] + for repository in self.sa_session.query( self.app.model.Repository ) \ + .filter( self.app.model.Repository.table.c.deleted == False ): + # Always reset metadata on all repositories of types repository_suite_definition and + # tool_dependency_definition. + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: + clause_list.append( self.app.model.Repository.table.c.id == repository.id ) + else: + allow_push = repository.allow_push( self.app ) + if allow_push: + # Include all repositories that are writable by the current user. + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + clause_list.append( self.app.model.Repository.table.c.id == repository.id ) + if clause_list: + if order: + return self.sa_session.query( self.app.model.Repository ) \ + .filter( or_( *clause_list ) ) \ + .order_by( self.app.model.Repository.table.c.name, + self.app.model.Repository.table.c.user_id ) + else: + return self.sa_session.query( self.app.model.Repository ) \ + .filter( or_( *clause_list ) ) + else: + # Return an empty query. + return self.sa_session.query( self.app.model.Repository ) \ + .filter( self.app.model.Repository.table.c.id == -1 ) + else: + if order: + return self.sa_session.query( self.app.model.Repository ) \ + .filter( self.app.model.Repository.table.c.deleted == False ) \ + .order_by( self.app.model.Repository.table.c.name, + self.app.model.Repository.table.c.user_id ) + else: + return self.sa_session.query( self.app.model.Repository ) \ + .filter( self.app.model.Repository.table.c.deleted == False ) + def new_datatypes_metadata_required( self, repository_metadata, metadata_dict ): """ Compare the last saved metadata for each datatype in the repository with the new metadata @@ -732,13 +795,13 @@ # repository_metadata table record is not needed. return False - def reset_all_metadata_on_repository_in_tool_shed( self, user, id ): + def reset_all_metadata_on_repository_in_tool_shed( self, id ): """Reset all metadata on a single repository in a tool shed.""" repository = suc.get_repository_in_tool_shed( self.app, id ) log.debug( "Resetting all metadata on repository: %s" % repository.name ) repo_dir = repository.repo_path( self.app ) repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( self.user, repository ) # The list of changeset_revisions refers to repository_metadata records that have been created # or updated. When the following loop completes, we'll delete all repository_metadata records # for this repository that do not have a changeset_revision value in this list. @@ -875,7 +938,7 @@ self.sa_session.add( repository_metadata ) self.sa_session.flush() - def reset_metadata_on_selected_repositories( self, user, **kwd ): + def reset_metadata_on_selected_repositories( self, **kwd ): """ Inspect the repository changelog to reset metadata for all appropriate changeset revisions. This method is called from both Galaxy and the Tool Shed. @@ -892,7 +955,7 @@ repository = suc.get_repository_in_tool_shed( self.app, repository_id ) owner = str( repository.user.username ) invalid_file_tups, metadata_dict = \ - self.reset_all_metadata_on_repository_in_tool_shed( user, repository_id ) + self.reset_all_metadata_on_repository_in_tool_shed( self.user, repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( self.app, invalid_file_tups, @@ -918,7 +981,7 @@ status = 'error' return message, status - def set_repository_metadata( self, host, user, repository, content_alert_str='', **kwd ): + def set_repository_metadata( self, host, repository, content_alert_str='', **kwd ): """ Set metadata using the repository's current disk files, returning specific error messages (if any) to alert the repository owner that the changeset has problems. @@ -926,7 +989,7 @@ message = '' status = 'done' encoded_id = self.app.security.encode_id( repository.id ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( self.user, repository ) repo_dir = repository.repo_path( self.app ) repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False ) metadata_dict, invalid_file_tups = \ @@ -1027,10 +1090,9 @@ tool_util.reset_tool_data_tables( self.app ) return message, status - def set_repository_metadata_due_to_new_tip( self, host, user, repository, content_alert_str=None, **kwd ): + def set_repository_metadata_due_to_new_tip( self, host, repository, content_alert_str=None, **kwd ): """Set metadata on the repository tip in the tool shed.""" error_message, status = self.set_repository_metadata( host, - user, repository, content_alert_str=content_alert_str, **kwd ) diff -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 -r 0eaf1edd08e3b79ebbbe68921baef2f94bb4804d lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -84,20 +84,6 @@ '${host}' """ -def build_repository_ids_select_field( trans, name='repository_ids', multiple=True, display='checkboxes', my_writable=False ): - """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" - repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) - query = get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=True ) - for repository in query: - if trans.webapp.name == 'tool_shed': - owner = str( repository.user.username ) - else: - owner = str( repository.owner ) - option_label = '%s (%s)' % ( str( repository.name ), owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - return repositories_select_field - def build_tool_dependencies_select_field( app, tool_shed_repository, name, multiple=True, display='checkboxes', uninstalled_only=False ): """Method called from Galaxy to generate the current list of tool dependency ids for an installed tool shed repository.""" tool_dependencies_select_field = SelectField( name=name, multiple=multiple, display=display ) @@ -645,66 +631,6 @@ prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids return prior_import_or_install_required_dict -def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=True ): - """ - Return a query containing repositories for resetting metadata. This method is called from both the - Tool Shed and Galaxy. The my_writable parameter is ignored unless called from the Tool Shed, and the - order parameter is used for displaying the list of repositories ordered alphabetically for display on - a page. When called from either the Tool Shed or Galaxy API, order is False. - """ - if trans.webapp.name == 'tool_shed': - # When called from the Tool Shed API, the metadata is reset on all repositories of types - # repository_suite_definition and tool_dependency_definition in addition to other selected - # repositories. - if my_writable: - username = trans.user.username - clause_list = [] - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ): - # Always reset metadata on all repositories of types repository_suite_definition and - # tool_dependency_definition. - if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: - clause_list.append( trans.model.Repository.table.c.id == repository.id ) - else: - allow_push = repository.allow_push( trans.app ) - if allow_push: - # Include all repositories that are writable by the current user. - allow_push_usernames = allow_push.split( ',' ) - if username in allow_push_usernames: - clause_list.append( trans.model.Repository.table.c.id == repository.id ) - if clause_list: - if order: - return trans.sa_session.query( trans.model.Repository ) \ - .filter( or_( *clause_list ) ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ) - else: - return trans.sa_session.query( trans.model.Repository ) \ - .filter( or_( *clause_list ) ) - else: - # Return an empty query. - return trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.id == -1 ) - else: - if order: - return trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ) - else: - return trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) - else: - # We're in Galaxy. - if order: - return trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .filter( trans.install_model.ToolShedRepository.table.c.uninstalled == False ) \ - .order_by( trans.install_model.ToolShedRepository.table.c.name, - trans.install_model.ToolShedRepository.table.c.owner ) - else: - return trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .filter( trans.install_model.ToolShedRepository.table.c.uninstalled == False ) - def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a ToolPanelManager for Galaxy installs from the Tool Shed.
by commits-noreply＠bitbucket.org 21 Jul '14

21 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f8bf7a37c694/ Changeset: f8bf7a37c694 User: greg Date: 2014-07-21 17:36:29 Summary: Add a ToolPanelManager for Galaxy installs from the Tool Shed. Affected #: 9 files diff -r cbca66cc16bfed165f3b38399857ad0b85d14489 -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -30,6 +30,7 @@ from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import tool_panel_manager log = logging.getLogger( __name__ ) @@ -244,10 +245,10 @@ if kwd.get( 'deactivate_or_uninstall_repository_button', False ): if tool_shed_repository.includes_tools_for_display_in_tool_panel: # Handle tool panel alterations. - tool_util.remove_from_tool_panel( trans.app, - tool_shed_repository, - shed_tool_conf, - uninstall=remove_from_disk_checked ) + tpm = tool_panel_manager.ToolPanelManager( trans.app ) + tpm.remove_from_tool_panel( tool_shed_repository, + shed_tool_conf, + uninstall=remove_from_disk_checked ) if tool_shed_repository.includes_data_managers: data_manager_util.remove_from_data_manager( trans.app, tool_shed_repository ) if tool_shed_repository.includes_datatypes: @@ -1336,13 +1337,14 @@ # its repository dependencies. includes_tool_dependencies = tool_shed_repository.includes_tool_dependencies if tool_shed_repository.includes_tools_for_display_in_tool_panel: + tpm = tool_panel_manager.ToolPanelManager( trans.app ) # Handle the selected tool panel location for loading tools included in the tool shed repository. tool_section, tool_panel_section_key = \ - tool_util.handle_tool_panel_selection( toolbox=trans.app.toolbox, - metadata=metadata, - no_changes_checked=no_changes_checked, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + tpm.handle_tool_panel_selection( toolbox=trans.app.toolbox, + metadata=metadata, + no_changes_checked=no_changes_checked, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) if tool_section is not None: # Just in case the tool_section.id differs from tool_panel_section_id, which it shouldn't... tool_panel_section_id = str( tool_section.id ) @@ -1942,19 +1944,19 @@ updating_installed_repository=True, persist=True ) if 'tools' in metadata_dict: + tpm = tool_panel_manager.ToolPanelManager( trans.app ) tool_panel_dict = metadata_dict.get( 'tool_panel_section', None ) if tool_panel_dict is None: tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( trans.app, repository ) repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) - tool_util.add_to_tool_panel( app=trans.app, - repository_name=str( repository.name ), - repository_clone_url=repository_clone_url, - changeset_revision=str( repository.installed_changeset_revision ), - repository_tools_tups=repository_tools_tups, - owner=str( repository.owner ), - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=False ) + tpm.add_to_tool_panel( repository_name=str( repository.name ), + repository_clone_url=repository_clone_url, + changeset_revision=str( repository.installed_changeset_revision ), + repository_tools_tups=repository_tools_tups, + owner=str( repository.owner ), + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=False ) # Add new Data Manager entries if 'data_manager' in metadata_dict: new_data_managers = data_manager_util.install_data_managers( trans.app, diff -r cbca66cc16bfed165f3b38399857ad0b85d14489 -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -33,6 +33,7 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager +from tool_shed.galaxy_install.tools import tool_panel_manager log = logging.getLogger( __name__ ) @@ -535,7 +536,8 @@ self.app.config.shed_tool_data_table_config, persist=True ) if 'tools' in metadata_dict: - tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) + tpm = tool_panel_manager.ToolPanelManager( self.app ) + tool_panel_dict = tpm.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) sample_files = metadata_dict.get( 'sample_files', [] ) tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) @@ -555,16 +557,18 @@ sample_files_copied ) # Copy remaining sample files included in the repository to the ~/tool-data directory of the # local Galaxy instance. - tool_util.copy_sample_files( self.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) - tool_util.add_to_tool_panel( app=self.app, - repository_name=tool_shed_repository.name, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - repository_tools_tups=repository_tools_tups, - owner=tool_shed_repository.owner, - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=True ) + tool_util.copy_sample_files( self.app, + sample_files, + tool_path=tool_path, + sample_files_copied=sample_files_copied ) + tpm.add_to_tool_panel( repository_name=tool_shed_repository.name, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + repository_tools_tups=repository_tools_tups, + owner=tool_shed_repository.owner, + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=True ) if 'data_manager' in metadata_dict: new_data_managers = data_manager_util.install_data_managers( self.app, self.app.config.shed_data_manager_config_file, @@ -644,10 +648,11 @@ tool_shed_url = installation_dict[ 'tool_shed_url' ] # Handle contained tools. if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section_label or tool_panel_section_id ): + tpm = tool_panel_manager.ToolPanelManager( self.app ) tool_panel_section_key, tool_section = \ - tool_util.handle_tool_panel_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + tpm.handle_tool_panel_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) else: tool_panel_section_key = None tool_section = None diff -r cbca66cc16bfed165f3b38399857ad0b85d14489 -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -11,11 +11,11 @@ from tool_shed.util import datatype_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util -from tool_shed.util import tool_util from tool_shed.util import xml_util from galaxy.model.orm import and_ from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import tool_panel_manager log = logging.getLogger( __name__ ) @@ -80,19 +80,19 @@ repository.deleted = False repository.status = self.install_model.ToolShedRepository.installation_status.INSTALLED if repository.includes_tools_for_display_in_tool_panel: + tpm = tool_panel_manager.ToolPanelManager( self.app ) metadata = repository.metadata repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata ) # Reload tools into the appropriate tool panel section. tool_panel_dict = repository.metadata[ 'tool_panel_section' ] - tool_util.add_to_tool_panel( self.app, - repository.name, - repository_clone_url, - repository.installed_changeset_revision, - repository_tools_tups, - repository.owner, - shed_tool_conf, - tool_panel_dict, - new_install=False ) + tpm.add_to_tool_panel( repository.name, + repository_clone_url, + repository.installed_changeset_revision, + repository_tools_tups, + repository.owner, + shed_tool_conf, + tool_panel_dict, + new_install=False ) if repository.includes_data_managers: tp, data_manager_relative_install_dir = repository.get_tool_relative_path( self.app ) # Hack to add repository.name here, which is actually the root of the installed repository diff -r cbca66cc16bfed165f3b38399857ad0b85d14489 -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -5,13 +5,14 @@ from tool_shed.galaxy_install import install_manager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager +from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import shed_util_common as suc from tool_shed.util import repository_maintenance_util from tool_shed.util import tool_dependency_util -from tool_shed.util import tool_util + class RepairRepositoryManager(): @@ -119,10 +120,11 @@ tool_panel_section_id = section_dict[ 'id' ] tool_panel_section_name = section_dict[ 'name' ] if tool_panel_section_id: + tpm = tool_panel_manager.ToolPanelManager( self.app ) tool_panel_section_key, tool_panel_section = \ - tool_util.get_or_create_tool_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=tool_panel_section_name ) + tpm.get_or_create_tool_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=tool_panel_section_name ) else: tool_dependencies = None repo_info_dict = repository_maintenance_util.create_repo_info_dict( app=self.app, @@ -156,15 +158,17 @@ log.debug( error_message ) repair_dict [ repository.name ] = error_message elif repository.status not in [ self.app.install_model.ToolShedRepository.installation_status.INSTALLED ]: - shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) + shed_tool_conf, tool_path, relative_install_dir = \ + suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) # Reset the repository attributes to the New state for installation. if metadata: + tpm = tool_panel_manager.ToolPanelManager( self.app ) tool_section, tool_panel_section_key = \ - tool_util.handle_tool_panel_selection( self.app.toolbox, - metadata, - no_changes_checked=True, - tool_panel_section_id=None, - new_tool_panel_section_label=None ) + tpm.handle_tool_panel_selection( self.app.toolbox, + metadata, + no_changes_checked=True, + tool_panel_section_id=None, + new_tool_panel_section_label=None ) else: # The tools will be loaded outside of any sections in the tool panel. tool_panel_section_key = None diff -r cbca66cc16bfed165f3b38399857ad0b85d14489 -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -10,11 +10,12 @@ from galaxy.util import asbool +from tool_shed.galaxy_install.tools import tool_panel_manager + from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import encoding_util from tool_shed.util import shed_util_common as suc -from tool_shed.util import tool_util log = logging.getLogger( __name__ ) @@ -216,19 +217,20 @@ if can_update_db_record: # The database record for the tool shed repository currently being processed can be updated. # Get the repository metadata to see where it was previously located in the tool panel. + tpm = tool_panel_manager.ToolPanelManager( self.app ) if repository_db_record and repository_db_record.metadata: tool_section, tool_panel_section_key = \ - tool_util.handle_tool_panel_selection( toolbox=self.app.toolbox, - metadata=repository_db_record.metadata, - no_changes_checked=no_changes_checked, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + tpm.handle_tool_panel_selection( toolbox=self.app.toolbox, + metadata=repository_db_record.metadata, + no_changes_checked=no_changes_checked, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) else: # We're installing a new tool shed repository that does not yet have a database record. tool_panel_section_key, tool_section = \ - tool_util.handle_tool_panel_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + tpm.handle_tool_panel_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) tool_shed_repository = \ suc.create_or_update_tool_shed_repository( app=self.app, name=name, diff -r cbca66cc16bfed165f3b38399857ad0b85d14489 -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -14,6 +14,7 @@ from tool_shed.galaxy_install import install_manager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager +from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.util import basic_util from tool_shed.util import common_util @@ -30,23 +31,31 @@ class ToolMigrationManager( object ): - def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ): + def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, + install_dependencies ): """ - Check tool settings in tool_shed_install_config and install all repositories that are not already installed. The tool - panel configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. + Check tool settings in tool_shed_install_config and install all repositories + that are not already installed. The tool panel configuration file is the received + migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. """ self.app = app self.toolbox = self.app.toolbox self.migrated_tools_config = migrated_tools_config - # If install_dependencies is True but tool_dependency_dir is not set, do not attempt to install but print informative error message. + self.tpm = tool_panel_manager.ToolPanelManager( self.app ) + # If install_dependencies is True but tool_dependency_dir is not set, do not attempt + # to install but print informative error message. if install_dependencies and app.config.tool_dependency_dir is None: - message = 'You are attempting to install tool dependencies but do not have a value for "tool_dependency_dir" set in your universe_wsgi.ini ' - message += 'file. Set this location value to the path where you want tool dependencies installed and rerun the migration script.' + message = 'You are attempting to install tool dependencies but do not have a value ' + message += 'for "tool_dependency_dir" set in your universe_wsgi.ini file. Set this ' + message += 'location value to the path where you want tool dependencies installed and ' + message += 'rerun the migration script.' raise Exception( message ) - # Get the local non-shed related tool panel configs (there can be more than one, and the default name is tool_conf.xml). + # Get the local non-shed related tool panel configs (there can be more than one, and the + # default name is tool_conf.xml). self.proprietary_tool_confs = self.non_shed_tool_panel_configs self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number ) - # Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config. + # Set the location where the repositories will be installed by retrieving the tool_path + # setting from migrated_tools_config. tree, error_message = xml_util.parse_xml( migrated_tools_config ) if tree is None: print error_message @@ -66,21 +75,24 @@ self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config ) - # Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in - # tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk. - # The default behavior is that the tool shed is down. + # Since tool migration scripts can be executed any number of times, we need to + # make sure the appropriate tools are defined in tool_conf.xml. If no tools + # associated with the migration stage are defined, no repositories will be installed + # on disk. The default behavior is that the tool shed is down. tool_shed_accessible = False tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app ) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} - tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, - tool_panel_configs, - latest_migration_script_number ) + tool_shed_accessible, missing_tool_configs_dict = \ + common_util.check_for_missing_tools( app, + tool_panel_configs, + latest_migration_script_number ) else: - # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but - # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in - # the database. + # It doesn't matter if the tool shed is accessible since there are no migrated + # tools defined in the local Galaxy instance, but we have to set the value of + # tool_shed_accessible to True so that the value of migrate_tools.version can + # be correctly set in the database. tool_shed_accessible = True missing_tool_configs_dict = odict() if tool_shed_accessible: @@ -93,15 +105,18 @@ if missing_tool_configs_dict: for proprietary_tool_conf in self.proprietary_tool_confs: # Create a backup of the tool configuration in the un-migrated state. - shutil.copy( proprietary_tool_conf, '%s-pre-stage-%04d' % ( proprietary_tool_conf, latest_migration_script_number ) ) + shutil.copy( proprietary_tool_conf, '%s-pre-stage-%04d' % ( proprietary_tool_conf, + latest_migration_script_number ) ) for repository_elem in root: # Make sure we have a valid repository tag. if self.__is_valid_repository_tag( repository_elem ): - # Get all repository dependencies for the repository defined by the current repository_elem. Repository dependency - # definitions contained in tool shed repositories with migrated tools must never define a relationship to a repository - # dependency that contains a tool. The repository dependency can only contain items that are not loaded into the Galaxy - # tool panel (e.g., tool dependency definitions, custom datatypes, etc). This restriction must be followed down the - # entire dependency hierarchy. + # Get all repository dependencies for the repository defined by the + # current repository_elem. Repository dependency definitions contained + # in tool shed repositories with migrated tools must never define a + # relationship to a repository dependency that contains a tool. The + # repository dependency can only contain items that are not loaded into + # the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes, + # etc). This restriction must be followed down the entire dependency hierarchy. name = repository_elem.get( 'name' ) changeset_revision = repository_elem.get( 'changeset_revision' ) tool_shed_accessible, repository_dependencies_dict = \ @@ -110,30 +125,38 @@ name, self.repository_owner, changeset_revision ) - # Make sure all repository dependency records exist (as tool_shed_repository table rows) in the Galaxy database. - created_tool_shed_repositories = self.create_or_update_tool_shed_repository_records( name, - changeset_revision, - repository_dependencies_dict ) + # Make sure all repository dependency records exist (as tool_shed_repository + # table rows) in the Galaxy database. + created_tool_shed_repositories = \ + self.create_or_update_tool_shed_repository_records( name, + changeset_revision, + repository_dependencies_dict ) # Order the repositories for proper installation. This process is similar to the # process used when installing tool shed repositories, but does not handle managing # tool panel sections and other components since repository dependency definitions # contained in tool shed repositories with migrated tools must never define a relationship # to a repository dependency that contains a tool. - ordered_tool_shed_repositories = self.order_repositories_for_installation( created_tool_shed_repositories, - repository_dependencies_dict ) + ordered_tool_shed_repositories = \ + self.order_repositories_for_installation( created_tool_shed_repositories, + repository_dependencies_dict ) for tool_shed_repository in ordered_tool_shed_repositories: - is_repository_dependency = self.__is_repository_dependency( name, changeset_revision, tool_shed_repository ) + is_repository_dependency = self.__is_repository_dependency( name, + changeset_revision, + tool_shed_repository ) self.install_repository( repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency=is_repository_dependency ) else: - message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number ) - message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names ) + message = "\nNo tools associated with migration stage %s are defined in your " % \ + str( latest_migration_script_number ) + message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \ + ( plural, file_names ) print message else: - message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number ) + message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \ + str( latest_migration_script_number ) message += "Try again later.\n" print message @@ -354,16 +377,21 @@ repo_install_dir = relative_install_dir if not is_repository_dependency: for tool_elem in repository_elem: - # The tool_elem looks something like this: <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> + # The tool_elem looks something like this: + # <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> tool_config = tool_elem.get( 'file' ) guid = self.get_guid( repository_clone_url, relative_install_dir, tool_config ) # See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems. is_displayed, tool_sections = self.get_containing_tool_sections( tool_config ) if is_displayed: tool_panel_dict_for_tool_config = \ - tool_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections ) + self.tpm.generate_tool_panel_dict_for_tool_config( guid, + tool_config, + tool_sections=tool_sections ) # The tool-panel_dict has the following structure. - # {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, + # {<Tool guid> : [{ tool_config : <tool_config_file>, + # id: <ToolSection id>, + # version : <ToolSection version>, # name : <TooSection name>}]} for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[ k ] = v @@ -433,15 +461,14 @@ tool_path=self.tool_path, sample_files_copied=sample_files_copied ) if not is_repository_dependency: - tool_util.add_to_tool_panel( self.app, - tool_shed_repository.name, - repository_clone_url, - tool_shed_repository.installed_changeset_revision, - repository_tools_tups, - self.repository_owner, - self.migrated_tools_config, - tool_panel_dict=tool_panel_dict_for_display, - new_install=True ) + self.tpm.add_to_tool_panel( tool_shed_repository.name, + repository_clone_url, + tool_shed_repository.installed_changeset_revision, + repository_tools_tups, + self.repository_owner, + self.migrated_tools_config, + tool_panel_dict=tool_panel_dict_for_display, + new_install=True ) if install_dependencies and tool_dependencies and has_tool_dependencies: # Install tool dependencies. suc.update_tool_shed_repository_status( self.app, diff -r cbca66cc16bfed165f3b38399857ad0b85d14489 -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -0,0 +1,465 @@ +import logging +import os + +import galaxy.tools +from galaxy.tools.search import ToolBoxSearch +from xml.etree import ElementTree as XmlET + +from tool_shed.util import basic_util +from tool_shed.util import common_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import xml_util + +log = logging.getLogger( __name__ ) + + +class ToolPanelManager( object ): + + def __init__( self, app ): + self.app = app + + def add_to_shed_tool_config( self, shed_tool_conf_dict, elem_list ): + # A tool shed repository is being installed so change the shed_tool_conf file. Parse the + # config file to generate the entire list of config_elems instead of using the in-memory list + # since it will be a subset of the entire list if one or more repositories have been deactivated. + shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] + tool_path = shed_tool_conf_dict[ 'tool_path' ] + config_elems = [] + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + config_elems.append( elem ) + # Add the elements to the in-memory list of config_elems. + for elem_entry in elem_list: + config_elems.append( elem_entry ) + # Persist the altered shed_tool_config file. + suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path ) + + def add_to_tool_panel( self, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner, + shed_tool_conf, tool_panel_dict, new_install=True ): + """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" + # We need to change the in-memory version and the file system version of the shed_tool_conf file. + index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + tool_path = shed_tool_conf_dict[ 'tool_path' ] + # Generate the list of ElementTree Element objects for each section or tool. + elem_list = self.generate_tool_panel_elem_list( repository_name, + repository_clone_url, + changeset_revision, + tool_panel_dict, + repository_tools_tups, + owner=owner ) + if new_install: + # Add the new elements to the shed_tool_conf file on disk. + self.add_to_shed_tool_config( shed_tool_conf_dict, elem_list ) + # Use the new elements to add entries to the + config_elems = shed_tool_conf_dict[ 'config_elems' ] + for config_elem in elem_list: + # Add the new elements to the in-memory list of config_elems. + config_elems.append( config_elem ) + # Load the tools into the in-memory tool panel. + if config_elem.tag == 'section': + self.app.toolbox.load_section_tag_set( config_elem, tool_path, load_panel_dict=True ) + elif config_elem.tag == 'workflow': + self.app.toolbox.load_workflow_tag_set( config_elem, + self.app.toolbox.tool_panel, + self.app.toolbox.integrated_tool_panel, + load_panel_dict=True ) + elif config_elem.tag == 'tool': + guid = config_elem.get( 'guid' ) + self.app.toolbox.load_tool_tag_set( config_elem, + self.app.toolbox.tool_panel, + self.app.toolbox.integrated_tool_panel, + tool_path, + load_panel_dict=True, + guid=guid ) + # Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict. + shed_tool_conf_dict[ 'config_elems' ] = config_elems + self.app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict + if self.app.config.update_integrated_tool_panel: + # Write the current in-memory version of the integrated_tool_panel.xml file to disk. + self.app.toolbox.write_integrated_tool_panel_config_file() + self.app.toolbox_search = ToolBoxSearch( self.app.toolbox ) + + def generate_tool_panel_dict_for_new_install( self, tool_dicts, tool_section=None ): + """ + When installing a repository that contains tools, all tools must currently be defined + within the same tool section in the tool panel or outside of any sections. + """ + tool_panel_dict = {} + if tool_section: + section_id = tool_section.id + section_name = tool_section.name + section_version = tool_section.version or '' + else: + section_id = '' + section_name = '' + section_version = '' + for tool_dict in tool_dicts: + if tool_dict.get( 'add_to_tool_panel', True ): + guid = tool_dict[ 'guid' ] + tool_config = tool_dict[ 'tool_config' ] + tool_section_dict = dict( tool_config=tool_config, id=section_id, name=section_name, version=section_version ) + if guid in tool_panel_dict: + tool_panel_dict[ guid ].append( tool_section_dict ) + else: + tool_panel_dict[ guid ] = [ tool_section_dict ] + return tool_panel_dict + + def generate_tool_panel_dict_for_tool_config( self, guid, tool_config, tool_sections=None ): + """ + Create a dictionary of the following type for a single tool config file name. + The intent is to call this method for every tool config in a repository and + append each of these as entries to a tool panel dictionary for the repository. + This enables each tool to be loaded into a different section in the tool panel. + {<Tool guid> : + [{ tool_config : <tool_config_file>, + id: <ToolSection id>, + version : <ToolSection version>, + name : <TooSection name>}]} + """ + tool_panel_dict = {} + file_name = basic_util.strip_path( tool_config ) + tool_section_dicts = self. generate_tool_section_dicts( tool_config=file_name, + tool_sections=tool_sections ) + tool_panel_dict[ guid ] = tool_section_dicts + return tool_panel_dict + + def generate_tool_panel_elem_list( self, repository_name, repository_clone_url, changeset_revision, + tool_panel_dict, repository_tools_tups, owner='' ): + """Generate a list of ElementTree Element objects for each section or tool.""" + elem_list = [] + tool_elem = None + cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) + if not owner: + owner = suc.get_repository_owner( cleaned_repository_clone_url ) + tool_shed = cleaned_repository_clone_url.split( '/repos/' )[ 0 ].rstrip( '/' ) + for guid, tool_section_dicts in tool_panel_dict.items(): + for tool_section_dict in tool_section_dicts: + tool_section = None + inside_section = False + section_in_elem_list = False + if tool_section_dict[ 'id' ]: + inside_section = True + # Create a new section element only if we haven't already created it. + for index, elem in enumerate( elem_list ): + if elem.tag == 'section': + section_id = elem.get( 'id', None ) + if section_id == tool_section_dict[ 'id' ]: + section_in_elem_list = True + tool_section = elem + break + if tool_section is None: + tool_section = self.generate_tool_section_element_from_dict( tool_section_dict ) + # Find the tuple containing the current guid from the list of repository_tools_tups. + for repository_tool_tup in repository_tools_tups: + tool_file_path, tup_guid, tool = repository_tool_tup + if tup_guid == guid: + break + if inside_section: + tool_elem = suc.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + tool_section ) + else: + tool_elem = suc.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + None ) + if inside_section: + if section_in_elem_list: + elem_list[ index ] = tool_section + else: + elem_list.append( tool_section ) + else: + elem_list.append( tool_elem ) + return elem_list + + def generate_tool_section_dicts( self, tool_config=None, tool_sections=None ): + tool_section_dicts = [] + if tool_config is None: + tool_config = '' + if tool_sections: + for tool_section in tool_sections: + # The value of tool_section will be None if the tool is displayed outside + # of any sections in the tool panel. + if tool_section: + section_id = tool_section.id or '' + section_version = tool_section.version or '' + section_name = tool_section.name or '' + else: + section_id = '' + section_version = '' + section_name = '' + tool_section_dicts.append( dict( tool_config=tool_config, + id=section_id, + version=section_version, + name=section_name ) ) + else: + tool_section_dicts.append( dict( tool_config=tool_config, id='', version='', name='' ) ) + return tool_section_dicts + + def generate_tool_section_element_from_dict( self, tool_section_dict ): + # The value of tool_section_dict looks like the following. + # { id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>} + if tool_section_dict[ 'id' ]: + # Create a new tool section. + tool_section = XmlET.Element( 'section' ) + tool_section.attrib[ 'id' ] = tool_section_dict[ 'id' ] + tool_section.attrib[ 'name' ] = tool_section_dict[ 'name' ] + tool_section.attrib[ 'version' ] = tool_section_dict[ 'version' ] + else: + tool_section = None + return tool_section + + def get_or_create_tool_section( self, toolbox, tool_panel_section_id, new_tool_panel_section_label=None ): + tool_panel_section_key = str( tool_panel_section_id ) + if tool_panel_section_key in toolbox.tool_panel: + # Appending a tool to an existing section in toolbox.tool_panel + tool_section = toolbox.tool_panel[ tool_panel_section_key ] + log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) + else: + # Appending a new section to toolbox.tool_panel + if new_tool_panel_section_label is None: + # This might add an ugly section label to the tool panel, but, oh well... + new_tool_panel_section_label = tool_panel_section_id + elem = XmlET.Element( 'section' ) + elem.attrib[ 'name' ] = new_tool_panel_section_label + elem.attrib[ 'id' ] = tool_panel_section_id + elem.attrib[ 'version' ] = '' + tool_section = galaxy.tools.ToolSection( elem ) + toolbox.tool_panel[ tool_panel_section_key ] = tool_section + log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) + return tool_panel_section_key, tool_section + + def handle_tool_panel_section( self, toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ): + """Return a ToolSection object retrieved from the current in-memory tool_panel.""" + # If tool_panel_section_id is received, the section exists in the tool panel. In this + # case, the value of the received tool_panel_section_id must be the id retrieved from a + # tool panel config (e.g., tool_conf.xml, which may have getext). If new_tool_panel_section_label + # is received, a new section will be added to the tool panel. + if new_tool_panel_section_label: + section_id = str( new_tool_panel_section_label.lower().replace( ' ', '_' ) ) + tool_panel_section_key, tool_section = \ + self.get_or_create_tool_section( toolbox, + tool_panel_section_id=section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) + elif tool_panel_section_id: + tool_panel_section_key = str( tool_panel_section_id ) + tool_section = toolbox.tool_panel[ tool_panel_section_key ] + else: + return None, None + return tool_panel_section_key, tool_section + + def handle_tool_panel_selection( self, toolbox, metadata, no_changes_checked, tool_panel_section_id, + new_tool_panel_section_label ): + """ + Handle the selected tool panel location for loading tools included in tool shed + repositories when installing or reinstalling them. + """ + # Get the location in the tool panel in which each tool was originally loaded. + tool_section = None + tool_panel_section_key = None + if 'tools' in metadata: + # This forces everything to be loaded into the same section (or no section) + # in the tool panel. + if no_changes_checked: + # Make sure the no_changes check box overrides the new_tool_panel_section_label + # if the user checked the check box and entered something into the field. + new_tool_panel_section_label = None + if 'tool_panel_section' in metadata: + tool_panel_dict = metadata[ 'tool_panel_section' ] + if not tool_panel_dict: + tool_panel_dict = self.generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) + else: + tool_panel_dict = self.generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) + if tool_panel_dict: + # The tool_panel_dict is empty when tools exist but are not installed into a tool panel section. + tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] + tool_section_dict = tool_section_dicts[ 0 ] + original_section_id = tool_section_dict[ 'id' ] + original_section_name = tool_section_dict[ 'name' ] + if original_section_id: + tool_panel_section_key, tool_section = \ + self.get_or_create_tool_section( toolbox, + tool_panel_section_id=original_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) + else: + # The user elected to change the tool panel section to contain the tools. + tool_panel_section_key, tool_section = \ + self.handle_tool_panel_section( toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) + return tool_section, tool_panel_section_key + + def remove_from_shed_tool_config( self, shed_tool_conf_dict, guids_to_remove ): + # A tool shed repository is being uninstalled so change the shed_tool_conf file. + # Parse the config file to generate the entire list of config_elems instead of + # using the in-memory list since it will be a subset of the entire list if one + # or more repositories have been deactivated. + shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] + tool_path = shed_tool_conf_dict[ 'tool_path' ] + config_elems = [] + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + config_elems.append( elem ) + config_elems_to_remove = [] + for config_elem in config_elems: + if config_elem.tag == 'section': + tool_elems_to_remove = [] + for tool_elem in config_elem: + if tool_elem.get( 'guid' ) in guids_to_remove: + tool_elems_to_remove.append( tool_elem ) + for tool_elem in tool_elems_to_remove: + # Remove all of the appropriate tool sub-elements from the section element. + config_elem.remove( tool_elem ) + if len( config_elem ) < 1: + # Keep a list of all empty section elements so they can be removed. + config_elems_to_remove.append( config_elem ) + elif config_elem.tag == 'tool': + if config_elem.get( 'guid' ) in guids_to_remove: + config_elems_to_remove.append( config_elem ) + for config_elem in config_elems_to_remove: + config_elems.remove( config_elem ) + # Persist the altered in-memory version of the tool config. + suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path ) + + def remove_from_tool_panel( self, repository, shed_tool_conf, uninstall ): + """ + A tool shed repository is being deactivated or uninstalled, so handle tool panel + alterations accordingly. + """ + # Determine where the tools are currently defined in the tool panel and store this + # information so the tools can be displayed in the same way when the repository is + # activated or reinstalled. + tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( self.app, repository ) + repository.metadata[ 'tool_panel_section' ] = tool_panel_dict + self.app.install_model.context.add( repository ) + self.app.install_model.context.flush() + # Create a list of guids for all tools that will be removed from the in-memory tool panel + # and config file on disk. + guids_to_remove = [ k for k in tool_panel_dict.keys() ] + # Remove the tools from the toolbox's tools_by_id dictionary. + for guid_to_remove in guids_to_remove: + if guid_to_remove in self.app.toolbox.tools_by_id: + del self.app.toolbox.tools_by_id[ guid_to_remove ] + index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + if uninstall: + # Remove from the shed_tool_conf file on disk. + self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) + config_elems = shed_tool_conf_dict[ 'config_elems' ] + config_elems_to_remove = [] + for config_elem in config_elems: + if config_elem.tag == 'section': + # Get the section key for the in-memory tool panel. + section_key = str( config_elem.get( "id" ) ) + # Generate the list of tool elements to remove. + tool_elems_to_remove = [] + for tool_elem in config_elem: + if tool_elem.get( 'guid' ) in guids_to_remove: + tool_elems_to_remove.append( tool_elem ) + for tool_elem in tool_elems_to_remove: + if tool_elem in config_elem: + # Remove the tool sub-element from the section element. + config_elem.remove( tool_elem ) + # Remove the tool from the section in the in-memory tool panel. + if section_key in self.app.toolbox.tool_panel: + tool_section = self.app.toolbox.tool_panel[ section_key ] + guid = tool_elem.get( 'guid' ) + tool_key = 'tool_%s' % str( guid ) + # Get the list of versions of this tool that are currently available in the toolbox. + available_tool_versions = self.app.toolbox.get_loaded_tools_by_lineage( guid ) + if tool_key in tool_section.elems: + if available_tool_versions: + available_tool_versions.reverse() + replacement_tool_key = None + replacement_tool_version = None + # Since we are going to remove the tool from the section, replace it with the + # newest loaded version of the tool. + for available_tool_version in available_tool_versions: + available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() + if available_tool_version.id in tool_section.elems.keys() or section_key == available_tool_section_id: + replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) + replacement_tool_version = available_tool_version + break + if replacement_tool_key and replacement_tool_version: + # Get the index of the tool_key in the tool_section. + for tool_section_elems_index, key in enumerate( tool_section.elems.keys() ): + if key == tool_key: + break + # Remove the tool from the tool section. + del tool_section.elems[ tool_key ] + # Add the replacement tool at the same location in the tool section. + tool_section.elems.insert( tool_section_elems_index, + replacement_tool_key, + replacement_tool_version ) + else: + del tool_section.elems[ tool_key ] + else: + del tool_section.elems[ tool_key ] + if uninstall: + # Remove the tool from the section in the in-memory integrated tool panel. + if section_key in self.app.toolbox.integrated_tool_panel: + tool_section = self.app.toolbox.integrated_tool_panel[ section_key ] + tool_key = 'tool_%s' % str( tool_elem.get( 'guid' ) ) + if tool_key in tool_section.elems: + del tool_section.elems[ tool_key ] + if len( config_elem ) < 1: + # Keep a list of all empty section elements so they can be removed. + config_elems_to_remove.append( config_elem ) + elif config_elem.tag == 'tool': + guid = config_elem.get( 'guid' ) + if guid in guids_to_remove: + tool_key = 'tool_%s' % str( config_elem.get( 'guid' ) ) + # Get the list of versions of this tool that are currently available in the toolbox. + available_tool_versions = self.app.toolbox.get_loaded_tools_by_lineage( guid ) + if tool_key in self.app.toolbox.tool_panel: + if available_tool_versions: + available_tool_versions.reverse() + replacement_tool_key = None + replacement_tool_version = None + # Since we are going to remove the tool from the section, replace it with + # the newest loaded version of the tool. + for available_tool_version in available_tool_versions: + available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() + if available_tool_version.id in self.app.toolbox.tool_panel.keys() or not available_tool_section_id: + replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) + replacement_tool_version = available_tool_version + break + if replacement_tool_key and replacement_tool_version: + # Get the index of the tool_key in the tool_section. + for tool_panel_index, key in enumerate( self.app.toolbox.tool_panel.keys() ): + if key == tool_key: + break + # Remove the tool from the tool panel. + del self.app.toolbox.tool_panel[ tool_key ] + # Add the replacement tool at the same location in the tool panel. + self.app.toolbox.tool_panel.insert( tool_panel_index, + replacement_tool_key, + replacement_tool_version ) + else: + del self.app.toolbox.tool_panel[ tool_key ] + else: + del self.app.toolbox.tool_panel[ tool_key ] + if uninstall: + if tool_key in self.app.toolbox.integrated_tool_panel: + del self.app.toolbox.integrated_tool_panel[ tool_key ] + config_elems_to_remove.append( config_elem ) + for config_elem in config_elems_to_remove: + # Remove the element from the in-memory list of elements. + config_elems.remove( config_elem ) + # Update the config_elems of the in-memory shed_tool_conf_dict. + shed_tool_conf_dict[ 'config_elems' ] = config_elems + self.app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict + self.app.toolbox_search = ToolBoxSearch( self.app.toolbox ) + if uninstall and self.app.config.update_integrated_tool_panel: + # Write the current in-memory version of the integrated_tool_panel.xml file to disk. + self.app.toolbox.write_integrated_tool_panel_config_file() diff -r cbca66cc16bfed165f3b38399857ad0b85d14489 -r f8bf7a37c69469d3b4350e0d6b1965cc87dc7a49 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -19,69 +19,6 @@ log = logging.getLogger( __name__ ) -def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ): - # A tool shed repository is being installed so change the shed_tool_conf file. Parse the - # config file to generate the entire list of config_elems instead of using the in-memory list - # since it will be a subset of the entire list if one or more repositories have been deactivated. - shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] - tool_path = shed_tool_conf_dict[ 'tool_path' ] - config_elems = [] - tree, error_message = xml_util.parse_xml( shed_tool_conf ) - if tree: - root = tree.getroot() - for elem in root: - config_elems.append( elem ) - # Add the elements to the in-memory list of config_elems. - for elem_entry in elem_list: - config_elems.append( elem_entry ) - # Persist the altered shed_tool_config file. - suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) - -def add_to_tool_panel( app, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner, - shed_tool_conf, tool_panel_dict, new_install=True ): - """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" - # We need to change the in-memory version and the file system version of the shed_tool_conf file. - index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( app, shed_tool_conf ) - tool_path = shed_tool_conf_dict[ 'tool_path' ] - # Generate the list of ElementTree Element objects for each section or tool. - elem_list = generate_tool_panel_elem_list( repository_name, - repository_clone_url, - changeset_revision, - tool_panel_dict, - repository_tools_tups, - owner=owner ) - if new_install: - # Add the new elements to the shed_tool_conf file on disk. - add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ) - # Use the new elements to add entries to the - config_elems = shed_tool_conf_dict[ 'config_elems' ] - for config_elem in elem_list: - # Add the new elements to the in-memory list of config_elems. - config_elems.append( config_elem ) - # Load the tools into the in-memory tool panel. - if config_elem.tag == 'section': - app.toolbox.load_section_tag_set( config_elem, tool_path, load_panel_dict=True ) - elif config_elem.tag == 'workflow': - app.toolbox.load_workflow_tag_set( config_elem, - app.toolbox.tool_panel, - app.toolbox.integrated_tool_panel, - load_panel_dict=True ) - elif config_elem.tag == 'tool': - guid = config_elem.get( 'guid' ) - app.toolbox.load_tool_tag_set( config_elem, - app.toolbox.tool_panel, - app.toolbox.integrated_tool_panel, - tool_path, - load_panel_dict=True, - guid=guid ) - # Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict. - shed_tool_conf_dict[ 'config_elems' ] = config_elems - app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict - if app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - app.toolbox.write_integrated_tool_panel_config_file() - app.toolbox_search = ToolBoxSearch( app.toolbox ) - def build_shed_tool_conf_select_field( app ): """Build a SelectField whose options are the keys in app.toolbox.shed_tool_confs.""" options = [] @@ -192,142 +129,6 @@ message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) return message -def generate_tool_panel_dict_for_new_install( tool_dicts, tool_section=None ): - """ - When installing a repository that contains tools, all tools must currently be defined - within the same tool section in the tool panel or outside of any sections. - """ - tool_panel_dict = {} - if tool_section: - section_id = tool_section.id - section_name = tool_section.name - section_version = tool_section.version or '' - else: - section_id = '' - section_name = '' - section_version = '' - for tool_dict in tool_dicts: - if tool_dict.get( 'add_to_tool_panel', True ): - guid = tool_dict[ 'guid' ] - tool_config = tool_dict[ 'tool_config' ] - tool_section_dict = dict( tool_config=tool_config, id=section_id, name=section_name, version=section_version ) - if guid in tool_panel_dict: - tool_panel_dict[ guid ].append( tool_section_dict ) - else: - tool_panel_dict[ guid ] = [ tool_section_dict ] - return tool_panel_dict - -def generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=None ): - """ - Create a dictionary of the following type for a single tool config file name. - The intent is to call this method for every tool config in a repository and - append each of these as entries to a tool panel dictionary for the repository. - This allows for each tool to be loaded into a different section in the tool panel. - {<Tool guid> : - [{ tool_config : <tool_config_file>, - id: <ToolSection id>, - version : <ToolSection version>, - name : <TooSection name>}]} - """ - tool_panel_dict = {} - file_name = basic_util.strip_path( tool_config ) - tool_section_dicts = generate_tool_section_dicts( tool_config=file_name, tool_sections=tool_sections ) - tool_panel_dict[ guid ] = tool_section_dicts - return tool_panel_dict - -def generate_tool_panel_elem_list( repository_name, repository_clone_url, changeset_revision, tool_panel_dict, - repository_tools_tups, owner='' ): - """Generate a list of ElementTree Element objects for each section or tool.""" - elem_list = [] - tool_elem = None - cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) - if not owner: - owner = suc.get_repository_owner( cleaned_repository_clone_url ) - tool_shed = cleaned_repository_clone_url.split( '/repos/' )[ 0 ].rstrip( '/' ) - for guid, tool_section_dicts in tool_panel_dict.items(): - for tool_section_dict in tool_section_dicts: - tool_section = None - inside_section = False - section_in_elem_list = False - if tool_section_dict[ 'id' ]: - inside_section = True - # Create a new section element only if we haven't already created it. - for index, elem in enumerate( elem_list ): - if elem.tag == 'section': - section_id = elem.get( 'id', None ) - if section_id == tool_section_dict[ 'id' ]: - section_in_elem_list = True - tool_section = elem - break - if tool_section is None: - tool_section = generate_tool_section_element_from_dict( tool_section_dict ) - # Find the tuple containing the current guid from the list of repository_tools_tups. - for repository_tool_tup in repository_tools_tups: - tool_file_path, tup_guid, tool = repository_tool_tup - if tup_guid == guid: - break - if inside_section: - tool_elem = suc.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - tool_section ) - else: - tool_elem = suc.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - None ) - if inside_section: - if section_in_elem_list: - elem_list[ index ] = tool_section - else: - elem_list.append( tool_section ) - else: - elem_list.append( tool_elem ) - return elem_list - -def generate_tool_section_dicts( tool_config=None, tool_sections=None ): - tool_section_dicts = [] - if tool_config is None: - tool_config = '' - if tool_sections: - for tool_section in tool_sections: - # The value of tool_section will be None if the tool is displayed outside - # of any sections in the tool panel. - if tool_section: - section_id = tool_section.id or '' - section_version = tool_section.version or '' - section_name = tool_section.name or '' - else: - section_id = '' - section_version = '' - section_name = '' - tool_section_dicts.append( dict( tool_config=tool_config, - id=section_id, - version=section_version, - name=section_name ) ) - else: - tool_section_dicts.append( dict( tool_config=tool_config, id='', version='', name='' ) ) - return tool_section_dicts - -def generate_tool_section_element_from_dict( tool_section_dict ): - # The value of tool_section_dict looks like the following. - # { id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>} - if tool_section_dict[ 'id' ]: - # Create a new tool section. - tool_section = XmlET.Element( 'section' ) - tool_section.attrib[ 'id' ] = tool_section_dict[ 'id' ] - tool_section.attrib[ 'name' ] = tool_section_dict[ 'name' ] - tool_section.attrib[ 'version' ] = tool_section_dict[ 'version' ] - else: - tool_section = None - return tool_section - def get_headers( fname, sep, count=60, is_multi_byte=False ): """Returns a list with the first 'count' lines split by 'sep'.""" headers = [] @@ -341,26 +142,6 @@ break return headers -def get_or_create_tool_section( toolbox, tool_panel_section_id, new_tool_panel_section_label=None ): - tool_panel_section_key = str( tool_panel_section_id ) - if tool_panel_section_key in toolbox.tool_panel: - # Appending a tool to an existing section in toolbox.tool_panel - tool_section = toolbox.tool_panel[ tool_panel_section_key ] - log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) - else: - # Appending a new section to toolbox.tool_panel - if new_tool_panel_section_label is None: - # This might add an ugly section label to the tool panel, but, oh well... - new_tool_panel_section_label = tool_panel_section_id - elem = XmlET.Element( 'section' ) - elem.attrib[ 'name' ] = new_tool_panel_section_label - elem.attrib[ 'id' ] = tool_panel_section_id - elem.attrib[ 'version' ] = '' - tool_section = galaxy.tools.ToolSection( elem ) - toolbox.tool_panel[ tool_panel_section_key ] = tool_section - log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) - return tool_panel_section_key, tool_section - def get_tool_path_install_dir( partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems ): for elem in config_elems: if elem.tag == 'tool': @@ -514,63 +295,6 @@ error = True return error, message -def handle_tool_panel_selection( toolbox, metadata, no_changes_checked, tool_panel_section_id, new_tool_panel_section_label ): - """ - Handle the selected tool panel location for loading tools included in tool shed repositories when installing - or reinstalling them. - """ - # Get the location in the tool panel in which each tool was originally loaded. - tool_section = None - tool_panel_section_key = None - if 'tools' in metadata: - # This forces everything to be loaded into the same section (or no section) in the tool panel. - if no_changes_checked: - # Make sure the no_changes check box overrides the new_tool_panel_section_label if the user checked the check - # box and entered something into the field. - new_tool_panel_section_label = None - if 'tool_panel_section' in metadata: - tool_panel_dict = metadata[ 'tool_panel_section' ] - if not tool_panel_dict: - tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) - else: - tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) - if tool_panel_dict: - # The tool_panel_dict is empty when tools exist but are not installed into a tool panel section. - tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] - tool_section_dict = tool_section_dicts[ 0 ] - original_section_id = tool_section_dict[ 'id' ] - original_section_name = tool_section_dict[ 'name' ] - if original_section_id: - tool_panel_section_key, tool_section = \ - get_or_create_tool_section( toolbox, - tool_panel_section_id=original_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) - else: - # The user elected to change the tool panel section to contain the tools. - tool_panel_section_key, tool_section = handle_tool_panel_section( toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) - return tool_section, tool_panel_section_key - -def handle_tool_panel_section( toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ): - """Return a ToolSection object retrieved from the current in-memory tool_panel.""" - # If tool_panel_section_id is received, the section exists in the tool panel. In this - # case, the value of the received tool_panel_section_id must be the id retrieved from a - # tool panel config (e.g., tool_conf.xml, which may have getext). If new_tool_panel_section_label - # is received, a new section will be added to the tool panel. - if new_tool_panel_section_label: - section_id = str( new_tool_panel_section_label.lower().replace( ' ', '_' ) ) - tool_panel_section_key, tool_section = \ - get_or_create_tool_section( toolbox, - tool_panel_section_id=section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) - elif tool_panel_section_id: - tool_panel_section_key = str( tool_panel_section_id ) - tool_section = toolbox.tool_panel[ tool_panel_section_key ] - else: - return None, None - return tool_panel_section_key, tool_section - def handle_tool_versions( app, tool_version_dicts, tool_shed_repository ): """ Using the list of tool_version_dicts retrieved from the tool shed (one per changeset @@ -741,170 +465,6 @@ if isinstance( tool.tool_action, UploadToolAction ): app.toolbox.reload_tool_by_id( tool_id ) -def remove_from_shed_tool_config( app, shed_tool_conf_dict, guids_to_remove ): - # A tool shed repository is being uninstalled so change the shed_tool_conf file. - # Parse the config file to generate the entire list of config_elems instead of - # using the in-memory list since it will be a subset of the entire list if one - # or more repositories have been deactivated. - shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] - tool_path = shed_tool_conf_dict[ 'tool_path' ] - config_elems = [] - tree, error_message = xml_util.parse_xml( shed_tool_conf ) - if tree: - root = tree.getroot() - for elem in root: - config_elems.append( elem ) - config_elems_to_remove = [] - for config_elem in config_elems: - if config_elem.tag == 'section': - tool_elems_to_remove = [] - for tool_elem in config_elem: - if tool_elem.get( 'guid' ) in guids_to_remove: - tool_elems_to_remove.append( tool_elem ) - for tool_elem in tool_elems_to_remove: - # Remove all of the appropriate tool sub-elements from the section element. - config_elem.remove( tool_elem ) - if len( config_elem ) < 1: - # Keep a list of all empty section elements so they can be removed. - config_elems_to_remove.append( config_elem ) - elif config_elem.tag == 'tool': - if config_elem.get( 'guid' ) in guids_to_remove: - config_elems_to_remove.append( config_elem ) - for config_elem in config_elems_to_remove: - config_elems.remove( config_elem ) - # Persist the altered in-memory version of the tool config. - suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) - -def remove_from_tool_panel( app, repository, shed_tool_conf, uninstall ): - """ - A tool shed repository is being deactivated or uninstalled, so handle tool panel - alterations accordingly. - """ - # Determine where the tools are currently defined in the tool panel and store this - # information so the tools can be displayed in the same way when the repository is - # activated or reinstalled. - tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ) - repository.metadata[ 'tool_panel_section' ] = tool_panel_dict - app.install_model.context.add( repository ) - app.install_model.context.flush() - # Create a list of guids for all tools that will be removed from the in-memory tool panel - # and config file on disk. - guids_to_remove = [ k for k in tool_panel_dict.keys() ] - # Remove the tools from the toolbox's tools_by_id dictionary. - for guid_to_remove in guids_to_remove: - if guid_to_remove in app.toolbox.tools_by_id: - del app.toolbox.tools_by_id[ guid_to_remove ] - index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( app, shed_tool_conf ) - if uninstall: - # Remove from the shed_tool_conf file on disk. - remove_from_shed_tool_config( app, shed_tool_conf_dict, guids_to_remove ) - config_elems = shed_tool_conf_dict[ 'config_elems' ] - config_elems_to_remove = [] - for config_elem in config_elems: - if config_elem.tag == 'section': - # Get the section key for the in-memory tool panel. - section_key = str( config_elem.get( "id" ) ) - # Generate the list of tool elements to remove. - tool_elems_to_remove = [] - for tool_elem in config_elem: - if tool_elem.get( 'guid' ) in guids_to_remove: - tool_elems_to_remove.append( tool_elem ) - for tool_elem in tool_elems_to_remove: - if tool_elem in config_elem: - # Remove the tool sub-element from the section element. - config_elem.remove( tool_elem ) - # Remove the tool from the section in the in-memory tool panel. - if section_key in app.toolbox.tool_panel: - tool_section = app.toolbox.tool_panel[ section_key ] - guid = tool_elem.get( 'guid' ) - tool_key = 'tool_%s' % str( guid ) - # Get the list of versions of this tool that are currently available in the toolbox. - available_tool_versions = app.toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in tool_section.elems: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with the - # newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - if available_tool_version.id in tool_section.elems.keys() or section_key == available_tool_section_id: - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_section_elems_index, key in enumerate( tool_section.elems.keys() ): - if key == tool_key: - break - # Remove the tool from the tool section. - del tool_section.elems[ tool_key ] - # Add the replacement tool at the same location in the tool section. - tool_section.elems.insert( tool_section_elems_index, - replacement_tool_key, - replacement_tool_version ) - else: - del tool_section.elems[ tool_key ] - else: - del tool_section.elems[ tool_key ] - if uninstall: - # Remove the tool from the section in the in-memory integrated tool panel. - if section_key in app.toolbox.integrated_tool_panel: - tool_section = app.toolbox.integrated_tool_panel[ section_key ] - tool_key = 'tool_%s' % str( tool_elem.get( 'guid' ) ) - if tool_key in tool_section.elems: - del tool_section.elems[ tool_key ] - if len( config_elem ) < 1: - # Keep a list of all empty section elements so they can be removed. - config_elems_to_remove.append( config_elem ) - elif config_elem.tag == 'tool': - guid = config_elem.get( 'guid' ) - if guid in guids_to_remove: - tool_key = 'tool_%s' % str( config_elem.get( 'guid' ) ) - # Get the list of versions of this tool that are currently available in the toolbox. - available_tool_versions = app.toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in app.toolbox.tool_panel: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with - # the newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - if available_tool_version.id in app.toolbox.tool_panel.keys() or not available_tool_section_id: - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( app.toolbox.tool_panel.keys() ): - if key == tool_key: - break - # Remove the tool from the tool panel. - del app.toolbox.tool_panel[ tool_key ] - # Add the replacement tool at the same location in the tool panel. - app.toolbox.tool_panel.insert( tool_panel_index, replacement_tool_key, replacement_tool_version ) - else: - del app.toolbox.tool_panel[ tool_key ] - else: - del app.toolbox.tool_panel[ tool_key ] - if uninstall: - if tool_key in app.toolbox.integrated_tool_panel: - del app.toolbox.integrated_tool_panel[ tool_key ] - config_elems_to_remove.append( config_elem ) - for config_elem in config_elems_to_remove: - # Remove the element from the in-memory list of elements. - config_elems.remove( config_elem ) - # Update the config_elems of the in-memory shed_tool_conf_dict. - shed_tool_conf_dict[ 'config_elems' ] = config_elems - app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict - app.toolbox_search = ToolBoxSearch( app.toolbox ) - if uninstall and app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - app.toolbox.write_integrated_tool_panel_config_file() - def reset_tool_data_tables( app ): # Reset the tool_data_tables to an empty dictionary. app.tool_data_tables.data_tables = {} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a missing function to the ToolValidator.
by commits-noreply＠bitbucket.org 18 Jul '14

18 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/cbca66cc16bf/ Changeset: cbca66cc16bf User: greg Date: 2014-07-18 22:44:54 Summary: Add a missing function to the ToolValidator. Affected #: 2 files diff -r 24437efed65c4db4b22b25df6b6d307ac5dcb2ff -r cbca66cc16bfed165f3b38399857ad0b85d14489 lib/tool_shed/tools/tool_validator.py --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -149,6 +149,45 @@ return tmp_filename return None + def get_list_of_copied_sample_files( self, repo, ctx, dir ): + """ + Find all sample files (files in the repository with the special .sample extension) + in the reversed repository manifest up to ctx. Copy each discovered file to dir and + return the list of filenames. If a .sample file was added in a changeset and then + deleted in a later changeset, it will be returned in the deleted_sample_files list. + The caller will set the value of app.config.tool_data_path to dir in order to load + the tools and generate metadata for them. + """ + deleted_sample_files = [] + sample_files = [] + for changeset in hg_util.reversed_upper_bounded_changelog( repo, ctx ): + changeset_ctx = repo.changectx( changeset ) + for ctx_file in changeset_ctx.files(): + ctx_file_name = basic_util.strip_path( ctx_file ) + # If we decide in the future that files deleted later in the changelog should + # not be used, we can use the following if statement. if ctx_file_name.endswith( '.sample' ) + # and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files: + if ctx_file_name.endswith( '.sample' ) and ctx_file_name not in sample_files: + fctx = hg_util.get_file_context_from_ctx( changeset_ctx, ctx_file ) + if fctx in [ 'DELETED' ]: + # Since the possibly future used if statement above is commented out, the + # same file that was initially added will be discovered in an earlier changeset + # in the change log and fall through to the else block below. In other words, + # if a file named blast2go.loc.sample was added in change set 0 and then deleted + # in changeset 3, the deleted file in changeset 3 will be handled here, but the + # later discovered file in changeset 0 will be handled in the else block below. + # In this way, the file contents will always be found for future tools even though + # the file was deleted. + if ctx_file_name not in deleted_sample_files: + deleted_sample_files.append( ctx_file_name ) + else: + sample_files.append( ctx_file_name ) + tmp_ctx_file_name = os.path.join( dir, ctx_file_name.replace( '.sample', '' ) ) + fh = open( tmp_ctx_file_name, 'wb' ) + fh.write( fctx.data() ) + fh.close() + return sample_files, deleted_sample_files + def handle_sample_files_and_load_tool_from_disk( self, repo_files_dir, repository_id, tool_config_filepath, work_dir ): """ Copy all sample files from disk to a temporary directory since the sample files may @@ -172,7 +211,7 @@ ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) # We're not currently doing anything with the returned list of deleted_sample_files here. It is # intended to help handle sample files that are in the manifest, but have been deleted from disk. - sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) + sample_files, deleted_sample_files = self.get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) if sample_files: self.app.config.tool_data_path = work_dir if 'tool_data_table_conf.xml.sample' in sample_files: diff -r 24437efed65c4db4b22b25df6b6d307ac5dcb2ff -r cbca66cc16bfed165f3b38399857ad0b85d14489 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -341,45 +341,6 @@ break return headers -def get_list_of_copied_sample_files( repo, ctx, dir ): - """ - Find all sample files (files in the repository with the special .sample extension) - in the reversed repository manifest up to ctx. Copy each discovered file to dir and - return the list of filenames. If a .sample file was added in a changeset and then - deleted in a later changeset, it will be returned in the deleted_sample_files list. - The caller will set the value of app.config.tool_data_path to dir in order to load - the tools and generate metadata for them. - """ - deleted_sample_files = [] - sample_files = [] - for changeset in hg_util.reversed_upper_bounded_changelog( repo, ctx ): - changeset_ctx = repo.changectx( changeset ) - for ctx_file in changeset_ctx.files(): - ctx_file_name = basic_util.strip_path( ctx_file ) - # If we decide in the future that files deleted later in the changelog should - # not be used, we can use the following if statement. if ctx_file_name.endswith( '.sample' ) - # and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files: - if ctx_file_name.endswith( '.sample' ) and ctx_file_name not in sample_files: - fctx = hg_util.get_file_context_from_ctx( changeset_ctx, ctx_file ) - if fctx in [ 'DELETED' ]: - # Since the possibly future used if statement above is commented out, the - # same file that was initially added will be discovered in an earlier changeset - # in the change log and fall through to the else block below. In other words, - # if a file named blast2go.loc.sample was added in change set 0 and then deleted - # in changeset 3, the deleted file in changeset 3 will be handled here, but the - # later discovered file in changeset 0 will be handled in the else block below. - # In this way, the file contents will always be found for future tools even though - # the file was deleted. - if ctx_file_name not in deleted_sample_files: - deleted_sample_files.append( ctx_file_name ) - else: - sample_files.append( ctx_file_name ) - tmp_ctx_file_name = os.path.join( dir, ctx_file_name.replace( '.sample', '' ) ) - fh = open( tmp_ctx_file_name, 'wb' ) - fh.write( fctx.data() ) - fh.close() - return sample_files, deleted_sample_files - def get_or_create_tool_section( toolbox, tool_panel_section_id, new_tool_panel_section_label=None ): tool_panel_section_key = str( tool_panel_section_id ) if tool_panel_section_key in toolbox.tool_panel: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0